switchroom 0.21.14 → 0.21.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +6 -2
- package/dist/auth-broker/index.js +6 -2
- package/dist/cli/notion-write-pretool.mjs +6 -2
- package/dist/cli/switchroom.js +999 -490
- package/dist/host-control/main.js +7 -3
- package/dist/vault/approvals/kernel-server.js +6 -2
- package/dist/vault/broker/server.js +6 -2
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +9 -0
- package/telegram-plugin/dist/gateway/gateway.js +10 -6
- package/vendor/hindsight-memory/scripts/lib/client.py +14 -0
- package/vendor/hindsight-memory/scripts/lib/config.py +22 -0
- package/vendor/hindsight-memory/scripts/lib/directives.py +45 -7
- package/vendor/hindsight-memory/scripts/lib/watermark.py +27 -0
- package/vendor/hindsight-memory/scripts/recall.py +335 -5
- package/vendor/hindsight-memory/scripts/reconcile_tail.py +4 -12
- package/vendor/hindsight-memory/scripts/retain.py +59 -3
- package/vendor/hindsight-memory/scripts/tests/test_config_retain_tool_calls_env.py +98 -0
- package/vendor/hindsight-memory/scripts/tests/test_directives.py +52 -0
- package/vendor/hindsight-memory/scripts/tests/test_incremental_sweep.py +293 -0
- package/vendor/hindsight-memory/scripts/tests/test_profile_capture_nudge.py +335 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_query_timestamp.py +376 -0
|
@@ -243,6 +243,58 @@ class FormatActiveDirectivesBlockTests(unittest.TestCase):
|
|
|
243
243
|
# The in-prompt footer is still emitted (agent-facing signal).
|
|
244
244
|
self.assertIn("(+4 more, omitted)", out)
|
|
245
245
|
|
|
246
|
+
def test_overflow_footer_is_loud_and_operator_directed(self):
|
|
247
|
+
"""memory-RFC P7 — the in-prompt overflow notice must be LOUD and tell
|
|
248
|
+
the agent to surface the drop to the operator this turn.
|
|
249
|
+
|
|
250
|
+
The pre-P7 block emitted only the quiet `(+N more, omitted)`
|
|
251
|
+
parenthetical, which reaches the agent but neither states that explicit
|
|
252
|
+
instructions were dropped nor directs the agent to relay it. This
|
|
253
|
+
asserts the OUTCOME of the loud in-turn channel: the rendered block, on
|
|
254
|
+
overflow, names the dropped count, the total, the cap, states the loss
|
|
255
|
+
in unmistakable terms, and instructs the agent to tell the operator.
|
|
256
|
+
A test that only checked `(+N more, omitted)` would still pass on the
|
|
257
|
+
pre-P7 code, so it would not guard this change; these assertions fail
|
|
258
|
+
without it.
|
|
259
|
+
"""
|
|
260
|
+
total = MAX_DIRECTIVES + 3
|
|
261
|
+
directives = [
|
|
262
|
+
_directive(f"d{i}", f"c{i}", priority=total - i) for i in range(total)
|
|
263
|
+
]
|
|
264
|
+
with patch("sys.stderr", new=StringIO()):
|
|
265
|
+
out = format_active_directives_block(directives)
|
|
266
|
+
# A loud, explicitly-labelled overflow banner (not a lone parenthetical).
|
|
267
|
+
self.assertIn("DIRECTIVE OVERFLOW", out)
|
|
268
|
+
# Names the loss in unmistakable terms, with count/total/cap.
|
|
269
|
+
self.assertIn("DROPPED", out)
|
|
270
|
+
self.assertIn("3 of", out) # omitted of total
|
|
271
|
+
self.assertIn(str(total), out)
|
|
272
|
+
self.assertIn(f"MAX_DIRECTIVES={MAX_DIRECTIVES}", out)
|
|
273
|
+
# Directs the agent to surface it to the OPERATOR this turn — the
|
|
274
|
+
# in-turn operator-visible channel P7 adds.
|
|
275
|
+
self.assertIn("operator", out.lower())
|
|
276
|
+
self.assertIn("ACTION", out)
|
|
277
|
+
# The literal marker is retained for the recall_log/doctor cross-checks
|
|
278
|
+
# and the dedup parser.
|
|
279
|
+
self.assertIn("(+3 more, omitted)", out)
|
|
280
|
+
|
|
281
|
+
def test_no_overflow_banner_when_under_cap(self):
|
|
282
|
+
"""The loud banner must appear ONLY on overflow — an under-cap block is
|
|
283
|
+
unchanged and carries neither the banner nor the action directive."""
|
|
284
|
+
directives = [_directive(f"d{i}", f"c{i}", priority=5 - i) for i in range(3)]
|
|
285
|
+
with patch("sys.stderr", new=StringIO()) as fake_err:
|
|
286
|
+
out = format_active_directives_block(directives)
|
|
287
|
+
self.assertNotIn("DIRECTIVE OVERFLOW", out)
|
|
288
|
+
self.assertNotIn("ACTION", out)
|
|
289
|
+
self.assertNotIn("more, omitted", out)
|
|
290
|
+
self.assertEqual(fake_err.getvalue(), "")
|
|
291
|
+
|
|
292
|
+
def test_max_directives_is_unchanged_by_p7(self):
|
|
293
|
+
"""P7 is the loud-channel PR only. Raising MAX_DIRECTIVES is a separate,
|
|
294
|
+
later change (RFC §5 P7: order is non-negotiable). Pin the constant so a
|
|
295
|
+
cap bump cannot ride in on this change unnoticed."""
|
|
296
|
+
self.assertEqual(MAX_DIRECTIVES, 30)
|
|
297
|
+
|
|
246
298
|
def test_count_omitted_directives_matches_the_rendered_footer(self):
|
|
247
299
|
"""The recall_log's `directives_omitted` number must equal what the
|
|
248
300
|
block actually dropped — it is the operator-visible record of it."""
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
"""Switchroom memory-RFC P1 — incremental SessionEnd sweep, OUTCOME tests.
|
|
2
|
+
|
|
3
|
+
Before P1 the SessionEnd hook called ``run_retain(force=True)`` which retained
|
|
4
|
+
the WHOLE transcript, duplicating content the per-window retains already landed
|
|
5
|
+
(RFC §1.2). P1 makes the forced chunked sweep slice only the transcript tail
|
|
6
|
+
after the committed watermark, degrading to the whole transcript on any failure
|
|
7
|
+
so the §4.3 hazard (a raise here DELETES a turn) is never triggered.
|
|
8
|
+
|
|
9
|
+
Stdlib-only (`python3 -m unittest discover tests/`). Every test drives the real
|
|
10
|
+
hook code against a FAKE in-process daemon (no network, no LLM) and asserts
|
|
11
|
+
OUTCOMES — the bytes/turns that actually landed in the bank — not that a code
|
|
12
|
+
path ran.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import shutil
|
|
18
|
+
import sys
|
|
19
|
+
import tempfile
|
|
20
|
+
import unittest
|
|
21
|
+
from unittest import mock
|
|
22
|
+
|
|
23
|
+
SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
24
|
+
if SCRIPTS_DIR not in sys.path:
|
|
25
|
+
sys.path.insert(0, SCRIPTS_DIR)
|
|
26
|
+
|
|
27
|
+
import retain # noqa: E402
|
|
28
|
+
from lib import watermark # noqa: E402
|
|
29
|
+
from lib.client import HindsightClient # noqa: E402
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class FakeDaemon:
|
|
33
|
+
"""Records retained documents by document_id with upsert semantics."""
|
|
34
|
+
|
|
35
|
+
def __init__(self):
|
|
36
|
+
self.docs = {} # document_id -> content
|
|
37
|
+
self.posts = [] # [(document_id, async_processing)]
|
|
38
|
+
|
|
39
|
+
def retain(self, bank_id, content, document_id="conversation", context=None,
|
|
40
|
+
metadata=None, tags=None, timeout=15, async_processing=True,
|
|
41
|
+
observation_scopes=None):
|
|
42
|
+
self.posts.append((document_id, async_processing))
|
|
43
|
+
# Upsert: same document_id overwrites (the daemon contract §1).
|
|
44
|
+
self.docs[document_id] = content
|
|
45
|
+
return {"ok": True}
|
|
46
|
+
|
|
47
|
+
def content_blob(self):
|
|
48
|
+
return "\n".join(self.docs.values())
|
|
49
|
+
|
|
50
|
+
def bytes_for(self, session_prefix: str) -> int:
|
|
51
|
+
"""Total stored content bytes across every document for a session."""
|
|
52
|
+
return sum(
|
|
53
|
+
len(c.encode("utf-8"))
|
|
54
|
+
for doc_id, c in self.docs.items()
|
|
55
|
+
if doc_id.startswith(session_prefix)
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def last_post_content(self) -> str:
|
|
59
|
+
return self.docs[self.posts[-1][0]]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _write_transcript(path, n_turns, prefix):
|
|
63
|
+
"""Flat-format JSONL: n_turns human turns, each user+assistant with a uuid."""
|
|
64
|
+
lines = []
|
|
65
|
+
for i in range(n_turns):
|
|
66
|
+
lines.append(json.dumps(
|
|
67
|
+
{"role": "user", "content": f"user turn {i}", "uuid": f"{prefix}-u{i}"}))
|
|
68
|
+
lines.append(json.dumps(
|
|
69
|
+
{"role": "assistant", "content": f"assistant turn {i}", "uuid": f"{prefix}-a{i}"}))
|
|
70
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
71
|
+
f.write("\n".join(lines))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _stdin(obj):
|
|
75
|
+
import io
|
|
76
|
+
return io.StringIO(json.dumps(obj))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class IncrementalSweepBase(unittest.TestCase):
|
|
80
|
+
def setUp(self):
|
|
81
|
+
self.tmp = tempfile.mkdtemp(prefix="hs-rfc-p1-")
|
|
82
|
+
self.plugin_root = os.path.join(self.tmp, "plugin_root")
|
|
83
|
+
self.home = os.path.join(self.tmp, "home")
|
|
84
|
+
self.data = os.path.join(self.tmp, "data")
|
|
85
|
+
self.transcripts = os.path.join(self.tmp, "transcripts")
|
|
86
|
+
for d in (self.plugin_root, self.home, self.data, self.transcripts):
|
|
87
|
+
os.makedirs(d)
|
|
88
|
+
|
|
89
|
+
self._write_settings(8) # default cadence for these tests; overridable
|
|
90
|
+
|
|
91
|
+
self.env = mock.patch.dict(os.environ, {
|
|
92
|
+
"CLAUDE_PLUGIN_ROOT": self.plugin_root,
|
|
93
|
+
"CLAUDE_PLUGIN_DATA": self.data,
|
|
94
|
+
"HOME": self.home,
|
|
95
|
+
"HINDSIGHT_PENDING_DIR": os.path.join(self.home, ".hindsight", "pending-retains"),
|
|
96
|
+
"HINDSIGHT_RETAINED_DIR": os.path.join(self.home, ".hindsight", "retained"),
|
|
97
|
+
"HINDSIGHT_INFLIGHT_LOCK": os.path.join(self.home, ".hindsight", "retain-inflight.lock"),
|
|
98
|
+
"HINDSIGHT_TRANSCRIPTS_DIR": self.transcripts,
|
|
99
|
+
}, clear=False)
|
|
100
|
+
self.env.start()
|
|
101
|
+
for k in list(os.environ):
|
|
102
|
+
if k.startswith("HINDSIGHT_") and k not in (
|
|
103
|
+
"HINDSIGHT_PENDING_DIR", "HINDSIGHT_RETAINED_DIR",
|
|
104
|
+
"HINDSIGHT_INFLIGHT_LOCK", "HINDSIGHT_TRANSCRIPTS_DIR",
|
|
105
|
+
):
|
|
106
|
+
os.environ.pop(k, None)
|
|
107
|
+
|
|
108
|
+
self.daemon = FakeDaemon()
|
|
109
|
+
self._patches = [
|
|
110
|
+
mock.patch.object(HindsightClient, "retain", self._fake_retain),
|
|
111
|
+
mock.patch("retain.get_api_url", return_value="http://fake"),
|
|
112
|
+
]
|
|
113
|
+
for p in self._patches:
|
|
114
|
+
p.start()
|
|
115
|
+
|
|
116
|
+
def _write_settings(self, every_n_turns: int):
|
|
117
|
+
settings = {
|
|
118
|
+
"autoRetain": True,
|
|
119
|
+
"retainMode": "chunked",
|
|
120
|
+
"retainEveryNTurns": every_n_turns,
|
|
121
|
+
"retainOverlapTurns": 0,
|
|
122
|
+
"bankId": "test-bank",
|
|
123
|
+
}
|
|
124
|
+
with open(os.path.join(self.plugin_root, "settings.json"), "w") as f:
|
|
125
|
+
json.dump(settings, f)
|
|
126
|
+
|
|
127
|
+
def _fake_retain(self, *a, **kw):
|
|
128
|
+
return self.daemon.retain(*a, **kw)
|
|
129
|
+
|
|
130
|
+
def tearDown(self):
|
|
131
|
+
for p in self._patches:
|
|
132
|
+
p.stop()
|
|
133
|
+
self.env.stop()
|
|
134
|
+
shutil.rmtree(self.tmp, ignore_errors=True)
|
|
135
|
+
|
|
136
|
+
def _hook(self, session):
|
|
137
|
+
return {
|
|
138
|
+
"session_id": session,
|
|
139
|
+
"transcript_path": os.path.join(self.transcripts, f"{session}.jsonl"),
|
|
140
|
+
"cwd": "/x",
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
def _fire_window(self, session, transcript_turns, turn_count):
|
|
144
|
+
"""Drive a live per-window Stop retain (advances the watermark)."""
|
|
145
|
+
_write_transcript(self._hook(session)["transcript_path"], transcript_turns, session)
|
|
146
|
+
with mock.patch("retain.increment_turn_count", return_value=turn_count), \
|
|
147
|
+
mock.patch("sys.stdin", _stdin(self._hook(session))):
|
|
148
|
+
retain.main()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class TestIncrementalSweep(IncrementalSweepBase):
|
|
152
|
+
|
|
153
|
+
def test_two_turn_no_watermark_sweep_retains_both_turns(self):
|
|
154
|
+
# A short session that never fired a per-window retain has no committed
|
|
155
|
+
# watermark, so the forced sweep must flush the WHOLE (2-turn) transcript
|
|
156
|
+
# (RFC §4.2 — short sessions still land on disk).
|
|
157
|
+
session = "sessShort"
|
|
158
|
+
_write_transcript(self._hook(session)["transcript_path"], 2, session)
|
|
159
|
+
self.assertIsNone(watermark.load(session)) # precondition: no watermark
|
|
160
|
+
|
|
161
|
+
result = retain.run_retain(self._hook(session), force=True)
|
|
162
|
+
|
|
163
|
+
self.assertEqual(result.get("status"), "ok")
|
|
164
|
+
blob = self.daemon.content_blob()
|
|
165
|
+
self.assertIn("user turn 0", blob)
|
|
166
|
+
self.assertIn("user turn 1", blob)
|
|
167
|
+
|
|
168
|
+
def test_thirty_turn_sweep_is_incremental_and_smaller_than_full(self):
|
|
169
|
+
# Windows fire at turns 8/16/24 (n=8, overlap 0), then SessionEnd sweeps.
|
|
170
|
+
# The sweep must contain ONLY turns after the last window's tail uuid,
|
|
171
|
+
# and the total bytes stored for the session must be strictly less than
|
|
172
|
+
# the pre-change (full-session sweep) total. This is the test that fails
|
|
173
|
+
# on the bug it guards: a non-incremental sweep re-stores all 30 turns,
|
|
174
|
+
# so the totals become equal and assertLess fires.
|
|
175
|
+
inc = "sessInc"
|
|
176
|
+
for fire_at in (8, 16, 24):
|
|
177
|
+
self._fire_window(inc, fire_at, fire_at)
|
|
178
|
+
# The last window committed its tail uuid as the watermark.
|
|
179
|
+
wm = watermark.load(inc)
|
|
180
|
+
self.assertIsNotNone(wm, "expected the per-window retains to commit a watermark")
|
|
181
|
+
self.assertEqual(wm["last_uuid"], f"{inc}-a23")
|
|
182
|
+
|
|
183
|
+
_write_transcript(self._hook(inc)["transcript_path"], 30, inc)
|
|
184
|
+
result = retain.run_retain(self._hook(inc), force=True)
|
|
185
|
+
self.assertEqual(result.get("status"), "ok")
|
|
186
|
+
|
|
187
|
+
sweep = self.daemon.last_post_content()
|
|
188
|
+
# Only turns AFTER the watermark (24..29); nothing at/ before it.
|
|
189
|
+
for i in range(24, 30):
|
|
190
|
+
self.assertIn(f"user turn {i}", sweep, f"turn {i} missing from incremental sweep")
|
|
191
|
+
for i in (0, 15, 23):
|
|
192
|
+
self.assertNotIn(f"user turn {i}", sweep,
|
|
193
|
+
f"turn {i} leaked into the incremental sweep")
|
|
194
|
+
|
|
195
|
+
# Pre-change baseline: identical scenario, but the SessionEnd sweep runs
|
|
196
|
+
# full-session (watermark unseen), on a distinct session id in the same
|
|
197
|
+
# daemon so window docs are directly comparable.
|
|
198
|
+
full = "sessFull"
|
|
199
|
+
for fire_at in (8, 16, 24):
|
|
200
|
+
self._fire_window(full, fire_at, fire_at)
|
|
201
|
+
_write_transcript(self._hook(full)["transcript_path"], 30, full)
|
|
202
|
+
with mock.patch("retain.watermark.load", return_value=None):
|
|
203
|
+
self.assertEqual(retain.run_retain(self._hook(full), force=True).get("status"), "ok")
|
|
204
|
+
|
|
205
|
+
self.assertLess(
|
|
206
|
+
self.daemon.bytes_for(inc),
|
|
207
|
+
self.daemon.bytes_for(full),
|
|
208
|
+
"incremental sweep did not reduce total retained bytes vs full sweep",
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
def test_corrupt_watermark_json_sweeps_whole_transcript_no_raise(self):
|
|
212
|
+
# A watermark file corrupted to invalid JSON must degrade to a whole
|
|
213
|
+
# -transcript sweep, and no exception may escape run_retain (§4.3).
|
|
214
|
+
session = "sessCorrupt"
|
|
215
|
+
_write_transcript(self._hook(session)["transcript_path"], 4, session)
|
|
216
|
+
retained_dir = os.environ["HINDSIGHT_RETAINED_DIR"]
|
|
217
|
+
os.makedirs(retained_dir, exist_ok=True)
|
|
218
|
+
with open(os.path.join(retained_dir, f"{session}.json"), "w") as f:
|
|
219
|
+
f.write("{ this is not valid json ]]]")
|
|
220
|
+
|
|
221
|
+
try:
|
|
222
|
+
result = retain.run_retain(self._hook(session), force=True)
|
|
223
|
+
except Exception as e: # noqa: BLE001 - the whole point is nothing escapes
|
|
224
|
+
self.fail(f"run_retain raised into the SessionEnd seam: {e!r}")
|
|
225
|
+
|
|
226
|
+
self.assertEqual(result.get("status"), "ok")
|
|
227
|
+
blob = self.daemon.content_blob()
|
|
228
|
+
for i in range(4):
|
|
229
|
+
self.assertIn(f"user turn {i}", blob)
|
|
230
|
+
|
|
231
|
+
def test_watermark_load_raising_degrades_to_full_sweep(self):
|
|
232
|
+
# Belt-and-braces for the §4.3 catch-all: even if the watermark READ
|
|
233
|
+
# itself raises an unexpected error, run_retain must NOT propagate it —
|
|
234
|
+
# it degrades to the whole-transcript sweep.
|
|
235
|
+
session = "sessBoom"
|
|
236
|
+
_write_transcript(self._hook(session)["transcript_path"], 4, session)
|
|
237
|
+
|
|
238
|
+
def _boom(_):
|
|
239
|
+
raise RuntimeError("simulated watermark read explosion")
|
|
240
|
+
|
|
241
|
+
with mock.patch("retain.watermark.load", side_effect=_boom):
|
|
242
|
+
try:
|
|
243
|
+
result = retain.run_retain(self._hook(session), force=True)
|
|
244
|
+
except Exception as e: # noqa: BLE001
|
|
245
|
+
self.fail(f"run_retain propagated a watermark read failure: {e!r}")
|
|
246
|
+
|
|
247
|
+
self.assertEqual(result.get("status"), "ok")
|
|
248
|
+
blob = self.daemon.content_blob()
|
|
249
|
+
for i in range(4):
|
|
250
|
+
self.assertIn(f"user turn {i}", blob)
|
|
251
|
+
|
|
252
|
+
def test_compacted_watermark_uuid_sweeps_whole_transcript(self):
|
|
253
|
+
# The watermark anchor was compacted out of the transcript: tail_after
|
|
254
|
+
# cannot find it and returns the whole transcript (a safe re-upsert).
|
|
255
|
+
session = "sessCompact"
|
|
256
|
+
_write_transcript(self._hook(session)["transcript_path"], 4, session)
|
|
257
|
+
# Commit a watermark whose uuid is NOT present in the transcript.
|
|
258
|
+
watermark.commit(session, "ghost-uuid-not-in-transcript", "doc-ghost",
|
|
259
|
+
ordered_uuids=["ghost-uuid-not-in-transcript"])
|
|
260
|
+
self.assertEqual(watermark.load(session)["last_uuid"], "ghost-uuid-not-in-transcript")
|
|
261
|
+
|
|
262
|
+
result = retain.run_retain(self._hook(session), force=True)
|
|
263
|
+
self.assertEqual(result.get("status"), "ok")
|
|
264
|
+
blob = self.daemon.content_blob()
|
|
265
|
+
for i in range(4):
|
|
266
|
+
self.assertIn(f"user turn {i}", blob)
|
|
267
|
+
|
|
268
|
+
def test_every_n_turns_1_is_byte_identical_full_session(self):
|
|
269
|
+
# At retainEveryNTurns==1 the document id is {session_id} and a tail slice
|
|
270
|
+
# would TRUNCATE it (RFC §4.2). The n==1 path must keep the full-session
|
|
271
|
+
# sweep even when a committed watermark exists — byte-identical to before.
|
|
272
|
+
self._write_settings(1)
|
|
273
|
+
session = "sessN1"
|
|
274
|
+
_write_transcript(self._hook(session)["transcript_path"], 4, session)
|
|
275
|
+
# A watermark exists (a prior every-turn fire) sitting mid-transcript.
|
|
276
|
+
watermark.commit(session, f"{session}-a1", "doc-prior",
|
|
277
|
+
ordered_uuids=[f"{session}-u{i//2}" if i % 2 == 0 else f"{session}-a{i//2}"
|
|
278
|
+
for i in range(8)])
|
|
279
|
+
self.assertEqual(watermark.load(session)["last_uuid"], f"{session}-a1")
|
|
280
|
+
|
|
281
|
+
result = retain.run_retain(self._hook(session), force=True)
|
|
282
|
+
self.assertEqual(result.get("status"), "ok")
|
|
283
|
+
|
|
284
|
+
# Whole transcript, under the plain {session_id} document id — the tail
|
|
285
|
+
# slice must NOT have applied.
|
|
286
|
+
self.assertIn(session, self.daemon.docs, "n==1 sweep must post under the {session_id} id")
|
|
287
|
+
content = self.daemon.docs[session]
|
|
288
|
+
for i in range(4):
|
|
289
|
+
self.assertIn(f"user turn {i}", content, f"turn {i} missing — n==1 sweep was truncated")
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
if __name__ == "__main__":
|
|
293
|
+
unittest.main()
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
"""RFC phase4 P3 — unit + end-to-end tests for the operator-profile capture nudge.
|
|
2
|
+
|
|
3
|
+
Ken's first stated want is "save memories about him". Auto-retain stores
|
|
4
|
+
transcript facts, but nothing gives a DETERMINISTIC signal that a durable
|
|
5
|
+
*profile fact* about the operator himself just went by — so profile capture is
|
|
6
|
+
left to model discretion (the same per-agent lottery Stage A measured for
|
|
7
|
+
directives). P3 mirrors the shipped directive-capture nudge (recall.py #2848):
|
|
8
|
+
a POSITIVE regex detects a first-person durable self-statement, a NEGATIVE
|
|
9
|
+
regex scrubs the two false-positive shapes (questions, attributions to others)
|
|
10
|
+
BEFORE the positive match, and on a hit the UserPromptSubmit hook appends a
|
|
11
|
+
terse advisory telling the model to persist it with an explicit retain tagged
|
|
12
|
+
`profile:ken` into the agent's OWN bank. Pure regex — no model callsite.
|
|
13
|
+
|
|
14
|
+
These tests pin (all OUTCOME assertions):
|
|
15
|
+
* True positives — the RFC's listed profile shapes fire.
|
|
16
|
+
* True negatives — questions and third-/second-party attributions do NOT
|
|
17
|
+
fire (the negative-lookaround guard scrubs them first).
|
|
18
|
+
* The advisory carries the `profile:ken` tag instruction and targets the
|
|
19
|
+
agent's OWN bank (not a shared / cross-agent person bank).
|
|
20
|
+
* End-to-end through recall.main(): the advisory reaches the emitted
|
|
21
|
+
additionalContext, the recall_log row carries `profile_nudge: true`, and
|
|
22
|
+
the config knob OFF (`profileCaptureNudge: false`) suppresses both.
|
|
23
|
+
|
|
24
|
+
Stdlib-only; runs under `python3 -m unittest discover tests/`. The end-to-end
|
|
25
|
+
harness mirrors test_recall_envelope_strip_telemetry.py.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
import io
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
import shutil
|
|
32
|
+
import sys
|
|
33
|
+
import tempfile
|
|
34
|
+
import unittest
|
|
35
|
+
from unittest.mock import patch
|
|
36
|
+
|
|
37
|
+
SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
38
|
+
if SCRIPTS_DIR not in sys.path:
|
|
39
|
+
sys.path.insert(0, SCRIPTS_DIR)
|
|
40
|
+
|
|
41
|
+
import recall # noqa: E402
|
|
42
|
+
from recall import ( # noqa: E402
|
|
43
|
+
_PROFILE_CAPTURE_NUDGE,
|
|
44
|
+
_combine_context,
|
|
45
|
+
_is_trivial_stateless,
|
|
46
|
+
looks_like_profile_statement,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# First-person durable self-statements that MUST fire the nudge. Drawn from the
|
|
51
|
+
# RFC P3 shapes ("I prefer …", "my … is …", "I always …", "remind me that I …")
|
|
52
|
+
# plus the conservative identity/situation set.
|
|
53
|
+
PROFILE_STATEMENTS = [
|
|
54
|
+
# --- stated preferences ---
|
|
55
|
+
"I prefer dark roast coffee",
|
|
56
|
+
"I'd prefer British spelling in my docs",
|
|
57
|
+
"my preference is tabs over spaces",
|
|
58
|
+
# --- durable self-facts: "my <ATTRIBUTE> is/are …" (tight allow-list) ---
|
|
59
|
+
"my timezone is Australia/Melbourne",
|
|
60
|
+
"my email is ken@example.com",
|
|
61
|
+
"my sister is Lisa",
|
|
62
|
+
"my kids are at school during the day",
|
|
63
|
+
"my name's Ken", # contraction form
|
|
64
|
+
# --- identity / situation ---
|
|
65
|
+
"I live in Melbourne",
|
|
66
|
+
"I work at Anthropic",
|
|
67
|
+
"I'm allergic to peanuts",
|
|
68
|
+
"I'm based in Australia",
|
|
69
|
+
# --- durable identity: diet / abstention / "I'm a <noun>" ---
|
|
70
|
+
"I'm a vegetarian",
|
|
71
|
+
"I don't eat meat",
|
|
72
|
+
"call me Ken",
|
|
73
|
+
# --- tastes (durable like/dislike framing) ---
|
|
74
|
+
"I hate em-dashes",
|
|
75
|
+
# --- durable habits ---
|
|
76
|
+
"I always take my coffee black",
|
|
77
|
+
"I usually work late on Thursdays",
|
|
78
|
+
"I never eat red meat",
|
|
79
|
+
# --- explicit memory framing about the operator himself ---
|
|
80
|
+
"remind me that I have a standing 9am standup",
|
|
81
|
+
"remember that I hate em-dashes",
|
|
82
|
+
]
|
|
83
|
+
|
|
84
|
+
# Questions and attributions that MUST NOT fire — the negative-lookaround guard
|
|
85
|
+
# scrubs these BEFORE the positive match. A false positive here is nudge noise.
|
|
86
|
+
NON_PROFILE = [
|
|
87
|
+
# --- questions about the operator (not statements of a durable fact) ---
|
|
88
|
+
"do I prefer tea or coffee?",
|
|
89
|
+
"what's my timezone?",
|
|
90
|
+
"where is my email address stored?",
|
|
91
|
+
"how do I fix this bug",
|
|
92
|
+
"should I always run the tests first?",
|
|
93
|
+
"what do I usually do here",
|
|
94
|
+
"remind me what my calendar looks like",
|
|
95
|
+
# --- attributions to a third / second party ---
|
|
96
|
+
"you said I prefer tea",
|
|
97
|
+
"she claims my code is broken",
|
|
98
|
+
"he thinks I always overcomplicate things",
|
|
99
|
+
"they told me my access was revoked",
|
|
100
|
+
# --- neither: no first-person durable self-fact ---
|
|
101
|
+
"the project timezone is UTC",
|
|
102
|
+
"please run the tests",
|
|
103
|
+
"what time is it",
|
|
104
|
+
# --- discourse-marker "my <X> is" — not a durable profile fact. The
|
|
105
|
+
# positive arm uses a TIGHT identity allow-list, so a free noun never
|
|
106
|
+
# reaches the matcher; these confirm that. ---
|
|
107
|
+
"my guess is the cache is stale",
|
|
108
|
+
"my point is that we should ship it",
|
|
109
|
+
"my concern is the timeout",
|
|
110
|
+
# --- transient dev state "my <transient> is/are …". These are the exact
|
|
111
|
+
# over-fires the free-`\w+` arm produced; the allow-list must NOT fire
|
|
112
|
+
# on them (RFC favour-false-negatives constraint on this agent). ---
|
|
113
|
+
"my container is down",
|
|
114
|
+
"my build is failing",
|
|
115
|
+
"my code is broken",
|
|
116
|
+
"my PR is ready",
|
|
117
|
+
"my worktree is dirty",
|
|
118
|
+
"my server is down",
|
|
119
|
+
"my deploy is stuck",
|
|
120
|
+
"my tests are green",
|
|
121
|
+
"my branch is merged",
|
|
122
|
+
# --- pleasantry embedding a bare always/never after "I" ---
|
|
123
|
+
"I always appreciate your help",
|
|
124
|
+
"I never enjoy waiting, but thanks",
|
|
125
|
+
# --- "I'm a <hedge>" is a transient mood, not an "I'm a <noun>" identity ---
|
|
126
|
+
"I'm a bit tired",
|
|
127
|
+
"I'm a little confused about the config",
|
|
128
|
+
"I'm a big fan of shipping fast",
|
|
129
|
+
# --- "call me <phrasing>" as a request, not a name form ---
|
|
130
|
+
"call me back later",
|
|
131
|
+
"call me when the build finishes",
|
|
132
|
+
# --- the <channel …> envelope wrapper on its own must never trigger ---
|
|
133
|
+
'<channel user="ken" chat_id="123">',
|
|
134
|
+
]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class TestProfileDetection(unittest.TestCase):
|
|
138
|
+
def test_profile_statements_fire_the_nudge(self):
|
|
139
|
+
for p in PROFILE_STATEMENTS:
|
|
140
|
+
with self.subTest(prompt=p):
|
|
141
|
+
self.assertTrue(
|
|
142
|
+
looks_like_profile_statement(p),
|
|
143
|
+
f"expected profile detection for {p!r}",
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
def test_questions_and_attributions_do_not_fire(self):
|
|
147
|
+
for p in NON_PROFILE:
|
|
148
|
+
with self.subTest(prompt=p):
|
|
149
|
+
self.assertFalse(
|
|
150
|
+
looks_like_profile_statement(p),
|
|
151
|
+
f"FALSE POSITIVE: profile nudge would fire for {p!r}",
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
def test_empty_and_non_string_are_false(self):
|
|
155
|
+
for bad in ("", " ", None, 123, [], {}):
|
|
156
|
+
with self.subTest(value=bad):
|
|
157
|
+
self.assertFalse(looks_like_profile_statement(bad))
|
|
158
|
+
|
|
159
|
+
def test_attribution_scrub_does_not_mask_a_real_self_fact(self):
|
|
160
|
+
# A message that opens with an attributed clause AND then states the
|
|
161
|
+
# operator's own durable fact must still fire — the negative guard
|
|
162
|
+
# scrubs only the attributed span (through end-of-sentence).
|
|
163
|
+
self.assertTrue(
|
|
164
|
+
looks_like_profile_statement(
|
|
165
|
+
"she thinks I'm wrong. my timezone is Melbourne"
|
|
166
|
+
)
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
def test_attribution_scrub_stops_at_a_comma(self):
|
|
170
|
+
# The attributed span must stop at a comma, not run to end-of-sentence:
|
|
171
|
+
# a real self-fact trailing the attributed clause in the SAME sentence
|
|
172
|
+
# must still reach the positive matcher (MINOR: greedy [^.?!]* → [^.?!,]*).
|
|
173
|
+
self.assertTrue(
|
|
174
|
+
looks_like_profile_statement(
|
|
175
|
+
"she said the deploy failed, my timezone is Melbourne"
|
|
176
|
+
)
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class TestProfileNudgeString(unittest.TestCase):
|
|
181
|
+
def test_nudge_carries_the_profile_ken_tag_instruction(self):
|
|
182
|
+
self.assertIn("profile:ken", _PROFILE_CAPTURE_NUDGE)
|
|
183
|
+
self.assertIn("retain", _PROFILE_CAPTURE_NUDGE)
|
|
184
|
+
self.assertIn("profile_capture_check", _PROFILE_CAPTURE_NUDGE)
|
|
185
|
+
|
|
186
|
+
def test_nudge_targets_the_agents_own_bank_not_a_shared_one(self):
|
|
187
|
+
# Constraint 2 forbids a cross-agent person bank — the advisory must
|
|
188
|
+
# route to the agent's OWN bank and say so explicitly.
|
|
189
|
+
self.assertIn("OWN bank", _PROFILE_CAPTURE_NUDGE)
|
|
190
|
+
self.assertIn("shared", _PROFILE_CAPTURE_NUDGE)
|
|
191
|
+
|
|
192
|
+
def test_combine_appends_nudge_after_recall_block(self):
|
|
193
|
+
base = "<hindsight_memories>\n…\n</hindsight_memories>"
|
|
194
|
+
out = _combine_context(base, _PROFILE_CAPTURE_NUDGE)
|
|
195
|
+
self.assertTrue(out.startswith(base))
|
|
196
|
+
self.assertTrue(out.endswith(_PROFILE_CAPTURE_NUDGE))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class TestTrivialSkipUnaffected(unittest.TestCase):
|
|
200
|
+
"""A profile statement carries personal/stateful signal and must never be
|
|
201
|
+
trivial-stateless-skipped; greetings still are."""
|
|
202
|
+
|
|
203
|
+
def test_profile_statements_are_never_trivial_skipped(self):
|
|
204
|
+
for p in PROFILE_STATEMENTS:
|
|
205
|
+
with self.subTest(prompt=p):
|
|
206
|
+
self.assertFalse(
|
|
207
|
+
_is_trivial_stateless("", p),
|
|
208
|
+
f"profile statement {p!r} was trivial-skipped",
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# --- End-to-end harness (mirrors test_recall_envelope_strip_telemetry.py) ---
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class _RecordingClient:
|
|
216
|
+
def __init__(self, memories=None, directives=None):
|
|
217
|
+
self._memories = memories if memories is not None else []
|
|
218
|
+
self._directives = directives if directives is not None else []
|
|
219
|
+
self.queries = []
|
|
220
|
+
|
|
221
|
+
def list_directives(self, bank_id, active_only=True, timeout=2):
|
|
222
|
+
return {"items": list(self._directives)}
|
|
223
|
+
|
|
224
|
+
def recall(self, bank_id, query, **kwargs):
|
|
225
|
+
self.queries.append(query)
|
|
226
|
+
return {"results": list(self._memories)}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _run_main_with(client, prompt, config_extra=None):
|
|
230
|
+
hook_input = {
|
|
231
|
+
"prompt": prompt,
|
|
232
|
+
"session_id": "test-session",
|
|
233
|
+
"transcript_path": "",
|
|
234
|
+
"cwd": "/tmp",
|
|
235
|
+
}
|
|
236
|
+
config = {
|
|
237
|
+
"autoRecall": True,
|
|
238
|
+
"bankId": "test-bank",
|
|
239
|
+
"recallMaxTokens": 1024,
|
|
240
|
+
"recallBudget": "mid",
|
|
241
|
+
"recallContextTurns": 1,
|
|
242
|
+
"recallMaxQueryChars": 800,
|
|
243
|
+
"recallPromptPreamble": "",
|
|
244
|
+
}
|
|
245
|
+
if config_extra:
|
|
246
|
+
config.update(config_extra)
|
|
247
|
+
|
|
248
|
+
stdout = io.StringIO()
|
|
249
|
+
stderr = io.StringIO()
|
|
250
|
+
with patch.object(recall, "load_config", return_value=config), patch.object(
|
|
251
|
+
recall, "get_api_url", return_value="http://localhost:18888"
|
|
252
|
+
), patch.object(recall, "HindsightClient", return_value=client), patch.object(
|
|
253
|
+
recall, "ensure_bank_mission", return_value=None
|
|
254
|
+
), patch.object(recall, "write_state", return_value=None), patch(
|
|
255
|
+
"sys.stdin", new=io.StringIO(json.dumps(hook_input))
|
|
256
|
+
), patch("sys.stdout", new=stdout), patch("sys.stderr", new=stderr):
|
|
257
|
+
recall.main()
|
|
258
|
+
|
|
259
|
+
raw = stdout.getvalue()
|
|
260
|
+
if not raw.strip():
|
|
261
|
+
return None, raw
|
|
262
|
+
parsed = json.loads(raw)
|
|
263
|
+
return parsed["hookSpecificOutput"]["additionalContext"], raw
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# A profile-only prompt: matches the profile regex but NOT the directive regex,
|
|
267
|
+
# so these end-to-end assertions isolate the profile nudge cleanly.
|
|
268
|
+
PROFILE_PROMPT = "my timezone is Australia/Melbourne"
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class _LogTestBase(unittest.TestCase):
|
|
272
|
+
def setUp(self):
|
|
273
|
+
self._tmpdir = tempfile.mkdtemp(prefix="profile-nudge-test-")
|
|
274
|
+
self._prev = os.environ.get("CLAUDE_PLUGIN_DATA")
|
|
275
|
+
os.environ["CLAUDE_PLUGIN_DATA"] = self._tmpdir
|
|
276
|
+
|
|
277
|
+
def tearDown(self):
|
|
278
|
+
shutil.rmtree(self._tmpdir, ignore_errors=True)
|
|
279
|
+
if self._prev is None:
|
|
280
|
+
os.environ.pop("CLAUDE_PLUGIN_DATA", None)
|
|
281
|
+
else:
|
|
282
|
+
os.environ["CLAUDE_PLUGIN_DATA"] = self._prev
|
|
283
|
+
|
|
284
|
+
def _read_log(self):
|
|
285
|
+
path = os.path.join(self._tmpdir, "state", "recall_log.jsonl")
|
|
286
|
+
if not os.path.isfile(path):
|
|
287
|
+
return []
|
|
288
|
+
with open(path, encoding="utf-8") as f:
|
|
289
|
+
return [json.loads(line) for line in f if line.strip()]
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
class ProfileNudgeEndToEnd(_LogTestBase):
|
|
293
|
+
def test_advisory_reaches_context_and_row_records_it(self):
|
|
294
|
+
client = _RecordingClient(memories=[{"text": "m", "type": "fact",
|
|
295
|
+
"mentioned_at": "2026-01-01", "id": "m1"}])
|
|
296
|
+
ctx, _raw = _run_main_with(client, prompt=PROFILE_PROMPT)
|
|
297
|
+
# The profile advisory is injected into the turn context, tagged and
|
|
298
|
+
# own-bank-scoped.
|
|
299
|
+
self.assertIsNotNone(ctx)
|
|
300
|
+
self.assertIn("profile:ken", ctx)
|
|
301
|
+
self.assertIn("OWN bank", ctx)
|
|
302
|
+
# The recall_log row carries the firing-rate boolean.
|
|
303
|
+
entries = self._read_log()
|
|
304
|
+
self.assertEqual(len(entries), 1)
|
|
305
|
+
self.assertTrue(entries[0]["profile_nudge"])
|
|
306
|
+
|
|
307
|
+
def test_knob_off_suppresses_nudge_and_row_is_false(self):
|
|
308
|
+
client = _RecordingClient(memories=[{"text": "m", "type": "fact",
|
|
309
|
+
"mentioned_at": "2026-01-01", "id": "m1"}])
|
|
310
|
+
ctx, _raw = _run_main_with(
|
|
311
|
+
client, prompt=PROFILE_PROMPT,
|
|
312
|
+
config_extra={"profileCaptureNudge": False},
|
|
313
|
+
)
|
|
314
|
+
# Nudge suppressed: the advisory is absent from context (memories may
|
|
315
|
+
# still be injected, but never the profile block).
|
|
316
|
+
if ctx is not None:
|
|
317
|
+
self.assertNotIn("profile:ken", ctx)
|
|
318
|
+
self.assertNotIn("profile_capture_check", ctx)
|
|
319
|
+
entries = self._read_log()
|
|
320
|
+
self.assertEqual(len(entries), 1)
|
|
321
|
+
self.assertFalse(entries[0]["profile_nudge"])
|
|
322
|
+
|
|
323
|
+
def test_non_profile_prompt_does_not_fire(self):
|
|
324
|
+
client = _RecordingClient(memories=[{"text": "m", "type": "fact",
|
|
325
|
+
"mentioned_at": "2026-01-01", "id": "m1"}])
|
|
326
|
+
ctx, _raw = _run_main_with(client, prompt="please run the tests")
|
|
327
|
+
if ctx is not None:
|
|
328
|
+
self.assertNotIn("profile:ken", ctx)
|
|
329
|
+
entries = self._read_log()
|
|
330
|
+
self.assertEqual(len(entries), 1)
|
|
331
|
+
self.assertFalse(entries[0]["profile_nudge"])
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
if __name__ == "__main__":
|
|
335
|
+
unittest.main()
|