switchroom 0.17.6 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/workspace-dynamic-hook.sh +12 -13
- package/dist/agent-scheduler/index.js +65 -5
- package/dist/auth-broker/index.js +6623 -514
- package/dist/cli/notion-write-pretool.mjs +64 -4
- package/dist/cli/switchroom.js +1888 -1162
- package/dist/host-control/main.js +6306 -162
- package/dist/vault/approvals/kernel-server.js +6014 -202
- package/dist/vault/broker/server.js +6741 -940
- package/package.json +1 -1
- package/profiles/_base/settings.json.hbs +2 -2
- package/profiles/_base/start.sh.hbs +218 -25
- package/profiles/coding/CLAUDE.md.hbs +1 -1
- package/profiles/default/CLAUDE.md +116 -0
- package/profiles/default/CLAUDE.md.hbs +2 -2
- package/profiles/executive-assistant/CLAUDE.md.hbs +1 -1
- package/profiles/health-coach/CLAUDE.md.hbs +1 -1
- package/skills/mental-model-curator/SKILL.md +162 -0
- package/telegram-plugin/auth-snapshot-format.ts +22 -24
- package/telegram-plugin/bridge/bridge.ts +80 -1
- package/telegram-plugin/bridge/ipc-client.ts +19 -0
- package/telegram-plugin/bridge/permission-ledger.ts +61 -0
- package/telegram-plugin/consolidation-legibility.ts +279 -0
- package/telegram-plugin/context-exhaustion.ts +124 -0
- package/telegram-plugin/dist/bridge/bridge.js +85 -1
- package/telegram-plugin/dist/gateway/gateway.js +25802 -8488
- package/telegram-plugin/dist/server.js +86 -2
- package/telegram-plugin/feed-heartbeat-climb.ts +206 -0
- package/telegram-plugin/gateway/activity-card-store.ts +369 -0
- package/telegram-plugin/gateway/gateway.ts +1861 -172
- package/telegram-plugin/gateway/inbound-delivery-gate.ts +26 -0
- package/telegram-plugin/gateway/inbound-spool.ts +22 -0
- package/telegram-plugin/gateway/mental-model-propose-card.ts +69 -0
- package/telegram-plugin/gateway/mental-model-propose-diff.ts +171 -0
- package/telegram-plugin/gateway/mental-model-propose-inbound-builders.ts +147 -0
- package/telegram-plugin/gateway/mental-model-propose-resolve.ts +201 -0
- package/telegram-plugin/gateway/missed-approvals-card.ts +161 -0
- package/telegram-plugin/gateway/missed-approvals-store.ts +167 -0
- package/telegram-plugin/gateway/model-command.ts +70 -10
- package/telegram-plugin/gateway/permission-rearm.ts +115 -0
- package/telegram-plugin/gateway/scoped-grant-store.ts +89 -0
- package/telegram-plugin/memory-legibility.ts +217 -0
- package/telegram-plugin/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json +1 -0
- package/telegram-plugin/package.json +6 -0
- package/telegram-plugin/quota-watch.ts +4 -6
- package/telegram-plugin/registry/turns-schema.test.ts +97 -0
- package/telegram-plugin/registry/turns-schema.ts +78 -0
- package/telegram-plugin/render/ir.ts +209 -0
- package/telegram-plugin/render/parse.ts +363 -0
- package/telegram-plugin/render/render.ts +440 -0
- package/telegram-plugin/render/rich-render.ts +72 -0
- package/telegram-plugin/scoped-approval.ts +59 -0
- package/telegram-plugin/silent-end.ts +78 -0
- package/telegram-plugin/stream-controller.ts +14 -3
- package/telegram-plugin/subagent-watcher.ts +60 -6
- package/telegram-plugin/tests/activity-card-store.test.ts +530 -0
- package/telegram-plugin/tests/activity-card-wiring.test.ts +88 -0
- package/telegram-plugin/tests/auth-command-format2.test.ts +1 -1
- package/telegram-plugin/tests/auth-snapshot-format.test.ts +30 -16
- package/telegram-plugin/tests/claude-code-event-contract.test.ts +48 -0
- package/telegram-plugin/tests/consolidation-legibility.test.ts +224 -0
- package/telegram-plugin/tests/emission-authority-facade.test.ts +25 -10
- package/telegram-plugin/tests/feed-heartbeat-liveness-open.test.ts +44 -9
- package/telegram-plugin/tests/feed-survival.test.ts +39 -0
- package/telegram-plugin/tests/gateway-boot-marker-clear.test.ts +3 -3
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +81 -0
- package/telegram-plugin/tests/inbound-emit-after-intercepts.test.ts +82 -0
- package/telegram-plugin/tests/inbound-spool.test.ts +105 -0
- package/telegram-plugin/tests/liveness-tracker.test.ts +228 -0
- package/telegram-plugin/tests/memory-legibility.test.ts +216 -0
- package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +67 -0
- package/telegram-plugin/tests/mental-model-propose-card.test.ts +56 -0
- package/telegram-plugin/tests/mental-model-propose-diff.test.ts +201 -0
- package/telegram-plugin/tests/mental-model-propose-inbound-builders.test.ts +68 -0
- package/telegram-plugin/tests/mental-model-propose-resolve.test.ts +157 -0
- package/telegram-plugin/tests/missed-approvals-card.test.ts +145 -0
- package/telegram-plugin/tests/missed-approvals-store.test.ts +147 -0
- package/telegram-plugin/tests/missed-approvals-wiring.test.ts +89 -0
- package/telegram-plugin/tests/model-command.test.ts +193 -16
- package/telegram-plugin/tests/narrative-render.test.ts +125 -0
- package/telegram-plugin/tests/orphaned-reply-rearm.test.ts +123 -163
- package/telegram-plugin/tests/permission-ledger.test.ts +166 -0
- package/telegram-plugin/tests/permission-no-repeat-wiring.test.ts +1 -1
- package/telegram-plugin/tests/permission-rearm-wiring.test.ts +175 -0
- package/telegram-plugin/tests/permission-rearm.test.ts +126 -0
- package/telegram-plugin/tests/quota-watch.test.ts +1 -4
- package/telegram-plugin/tests/rapid-fire-delivery-ordering.test.ts +149 -0
- package/telegram-plugin/tests/render/parse-torture.test.ts +136 -0
- package/telegram-plugin/tests/render/parse.test.ts +393 -0
- package/telegram-plugin/tests/render/render.test.ts +436 -0
- package/telegram-plugin/tests/render/rich-render.test.ts +85 -0
- package/telegram-plugin/tests/scoped-grant-persist.test.ts +223 -0
- package/telegram-plugin/tests/silent-end-transport.test.ts +290 -0
- package/telegram-plugin/tests/silent-turn-climb-transport.test.ts +337 -0
- package/telegram-plugin/tests/subagent-watcher.test.ts +139 -0
- package/telegram-plugin/tests/telegram-activity-visibility-integration.test.ts +155 -1
- package/telegram-plugin/tests/worktree-watch-cwds.test.ts +198 -0
- package/telegram-plugin/turn-liveness-floor.ts +35 -1
- package/telegram-plugin/uat/assertions.ts +88 -4
- package/telegram-plugin/uat/feed-matcher.test.ts +69 -0
- package/telegram-plugin/uat/scenarios/fuzz-liveness-climb-dm.test.ts +155 -0
- package/telegram-plugin/uat/scenarios/jtbd-directive-capture-nudge-dm.test.ts +185 -0
- package/telegram-plugin/uat/scenarios/jtbd-liveness-climb-channel.test.ts +192 -0
- package/telegram-plugin/uat/scenarios/jtbd-liveness-climb-dm.test.ts +220 -0
- package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-channel.test.ts +137 -0
- package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-dm.test.ts +148 -0
- package/telegram-plugin/uat/scenarios/jtbd-memory-legibility-channel.test.ts +66 -0
- package/telegram-plugin/uat/scenarios/jtbd-memory-legibility-dm.test.ts +61 -0
- package/telegram-plugin/uat/scenarios/jtbd-rich-formatting-render-dm.test.ts +99 -7
- package/telegram-plugin/uat/scenarios/silent-end-recovery-channel.test.ts +136 -0
- package/telegram-plugin/uat/scenarios/silent-end-recovery-dm.test.ts +24 -2
- package/telegram-plugin/worktree-watch-cwds.ts +135 -0
- package/vendor/hindsight-memory/hooks/hooks.json +9 -0
- package/vendor/hindsight-memory/scripts/__pycache__/directive_verify.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/__pycache__/drain_pending.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/__pycache__/recall.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/__pycache__/retain.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/__pycache__/session_end.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/directive_verify.py +445 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/__init__.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/bank.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/client.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/config.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/content.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/daemon.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/directives.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/gateway_ipc.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/llm.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/pending.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/state.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/__pycache__/switchroom_envelope.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/lib/client.py +11 -1
- package/vendor/hindsight-memory/scripts/lib/config.py +46 -2
- package/vendor/hindsight-memory/scripts/lib/directives.py +88 -0
- package/vendor/hindsight-memory/scripts/lib/switchroom_envelope.py +77 -0
- package/vendor/hindsight-memory/scripts/recall.py +217 -10
- package/vendor/hindsight-memory/scripts/retain.py +17 -0
- package/vendor/hindsight-memory/scripts/setup_hooks.py +9 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/__init__.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_config_client_casts.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_config_client_casts.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_directive_capture_nudge.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_directive_capture_nudge.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_directive_verify.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_directive_verify.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_directives.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_directives.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_gateway_ipc.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_gateway_ipc.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_context_slice.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_context_slice.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_integration.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_integration.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_tag_filters.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_tag_filters.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_topic_filter.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_topic_filter.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_trivial_skip.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_recall_trivial_skip.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_retain_window.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_retain_window.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_sender_routing.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_sender_routing.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/__pycache__/test_switchroom_envelope.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/scripts/tests/test_directive_capture_nudge.py +185 -0
- package/vendor/hindsight-memory/scripts/tests/test_directive_verify.py +516 -0
- package/vendor/hindsight-memory/scripts/tests/test_directives.py +49 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_integration.py +1 -0
- package/vendor/hindsight-memory/scripts/tests/test_retain_window.py +66 -1
- package/vendor/hindsight-memory/scripts/tests/test_switchroom_envelope.py +69 -0
- package/vendor/hindsight-memory/tests/__pycache__/conftest.cpython-313-pytest-9.0.3.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/conftest.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_bank.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_bank.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_client.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_client.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_config.cpython-313-pytest-9.0.3.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_config.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_config.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_content.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_content.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_drain_pending.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_drain_pending.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_hooks.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_hooks.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_manifest.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_manifest.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_pending.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_pending.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_recall_exit_codes.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_recall_exit_codes.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_session_end_pending.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_session_end_pending.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_state.cpython-313-pytest-9.1.1.pyc +0 -0
- package/vendor/hindsight-memory/tests/__pycache__/test_state.cpython-313.pyc +0 -0
- package/vendor/hindsight-memory/tests/test_client.py +43 -0
- package/vendor/hindsight-memory/tests/test_recall_exit_codes.py +49 -2
- package/vendor/hindsight-memory/tests/test_recall_precision.py +114 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JTBD scenario — a correction lands as a directive and survives `/reset`.
|
|
3
|
+
*
|
|
4
|
+
* Serves: `reference/jobs/remember-across-sessions.md` — the job's headline
|
|
5
|
+
* criterion is "a rule set once stays respected / a correction sticks."
|
|
6
|
+
*
|
|
7
|
+
* ## Why this exists (issue #2848 Stage B)
|
|
8
|
+
*
|
|
9
|
+
* Directive capture was guidance-only: the model is *told* to call
|
|
10
|
+
* `create_directive` on a durable correction, and a Stage A audit measured a
|
|
11
|
+
* **~55% miss rate** — the same broadcast correction captured by one agent,
|
|
12
|
+
* silently dropped by two others. Stage B adds a DETERMINISTIC regex nudge in
|
|
13
|
+
* the vendored recall hook (`vendor/hindsight-memory/scripts/recall.py`): on a
|
|
14
|
+
* correction-shaped inbound it appends a terse advisory telling the model to
|
|
15
|
+
* persist the rule with `create_directive` before answering. Detection is pure
|
|
16
|
+
* regex — the judgment happens IN the interactive session (claude-native
|
|
17
|
+
* invariant: no model callsite, no silent hook-side write).
|
|
18
|
+
*
|
|
19
|
+
* ## Contract this asserts
|
|
20
|
+
*
|
|
21
|
+
* 1. **Capture**: after a correction-shaped DM ("from now on, always end every
|
|
22
|
+
* reply with <MARKER>"), an ACTIVE directive referencing the rule exists in
|
|
23
|
+
* the agent's hindsight bank (queried via the REST API, the same surface
|
|
24
|
+
* Stage A used).
|
|
25
|
+
* 2. **Survival across `/reset`**: after `/reset` clears the session, a fresh
|
|
26
|
+
* neutral follow-up is still answered honoring the rule (the MARKER appears
|
|
27
|
+
* in the reply) — proving the directive was re-injected from the bank, not
|
|
28
|
+
* merely held in the wiped session context.
|
|
29
|
+
*
|
|
30
|
+
* ## Self-skip
|
|
31
|
+
*
|
|
32
|
+
* Self-skips GREEN when the mtcute driver isn't wired (no
|
|
33
|
+
* TELEGRAM_UAT_DRIVER_SESSION etc.), so it never reds an unwired host. The
|
|
34
|
+
* whole uat/** tree is excluded from gating CI regardless; run live with
|
|
35
|
+
* `bun run --cwd telegram-plugin test:uat jtbd-directive-capture-nudge-dm`.
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
import { describe, it, expect } from "vitest";
|
|
39
|
+
import { randomBytes } from "node:crypto";
|
|
40
|
+
import { spinUp } from "../harness.js";
|
|
41
|
+
import { isActivityFeedMessage, isWorkerFeedMessage } from "../assertions.js";
|
|
42
|
+
import type { ObservedMessage } from "../driver.js";
|
|
43
|
+
|
|
44
|
+
const AGENT = "test-harness";
|
|
45
|
+
|
|
46
|
+
// Match the first non-empty bot reply that is neither the worker-activity
|
|
47
|
+
// feed nor the tool-activity feed (same guard as jtbd-memory-survives).
|
|
48
|
+
const isReply = (m: ObservedMessage): boolean =>
|
|
49
|
+
/\S/.test(m.text) && !isWorkerFeedMessage(m) && !isActivityFeedMessage(m);
|
|
50
|
+
|
|
51
|
+
const CORRECTION_REPLY_BUDGET_MS = 60_000;
|
|
52
|
+
const DIRECTIVE_SETTLE_MS = 12_000;
|
|
53
|
+
const RESET_SETTLE_MS = 45_000;
|
|
54
|
+
const POSTRESET_REPLY_BUDGET_MS = 120_000;
|
|
55
|
+
|
|
56
|
+
// Unique per-run marker so directive-content matching and the honored-rule
|
|
57
|
+
// check can't latch onto stale state from a prior run.
|
|
58
|
+
const MARKER = `SR_UAT_DIRECTIVE_${randomBytes(6).toString("hex").toUpperCase()}`;
|
|
59
|
+
|
|
60
|
+
// Hindsight REST base. Bank id == agent name (Stage A + CLAUDE.md). Override
|
|
61
|
+
// via HINDSIGHT_UAT_API_URL if the host runs it elsewhere.
|
|
62
|
+
const HINDSIGHT_BASE =
|
|
63
|
+
process.env.HINDSIGHT_UAT_API_URL ??
|
|
64
|
+
process.env.HINDSIGHT_API_URL ??
|
|
65
|
+
"http://127.0.0.1:18888";
|
|
66
|
+
|
|
67
|
+
interface Directive {
|
|
68
|
+
content?: string;
|
|
69
|
+
name?: string;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function fetchActiveDirectives(bank: string): Promise<Directive[]> {
|
|
73
|
+
const url = `${HINDSIGHT_BASE}/v1/default/banks/${encodeURIComponent(bank)}/directives?active_only=false`;
|
|
74
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(8_000) });
|
|
75
|
+
if (!res.ok) {
|
|
76
|
+
throw new Error(`hindsight directives HTTP ${res.status} for bank ${bank} (${url})`);
|
|
77
|
+
}
|
|
78
|
+
const body = (await res.json()) as { items?: Directive[] };
|
|
79
|
+
return Array.isArray(body.items) ? body.items : [];
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const uatWired = Boolean(
|
|
83
|
+
process.env.TELEGRAM_API_ID &&
|
|
84
|
+
process.env.TELEGRAM_API_HASH &&
|
|
85
|
+
process.env.TELEGRAM_UAT_DRIVER_SESSION &&
|
|
86
|
+
(process.env.TELEGRAM_TEST_BOT_USERNAME || false),
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
(uatWired ? describe : describe.skip)(
|
|
90
|
+
"uat: correction lands as a directive and survives /reset (remember-across-sessions JTBD)",
|
|
91
|
+
() => {
|
|
92
|
+
it(
|
|
93
|
+
"a standing-rule correction is persisted as a directive and honored after /reset",
|
|
94
|
+
async () => {
|
|
95
|
+
// --- Phase 1: give the agent a durable, correction-shaped rule ---
|
|
96
|
+
const sc1 = await spinUp({ agent: AGENT });
|
|
97
|
+
try {
|
|
98
|
+
await sc1.sendDM(
|
|
99
|
+
`From now on, always end every reply you send me with this exact ` +
|
|
100
|
+
`marker on its own final line: ${MARKER}. This is a standing ` +
|
|
101
|
+
`rule for how you should behave going forward — please persist ` +
|
|
102
|
+
`it so it sticks across sessions. Confirm you've noted it.`,
|
|
103
|
+
);
|
|
104
|
+
|
|
105
|
+
const reply = await sc1.expectMessage(isReply, {
|
|
106
|
+
from: "bot",
|
|
107
|
+
timeout: CORRECTION_REPLY_BUDGET_MS,
|
|
108
|
+
});
|
|
109
|
+
expect(reply.text.length).toBeGreaterThan(0);
|
|
110
|
+
|
|
111
|
+
// Give the in-turn create_directive call (nudged by the hook) time
|
|
112
|
+
// to land in the bank before we query it.
|
|
113
|
+
await new Promise((r) => setTimeout(r, DIRECTIVE_SETTLE_MS));
|
|
114
|
+
|
|
115
|
+
// --- Phase 2: CAPTURE — an active directive references the rule ---
|
|
116
|
+
const directives = await fetchActiveDirectives(AGENT);
|
|
117
|
+
const matched = directives.some((d) => {
|
|
118
|
+
const hay = `${d.content ?? ""} ${d.name ?? ""}`;
|
|
119
|
+
return (
|
|
120
|
+
hay.includes(MARKER) ||
|
|
121
|
+
/end (?:every|each|your) repl/i.test(hay) ||
|
|
122
|
+
/marker/i.test(hay)
|
|
123
|
+
);
|
|
124
|
+
});
|
|
125
|
+
if (!matched) {
|
|
126
|
+
throw new Error(
|
|
127
|
+
`[directive-capture] CONTRACT FAILED (capture): no active directive ` +
|
|
128
|
+
`references the standing rule. The correction was correction-shaped ` +
|
|
129
|
+
`(the recall hook should have nudged create_directive) but nothing ` +
|
|
130
|
+
`persisted. Directives seen: ` +
|
|
131
|
+
`${JSON.stringify(directives.map((d) => d.content ?? d.name).slice(0, 8))}`,
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
expect(matched).toBe(true);
|
|
135
|
+
} finally {
|
|
136
|
+
await sc1.tearDown();
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// --- Phase 3: /reset clears the session ---
|
|
140
|
+
const scReset = await spinUp({ agent: AGENT });
|
|
141
|
+
try {
|
|
142
|
+
await scReset.sendDM("/reset");
|
|
143
|
+
} finally {
|
|
144
|
+
await scReset.tearDown();
|
|
145
|
+
}
|
|
146
|
+
// Let the reset complete + the bridge reattach before probing.
|
|
147
|
+
await new Promise((r) => setTimeout(r, RESET_SETTLE_MS));
|
|
148
|
+
|
|
149
|
+
// --- Phase 4: SURVIVAL — a fresh neutral turn still honors the rule ---
|
|
150
|
+
const sc2 = await spinUp({ agent: AGENT });
|
|
151
|
+
try {
|
|
152
|
+
await sc2.sendDM(
|
|
153
|
+
`What is 2 + 2? Answer normally — nothing special about this message.`,
|
|
154
|
+
);
|
|
155
|
+
const reply = await sc2.expectMessage(isReply, {
|
|
156
|
+
from: "bot",
|
|
157
|
+
timeout: POSTRESET_REPLY_BUDGET_MS,
|
|
158
|
+
});
|
|
159
|
+
expect(reply.text.length).toBeGreaterThan(0);
|
|
160
|
+
|
|
161
|
+
// The rule was NOT in this session's context (it was wiped by
|
|
162
|
+
// /reset). If the marker appears, the directive was re-injected from
|
|
163
|
+
// the bank on recall — the correction stuck.
|
|
164
|
+
const honored = reply.text.includes(MARKER);
|
|
165
|
+
if (!honored) {
|
|
166
|
+
throw new Error(
|
|
167
|
+
`[directive-capture] CONTRACT FAILED (survival): after /reset the ` +
|
|
168
|
+
`agent did not honor the standing rule — marker ${MARKER} absent ` +
|
|
169
|
+
`from the reply, so the directive did not survive/re-inject. Reply: ` +
|
|
170
|
+
`${JSON.stringify(reply.text.slice(0, 400))}`,
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
expect(honored).toBe(true);
|
|
174
|
+
} finally {
|
|
175
|
+
await sc2.tearDown();
|
|
176
|
+
}
|
|
177
|
+
},
|
|
178
|
+
CORRECTION_REPLY_BUDGET_MS +
|
|
179
|
+
DIRECTIVE_SETTLE_MS +
|
|
180
|
+
RESET_SETTLE_MS +
|
|
181
|
+
POSTRESET_REPLY_BUDGET_MS +
|
|
182
|
+
30_000,
|
|
183
|
+
);
|
|
184
|
+
},
|
|
185
|
+
);
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic-turn-liveness climb — CHANNEL twin of
|
|
3
|
+
* `jtbd-liveness-climb-dm.test.ts` (Phase 4a, `deterministic-turn-liveness.md`).
|
|
4
|
+
*
|
|
5
|
+
* Surface parity is a production-readiness bar for this job (job spec §
|
|
6
|
+
* Production-readiness: "every signal is proven in both DM and forum
|
|
7
|
+
* channel"). Phase 1's climb is keyed on the SAME `feedHeartbeatTick` /
|
|
8
|
+
* `runSilentTurnHeartbeatTick` path regardless of chat type — this scenario
|
|
9
|
+
* proves the card edits land, CLIMB, and stay ping-free in a supergroup too,
|
|
10
|
+
* not just a DM. It is brought to full parity with the DM twin: it parses the
|
|
11
|
+
* elapsed suffix, asserts non-decreasing climb, and HARD-FAILS on a frozen
|
|
12
|
+
* card (zero edits across a clearly-long silent tool) — the exact freeze this
|
|
13
|
+
* fix exists to detect. (Previously it only warned "INCONCLUSIVE" on a freeze,
|
|
14
|
+
* so it could never catch the regression it exists for.)
|
|
15
|
+
*
|
|
16
|
+
* Self-skips green when no test supergroup is wired (`SWITCHROOM_UAT_CHAT_ID`
|
|
17
|
+
* unset / unresolvable) — mirrors the existing channel-twin convention (see
|
|
18
|
+
* `jtbd-foreground-subagent-activity-channel.test.ts`).
|
|
19
|
+
*
|
|
20
|
+
* mtcute has no forum-topic API here, so this uses the General topic — same
|
|
21
|
+
* limitation as every other channel twin in this suite.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { describe, expect, it } from "vitest";
|
|
25
|
+
import { spinUp } from "../harness.js";
|
|
26
|
+
import { isActivityFeedMessage, isFrameworkFallbackText } from "../assertions.js";
|
|
27
|
+
import type { ObservedMessage } from "../driver.js";
|
|
28
|
+
|
|
29
|
+
const AGENT = "test-harness";
|
|
30
|
+
const SUPERGROUP_ID = Number.parseInt(process.env.SWITCHROOM_UAT_CHAT_ID ?? "", 10);
|
|
31
|
+
|
|
32
|
+
const MIN_CLIMB_EDITS = 4;
|
|
33
|
+
const OVERALL_BUDGET_MS = 160_000;
|
|
34
|
+
const SILENT_TOOL_SECONDS = 40;
|
|
35
|
+
|
|
36
|
+
const SILENT_TOOL_PROMPT =
|
|
37
|
+
"Do exactly this and nothing else: run a single Bash command " +
|
|
38
|
+
`\`sleep ${SILENT_TOOL_SECONDS}\` and wait for it to finish. Do NOT narrate ` +
|
|
39
|
+
"anything before or during the sleep — no commentary, no other tool calls, " +
|
|
40
|
+
"no intermediate messages. Only after the sleep completes, reply with a " +
|
|
41
|
+
"one-line confirmation that you waited.";
|
|
42
|
+
|
|
43
|
+
/** Parse the `Working · Ns` (or `· Nm Ss`) elapsed suffix out of a card body.
|
|
44
|
+
* Returns null when the shape isn't present. Mirrors the DM twin. */
|
|
45
|
+
function parseElapsedSeconds(text: string): number | null {
|
|
46
|
+
const m = text.match(/(?:^|\s)·\s*(?:(\d+)\s*m)?\s*(\d+)\s*s\b/i);
|
|
47
|
+
if (!m) return null;
|
|
48
|
+
const minutes = m[1] ? Number.parseInt(m[1], 10) : 0;
|
|
49
|
+
const seconds = Number.parseInt(m[2], 10);
|
|
50
|
+
return minutes * 60 + seconds;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
describe("uat: deterministic turn liveness — silent-tool card climb (channel parity)", () => {
|
|
54
|
+
it(
|
|
55
|
+
"climbs the activity card in a supergroup ≥4 times with non-decreasing elapsed, no mid-turn ping",
|
|
56
|
+
async () => {
|
|
57
|
+
if (!Number.isFinite(SUPERGROUP_ID)) {
|
|
58
|
+
console.warn("[liveness-climb-channel] SWITCHROOM_UAT_CHAT_ID unset — skipping");
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
const sc = await spinUp({ agent: AGENT, settleMs: 0 });
|
|
62
|
+
try {
|
|
63
|
+
await sc.driver.primeDialogs();
|
|
64
|
+
if (!(await sc.driver.canResolve(SUPERGROUP_ID))) {
|
|
65
|
+
console.warn(`[liveness-climb-channel] supergroup ${SUPERGROUP_ID} not resolvable — skipping`);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const iter = sc.driver
|
|
70
|
+
.observeMessages(SUPERGROUP_ID)
|
|
71
|
+
[Symbol.asyncIterator]();
|
|
72
|
+
|
|
73
|
+
await sc.driver.sendText(SUPERGROUP_ID, SILENT_TOOL_PROMPT);
|
|
74
|
+
const sentAt = Date.now();
|
|
75
|
+
console.log("[liveness-climb-channel] prompt sent; watching card edits…");
|
|
76
|
+
|
|
77
|
+
let cardMessageId: number | null = null;
|
|
78
|
+
const elapsedSamples: number[] = [];
|
|
79
|
+
let cardEditCount = 0;
|
|
80
|
+
let answer: ObservedMessage | null = null;
|
|
81
|
+
let otherLoudMessage: ObservedMessage | null = null;
|
|
82
|
+
|
|
83
|
+
const deadline = Date.now() + 110_000;
|
|
84
|
+
while (Date.now() < deadline) {
|
|
85
|
+
if (answer && cardEditCount >= MIN_CLIMB_EDITS) break;
|
|
86
|
+
const remaining = deadline - Date.now();
|
|
87
|
+
const next = await Promise.race([
|
|
88
|
+
iter.next(),
|
|
89
|
+
new Promise<{ done: true; value: undefined }>((r) =>
|
|
90
|
+
setTimeout(() => r({ done: true, value: undefined }), Math.max(0, remaining)),
|
|
91
|
+
),
|
|
92
|
+
]);
|
|
93
|
+
if (next.done || next.value == null) break;
|
|
94
|
+
const m = next.value as ObservedMessage;
|
|
95
|
+
if (m.senderUserId === sc.driverUserId) continue;
|
|
96
|
+
|
|
97
|
+
if (isActivityFeedMessage(m)) {
|
|
98
|
+
if (cardMessageId == null) cardMessageId = m.messageId;
|
|
99
|
+
if (m.messageId === cardMessageId && m.edited) {
|
|
100
|
+
cardEditCount++;
|
|
101
|
+
const secs = parseElapsedSeconds(m.text);
|
|
102
|
+
if (secs != null) elapsedSamples.push(secs);
|
|
103
|
+
console.log(
|
|
104
|
+
`[liveness-climb-channel] card edit #${cardEditCount} at +${Date.now() - sentAt}ms: ` +
|
|
105
|
+
JSON.stringify(m.text.slice(0, 100)),
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
if (m.edited && m.silent === false) otherLoudMessage = otherLoudMessage ?? m;
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (m.edited) continue;
|
|
113
|
+
// A framework fallback TEXT send is a mid-turn erosion, not the
|
|
114
|
+
// completion proof — reject it from the answer lane (see DM twin).
|
|
115
|
+
if (isFrameworkFallbackText(m.text)) {
|
|
116
|
+
otherLoudMessage = otherLoudMessage ?? m;
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
if (!answer && m.text.trim().length > 0) {
|
|
120
|
+
answer = m;
|
|
121
|
+
console.log(`[liveness-climb-channel] answer at +${Date.now() - sentAt}ms.`);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (m.silent === false) otherLoudMessage = otherLoudMessage ?? m;
|
|
125
|
+
}
|
|
126
|
+
await iter.return?.();
|
|
127
|
+
|
|
128
|
+
// No mid-turn erosion: the climb is edit-only in a supergroup too.
|
|
129
|
+
expect(
|
|
130
|
+
otherLoudMessage,
|
|
131
|
+
`a mid-turn message eroded the ping-free/edit-only guarantee in the ` +
|
|
132
|
+
`supergroup (loud ping or framework fallback text): ` +
|
|
133
|
+
JSON.stringify(otherLoudMessage?.text?.slice(0, 120)),
|
|
134
|
+
).toBeNull();
|
|
135
|
+
|
|
136
|
+
if (cardEditCount === 0 && cardMessageId == null) {
|
|
137
|
+
console.warn(
|
|
138
|
+
"[liveness-climb-channel] INCONCLUSIVE — no activity card observed at all " +
|
|
139
|
+
"(the model may have narrated, or the turn completed too fast).",
|
|
140
|
+
);
|
|
141
|
+
expect(answer, "even in the inconclusive branch the turn must complete").not.toBeNull();
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
if (cardEditCount > 0 && cardEditCount < MIN_CLIMB_EDITS) {
|
|
146
|
+
console.warn(
|
|
147
|
+
`[liveness-climb-channel] INCONCLUSIVE — card edited ${cardEditCount} time(s), ` +
|
|
148
|
+
`below the ${MIN_CLIMB_EDITS} target (model may have narrated a tool label ` +
|
|
149
|
+
"partway through, handing to the labelled-feed heartbeat).",
|
|
150
|
+
);
|
|
151
|
+
} else {
|
|
152
|
+
expect(
|
|
153
|
+
cardEditCount,
|
|
154
|
+
`HARD FAIL — the silent tool ran ~${SILENT_TOOL_SECONDS}s in a supergroup but the ` +
|
|
155
|
+
`activity card edited only ${cardEditCount} time(s) (< ${MIN_CLIMB_EDITS}). This is ` +
|
|
156
|
+
"the pre-Phase-1 freeze regression, in the channel surface.",
|
|
157
|
+
).toBeGreaterThanOrEqual(MIN_CLIMB_EDITS);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Climbing property: elapsed samples must be non-decreasing…
|
|
161
|
+
for (let i = 1; i < elapsedSamples.length; i++) {
|
|
162
|
+
expect(
|
|
163
|
+
elapsedSamples[i],
|
|
164
|
+
`elapsed suffix went backwards/froze across edits: ${JSON.stringify(elapsedSamples)}`,
|
|
165
|
+
).toBeGreaterThanOrEqual(elapsedSamples[i - 1]);
|
|
166
|
+
}
|
|
167
|
+
// …and must actually climb across the full window (frozen == the freeze
|
|
168
|
+
// this fix closes; backs the job-spec Prove-it claim).
|
|
169
|
+
if (elapsedSamples.length >= MIN_CLIMB_EDITS) {
|
|
170
|
+
expect(
|
|
171
|
+
elapsedSamples[elapsedSamples.length - 1],
|
|
172
|
+
`HARD FAIL — the elapsed suffix never advanced across ${elapsedSamples.length} ` +
|
|
173
|
+
`card edits in the supergroup (frozen at ${JSON.stringify(elapsedSamples)}).`,
|
|
174
|
+
).toBeGreaterThan(elapsedSamples[0]);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Completion proof: an ANSWER (not the card itself) resumes in the same
|
|
178
|
+
// chat. Guarding on `!isActivityFeedMessage` stops the old
|
|
179
|
+
// `expectMessage(SUPERGROUP_ID, /\S/)` from matching a card edit and
|
|
180
|
+
// declaring the turn done while the agent was still climbing.
|
|
181
|
+
expect(
|
|
182
|
+
answer,
|
|
183
|
+
"FAIL — no answer (distinct from the activity card) arrived within budget; the turn may be wedged.",
|
|
184
|
+
).not.toBeNull();
|
|
185
|
+
expect(answer?.chatId).toBe(SUPERGROUP_ID);
|
|
186
|
+
} finally {
|
|
187
|
+
await sc.tearDown();
|
|
188
|
+
}
|
|
189
|
+
},
|
|
190
|
+
OVERALL_BUDGET_MS,
|
|
191
|
+
);
|
|
192
|
+
});
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic-turn-liveness climb — DM (Phase 4a, `deterministic-turn-liveness.md`).
|
|
3
|
+
*
|
|
4
|
+
* The RFC's keystone (Phase 1) gives a busy-but-silent foreground turn a
|
|
5
|
+
* climbing activity card instead of the pre-#2143/#2527/#2667 freeze: once
|
|
6
|
+
* the card is open and no tool emits a label, `feedHeartbeatTick`'s 0-label
|
|
7
|
+
* branch re-renders the SAME message with a fresh `Working · Ns` elapsed
|
|
8
|
+
* every `FEED_HEARTBEAT_MIN_STALE_MS` (~6s), edit-only (no device buzz).
|
|
9
|
+
*
|
|
10
|
+
* IMPORTANT product fact this scenario respects: switchroom has NO streaming
|
|
11
|
+
* REPLY path — the final answer is sent whole (chunked over the char cap if
|
|
12
|
+
* needed). The card observed here is the activity/liveness card — a distinct,
|
|
13
|
+
* pre-existing card-edit surface (worker-activity-feed / liveness card), not
|
|
14
|
+
* a streamed reply. This scenario never asserts a streamed reply edit.
|
|
15
|
+
*
|
|
16
|
+
* ## Trigger
|
|
17
|
+
*
|
|
18
|
+
* A single `Bash` call that sleeps ~40s with no intermediate narration and no
|
|
19
|
+
* tool label — the exact "silent tool" shape the Phase-1 climb exists for.
|
|
20
|
+
*
|
|
21
|
+
* ## What it asserts (asymmetric — timing on a live model is not fully
|
|
22
|
+
* forceable)
|
|
23
|
+
*
|
|
24
|
+
* PASS — the SAME activity-card message is edited (non-initial
|
|
25
|
+
* renders, i.e. `edited === true` on repeat observations of the
|
|
26
|
+
* same `messageId`) at least `MIN_CLIMB_EDITS` (4) times across
|
|
27
|
+
* the ~40s window, and the elapsed suffix parsed out of the
|
|
28
|
+
* body is monotonically non-decreasing across those edits
|
|
29
|
+
* (the "climbing" property — hard-fails on a frozen/repeated
|
|
30
|
+
* elapsed value across all edits, which reproduces the pre-
|
|
31
|
+
* Phase-1 freeze).
|
|
32
|
+
* PASS — no bot message OTHER than the one activity-card message id
|
|
33
|
+
* appears during the silent window (no mid-turn notification
|
|
34
|
+
* ping; the climb is edit-only, per Phase 1 + the job spec's
|
|
35
|
+
* "no device buzz" bad-list item).
|
|
36
|
+
* INCONCLUSIVE — the model narrated anyway (a tool label / narrative text
|
|
37
|
+
* landed), so the 0-label silent-tool path was not exercised.
|
|
38
|
+
* Warn + pass — not a regression of this fix.
|
|
39
|
+
* FAIL — no answer at all within budget (wedged), or a mid-turn ping
|
|
40
|
+
* is observed (silent === false on a non-final message).
|
|
41
|
+
* HARD FAIL — the turn ran clearly longer than the climb bound with fewer
|
|
42
|
+
* than `MIN_CLIMB_EDITS` edits on the card and no narration —
|
|
43
|
+
* the exact freeze this fix exists to close.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
import { describe, expect, it } from "vitest";
|
|
47
|
+
import { spinUp } from "../harness.js";
|
|
48
|
+
import { isActivityFeedMessage, isFrameworkFallbackText } from "../assertions.js";
|
|
49
|
+
import type { ObservedMessage } from "../driver.js";
|
|
50
|
+
|
|
51
|
+
const MIN_CLIMB_EDITS = 4;
|
|
52
|
+
const OVERALL_BUDGET_MS = 140_000;
|
|
53
|
+
const SILENT_TOOL_SECONDS = 40;
|
|
54
|
+
|
|
55
|
+
const SILENT_TOOL_PROMPT =
|
|
56
|
+
"Do exactly this and nothing else: run a single Bash command " +
|
|
57
|
+
`\`sleep ${SILENT_TOOL_SECONDS}\` and wait for it to finish. Do NOT narrate ` +
|
|
58
|
+
"anything before or during the sleep — no commentary, no other tool calls, " +
|
|
59
|
+
"no intermediate messages. Only after the sleep completes, reply with a " +
|
|
60
|
+
"one-line confirmation that you waited.";
|
|
61
|
+
|
|
62
|
+
/** Parse the `Working · Ns` (or `· Nm Ss`) elapsed suffix out of a card body.
|
|
63
|
+
* Returns null when the shape isn't present (e.g. a labelled feed). */
|
|
64
|
+
function parseElapsedSeconds(text: string): number | null {
|
|
65
|
+
const m = text.match(/(?:^|\s)·\s*(?:(\d+)\s*m)?\s*(\d+)\s*s\b/i);
|
|
66
|
+
if (!m) return null;
|
|
67
|
+
const minutes = m[1] ? Number.parseInt(m[1], 10) : 0;
|
|
68
|
+
const seconds = Number.parseInt(m[2], 10);
|
|
69
|
+
return minutes * 60 + seconds;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
describe("uat: deterministic turn liveness — silent-tool card climb (DM)", () => {
|
|
73
|
+
it(
|
|
74
|
+
"climbs the SAME activity card ≥4 times with non-decreasing elapsed, no mid-turn ping",
|
|
75
|
+
async () => {
|
|
76
|
+
const sc = await spinUp({ agent: "test-harness" });
|
|
77
|
+
try {
|
|
78
|
+
const iter = sc.driver
|
|
79
|
+
.observeMessages(sc.botUserId)
|
|
80
|
+
[Symbol.asyncIterator]();
|
|
81
|
+
|
|
82
|
+
await sc.sendDM(SILENT_TOOL_PROMPT);
|
|
83
|
+
const sentAt = Date.now();
|
|
84
|
+
console.log("[liveness-climb-dm] prompt sent; watching card edits…");
|
|
85
|
+
|
|
86
|
+
let cardMessageId: number | null = null;
|
|
87
|
+
const elapsedSamples: number[] = [];
|
|
88
|
+
let cardEditCount = 0;
|
|
89
|
+
let answer: ObservedMessage | null = null;
|
|
90
|
+
let otherLoudMessage: ObservedMessage | null = null;
|
|
91
|
+
|
|
92
|
+
const deadline = Date.now() + 100_000;
|
|
93
|
+
while (Date.now() < deadline) {
|
|
94
|
+
if (answer && cardEditCount >= MIN_CLIMB_EDITS) break;
|
|
95
|
+
const remaining = deadline - Date.now();
|
|
96
|
+
const next = await Promise.race([
|
|
97
|
+
iter.next(),
|
|
98
|
+
new Promise<{ done: true; value: undefined }>((r) =>
|
|
99
|
+
setTimeout(() => r({ done: true, value: undefined }), Math.max(0, remaining)),
|
|
100
|
+
),
|
|
101
|
+
]);
|
|
102
|
+
if (next.done || next.value == null) break;
|
|
103
|
+
const m = next.value as ObservedMessage;
|
|
104
|
+
if (m.senderUserId === sc.driverUserId) continue;
|
|
105
|
+
|
|
106
|
+
if (isActivityFeedMessage(m)) {
|
|
107
|
+
if (cardMessageId == null) cardMessageId = m.messageId;
|
|
108
|
+
if (m.messageId === cardMessageId && m.edited) {
|
|
109
|
+
cardEditCount++;
|
|
110
|
+
const secs = parseElapsedSeconds(m.text);
|
|
111
|
+
if (secs != null) elapsedSamples.push(secs);
|
|
112
|
+
console.log(
|
|
113
|
+
`[liveness-climb-dm] card edit #${cardEditCount} at +${Date.now() - sentAt}ms: ` +
|
|
114
|
+
JSON.stringify(m.text.slice(0, 100)),
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
// A non-silent card edit would still be a ping; card edits are
|
|
118
|
+
// asserted separately below via `silent`.
|
|
119
|
+
if (m.edited && m.silent === false) {
|
|
120
|
+
otherLoudMessage = otherLoudMessage ?? m;
|
|
121
|
+
}
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (m.edited) continue;
|
|
126
|
+
// Attributable-answer gate: a framework mid-turn/dark-turn fallback
|
|
127
|
+
// TEXT send (the exact #2667 erosion this wall guards against — a
|
|
128
|
+
// re-added "still working…" ping) must NOT be swallowed as the
|
|
129
|
+
// turn's answer. Classify it as a mid-turn erosion regardless of its
|
|
130
|
+
// silent flag, so the first loud mid-turn message trips the
|
|
131
|
+
// assertion instead of masquerading as the reply.
|
|
132
|
+
if (isFrameworkFallbackText(m.text)) {
|
|
133
|
+
otherLoudMessage = otherLoudMessage ?? m;
|
|
134
|
+
console.log(
|
|
135
|
+
`[liveness-climb-dm] framework fallback text mid-turn at +${Date.now() - sentAt}ms: ` +
|
|
136
|
+
JSON.stringify(m.text.slice(0, 120)),
|
|
137
|
+
);
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
if (!answer && m.text.trim().length > 0) {
|
|
141
|
+
answer = m;
|
|
142
|
+
console.log(`[liveness-climb-dm] answer at +${Date.now() - sentAt}ms.`);
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
// Any OTHER non-edit bot message during the window is a candidate
|
|
146
|
+
// mid-turn ping — track it for the loud-message assertion below.
|
|
147
|
+
if (m.silent === false) {
|
|
148
|
+
otherLoudMessage = otherLoudMessage ?? m;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
await iter.return?.();
|
|
152
|
+
|
|
153
|
+
// No mid-turn erosion: neither a device-buzzing ping (silent=false on
|
|
154
|
+
// a non-final message) NOR a framework fallback TEXT send (a
|
|
155
|
+
// "still working…"-class mid-turn message — the climb is an edit-only
|
|
156
|
+
// card, never a text send).
|
|
157
|
+
expect(
|
|
158
|
+
otherLoudMessage,
|
|
159
|
+
`a mid-turn message eroded the ping-free/edit-only guarantee ` +
|
|
160
|
+
`(loud ping or framework fallback text): ` +
|
|
161
|
+
JSON.stringify(otherLoudMessage?.text?.slice(0, 120)),
|
|
162
|
+
).toBeNull();
|
|
163
|
+
|
|
164
|
+
if (cardEditCount === 0 && cardMessageId == null) {
|
|
165
|
+
console.warn(
|
|
166
|
+
"[liveness-climb-dm] INCONCLUSIVE — no activity card observed at all " +
|
|
167
|
+
"(the model may have narrated before/instead of the silent sleep, " +
|
|
168
|
+
"or the turn completed too fast). Not a regression of this fix.",
|
|
169
|
+
);
|
|
170
|
+
expect(answer, "even in the inconclusive branch the turn must complete").not.toBeNull();
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (cardEditCount > 0 && cardEditCount < MIN_CLIMB_EDITS) {
|
|
175
|
+
console.warn(
|
|
176
|
+
`[liveness-climb-dm] INCONCLUSIVE — card edited ${cardEditCount} time(s), ` +
|
|
177
|
+
`below the ${MIN_CLIMB_EDITS} target (model may have narrated a tool ` +
|
|
178
|
+
"label partway through, handing off to the labelled-feed heartbeat " +
|
|
179
|
+
"with a different cadence). Not a hard failure.",
|
|
180
|
+
);
|
|
181
|
+
} else {
|
|
182
|
+
expect(
|
|
183
|
+
cardEditCount,
|
|
184
|
+
`HARD FAIL — the silent tool ran ~${SILENT_TOOL_SECONDS}s but the activity ` +
|
|
185
|
+
`card edited only ${cardEditCount} time(s) (< ${MIN_CLIMB_EDITS}). This is ` +
|
|
186
|
+
"the pre-Phase-1 freeze regression (#2143/#2527/#2667 archaeology in " +
|
|
187
|
+
"reference/rfcs/deterministic-turn-liveness.md).",
|
|
188
|
+
).toBeGreaterThanOrEqual(MIN_CLIMB_EDITS);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Climbing property: elapsed samples must be non-decreasing (never
|
|
192
|
+
// rewinds across edits of the same card)…
|
|
193
|
+
for (let i = 1; i < elapsedSamples.length; i++) {
|
|
194
|
+
expect(
|
|
195
|
+
elapsedSamples[i],
|
|
196
|
+
`elapsed suffix went backwards/froze across edits: ${JSON.stringify(elapsedSamples)}`,
|
|
197
|
+
).toBeGreaterThanOrEqual(elapsedSamples[i - 1]);
|
|
198
|
+
}
|
|
199
|
+
// …and, when we have the full climb (≥MIN_CLIMB_EDITS parsed samples),
|
|
200
|
+
// it must actually CLIMB — a frozen/repeated elapsed value across all
|
|
201
|
+
// edits is the exact pre-Phase-1 freeze and a hard failure (this backs
|
|
202
|
+
// the job-spec Prove-it claim; non-decreasing alone would pass a
|
|
203
|
+
// frozen card).
|
|
204
|
+
if (elapsedSamples.length >= MIN_CLIMB_EDITS) {
|
|
205
|
+
expect(
|
|
206
|
+
elapsedSamples[elapsedSamples.length - 1],
|
|
207
|
+
`HARD FAIL — the elapsed suffix never advanced across ${elapsedSamples.length} ` +
|
|
208
|
+
`card edits (frozen at ${JSON.stringify(elapsedSamples)}). A repeated elapsed ` +
|
|
209
|
+
"value across all edits is the pre-Phase-1 freeze this fix closes.",
|
|
210
|
+
).toBeGreaterThan(elapsedSamples[0]);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
expect(answer, "FAIL — no answer arrived within budget; the turn may be wedged.").not.toBeNull();
|
|
214
|
+
} finally {
|
|
215
|
+
await sc.tearDown();
|
|
216
|
+
}
|
|
217
|
+
},
|
|
218
|
+
OVERALL_BUDGET_MS,
|
|
219
|
+
);
|
|
220
|
+
});
|