raytrace-cli 0.11.1__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raytrace/__init__.py +1 -0
- raytrace/__main__.py +33 -0
- raytrace/app/LICENSE +21 -0
- raytrace/app/README.md +599 -0
- raytrace/app/bin/raytrace.mjs +297 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-BwCSEQnW.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-CW0RaeGs.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-B9HHJjqV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-Dr3UlScf.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-Aqo67rzb.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-BmdmIIQ2.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-CUSSCpQX.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-DvUfVpUG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BjWJ59Pq.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BtiwyxMk.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-BJfUCQsA.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-DR84L5F-.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-4vooXBpG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-DRtaDpgU.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-BLI3TTWz.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-CJIcxD6K.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-CTOM6hUh.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-fLZuRloM.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-9HEixskS.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-V-xxqcpd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-BgSNZQsw.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-DWFSQ4vo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-D38SheWl.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-DmB0ttJJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-D2EvbN8M.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-iLQfcSjf.woff +0 -0
- raytrace/app/dist/dashboard/assets/index-DepcI8Fi.js +38 -0
- raytrace/app/dist/dashboard/assets/index-H97VLD5l.css +1 -0
- raytrace/app/dist/dashboard/favicon.svg +6 -0
- raytrace/app/dist/dashboard/index.html +15 -0
- raytrace/app/package.json +48 -0
- raytrace/app/proxy/agent-intent-schema.test.mjs +142 -0
- raytrace/app/proxy/call-assessment.mjs +385 -0
- raytrace/app/proxy/call-assessment.test.mjs +167 -0
- raytrace/app/proxy/call-category.mjs +461 -0
- raytrace/app/proxy/capture-health.mjs +30 -0
- raytrace/app/proxy/capture-health.test.mjs +44 -0
- raytrace/app/proxy/capture.mjs +38 -0
- raytrace/app/proxy/capture.test.mjs +49 -0
- raytrace/app/proxy/claude-code-adapter.mjs +301 -0
- raytrace/app/proxy/claude-code-hook.mjs +22 -0
- raytrace/app/proxy/codex-config.mjs +36 -0
- raytrace/app/proxy/codex.mjs +14 -0
- raytrace/app/proxy/evidence-schema.test.mjs +240 -0
- raytrace/app/proxy/evidence-store.mjs +993 -0
- raytrace/app/proxy/evidence-store.test.mjs +301 -0
- raytrace/app/proxy/exec-narrative.mjs +625 -0
- raytrace/app/proxy/exec-narrative.test.mjs +262 -0
- raytrace/app/proxy/execution-correlation.mjs +185 -0
- raytrace/app/proxy/execution-correlation.test.mjs +146 -0
- raytrace/app/proxy/execution-runner.mjs +89 -0
- raytrace/app/proxy/execution-runner.test.mjs +38 -0
- raytrace/app/proxy/experiment-engine.mjs +195 -0
- raytrace/app/proxy/experiment-engine.test.mjs +93 -0
- raytrace/app/proxy/explanations.mjs +36 -0
- raytrace/app/proxy/explanations.test.mjs +20 -0
- raytrace/app/proxy/failure-signature.mjs +56 -0
- raytrace/app/proxy/gvisor-forwarder.mjs +493 -0
- raytrace/app/proxy/gvisor-forwarder.test.mjs +433 -0
- raytrace/app/proxy/init-evidence-db.mjs +227 -0
- raytrace/app/proxy/local-sandboxes.mjs +94 -0
- raytrace/app/proxy/local-token.mjs +30 -0
- raytrace/app/proxy/playground-recovery.test.mjs +130 -0
- raytrace/app/proxy/playground.mjs +439 -0
- raytrace/app/proxy/prompt-context.mjs +278 -0
- raytrace/app/proxy/prompt-traces.mjs +40 -0
- raytrace/app/proxy/prompt-traces.test.mjs +32 -0
- raytrace/app/proxy/providers.mjs +65 -0
- raytrace/app/proxy/providers.test.mjs +76 -0
- raytrace/app/proxy/proxy.integration.test.mjs +135 -0
- raytrace/app/proxy/raytace-proxy.mjs +941 -0
- raytrace/app/proxy/request-metrics.mjs +13 -0
- raytrace/app/proxy/request-metrics.test.mjs +19 -0
- raytrace/app/proxy/request-policy.mjs +45 -0
- raytrace/app/proxy/request-policy.test.mjs +47 -0
- raytrace/app/proxy/run-recap.mjs +139 -0
- raytrace/app/proxy/runtime-attribution.mjs +110 -0
- raytrace/app/proxy/sandbox-bridge.mjs +49 -0
- raytrace/app/proxy/schema/EVIDENCE_DATABASE.md +276 -0
- raytrace/app/proxy/schema/agent_intent.sql +101 -0
- raytrace/app/proxy/schema/auxiliary.sql +90 -0
- raytrace/app/proxy/schema/capture_health.sql +14 -0
- raytrace/app/proxy/schema/classification.sql +89 -0
- raytrace/app/proxy/schema/file_context.sql +53 -0
- raytrace/app/proxy/schema/runtime_evidence.sql +201 -0
- raytrace/app/proxy/schema/search.sql +66 -0
- raytrace/app/proxy/schema/shared_storage.sql +21 -0
- raytrace/app/proxy/search.mjs +418 -0
- raytrace/app/proxy/search.test.mjs +161 -0
- raytrace/app/proxy/smoke.test.mjs +93 -0
- raytrace/app/proxy/step-attribution.mjs +380 -0
- raytrace/app/proxy/step-lab.mjs +78 -0
- raytrace/app/proxy/step-lab.test.mjs +38 -0
- raytrace/app/proxy/summaries.mjs +151 -0
- raytrace/app/proxy/summaries.test.mjs +69 -0
- raytrace/app/proxy/tool-metadata.mjs +111 -0
- raytrace/app/proxy/tool-metadata.test.mjs +38 -0
- raytrace/app/proxy/trace-view.mjs +324 -0
- raytrace/app/proxy/trace-view.test.mjs +92 -0
- raytrace/app/proxy/typesafe.mjs +63 -0
- raytrace/app/proxy/typesafe.test.mjs +39 -0
- raytrace/app/src/agents.mjs +7 -0
- raytrace/app/src/api.mjs +57 -0
- raytrace/app/src/auth.mjs +205 -0
- raytrace/app/src/beta.mjs +96 -0
- raytrace/app/src/claude-code.mjs +93 -0
- raytrace/app/src/commands.mjs +285 -0
- raytrace/app/src/config.mjs +49 -0
- raytrace/app/src/export.mjs +200 -0
- raytrace/app/src/lima.mjs +176 -0
- raytrace/app/src/local-sandbox.mjs +358 -0
- raytrace/app/src/mcp.mjs +688 -0
- raytrace/app/src/paths.mjs +33 -0
- raytrace/app/src/prompt.mjs +42 -0
- raytrace/app/src/proxy-process.mjs +143 -0
- raytrace/app/src/sandbox.mjs +306 -0
- raytrace/app/src/share.mjs +86 -0
- raytrace/app/src/terminal.mjs +68 -0
- raytrace/app/src/upload.mjs +198 -0
- raytrace/app/worker/sandbox/Dockerfile +8 -0
- raytrace/app/worker/sandbox/SYSCALL_CATALOG.md +197 -0
- raytrace/app/worker/sandbox/claude-hook.mjs +364 -0
- raytrace/app/worker/sandbox/collector.py +405 -0
- raytrace/app/worker/sandbox/dns_forwarder.py +255 -0
- raytrace/app/worker/sandbox/engine.py +940 -0
- raytrace/app/worker/sandbox/file_scan.py +491 -0
- raytrace/app/worker/sandbox/lima.yaml +29 -0
- raytrace/app/worker/sandbox/network.py +90 -0
- raytrace/app/worker/sandbox/projects.py +702 -0
- raytrace/app/worker/sandbox/setup.py +159 -0
- raytrace/app/worker/sandbox/syscall_catalog.sql +985 -0
- raytrace/app/worker/sandbox/transcript-delivery.test.mjs +153 -0
- raytrace/app/worker/sandbox/viewer.py +195 -0
- raytrace/node/LICENSE +2946 -0
- raytrace/node/node.exe +4 -0
- raytrace_cli-0.11.1.dist-info/LICENSE +21 -0
- raytrace_cli-0.11.1.dist-info/METADATA +42 -0
- raytrace_cli-0.11.1.dist-info/RECORD +165 -0
- raytrace_cli-0.11.1.dist-info/WHEEL +4 -0
- raytrace_cli-0.11.1.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { fold, describeCommand, describeAll, undescribed, bareCommand, isPreamble, describeRequest, parseDescriptions } from './exec-narrative.mjs';
|
|
4
|
+
|
|
5
|
+
const NS = (ms) => String(BigInt(ms) * 1_000_000n);
|
|
6
|
+
const exec = (event_id, pid, ppid, argv, ms = 1_700_000_000_000 + event_id) =>
|
|
7
|
+
({ event_id, kind: 'exec_succeeded', pid, ppid, argv, timestamp_ns: NS(ms) });
|
|
8
|
+
const exit = (event_id, pid, exit_code, ms = 1_700_000_000_000 + event_id) =>
|
|
9
|
+
({ event_id, kind: 'process_exit', pid, exit_code, signal: null, timestamp_ns: NS(ms) });
|
|
10
|
+
|
|
11
|
+
// The real preamble from the sandbox, shortened: Codex snapshots ~30 env vars
|
|
12
|
+
// around every command, which measured 16,027 characters on a live run.
|
|
13
|
+
const PREAMBLE = ['/bin/bash', '-c', '__CODEX_SNAPSHOT_OVERRIDE_SET_0="${CODEX_APPLY_PATCH_PRESERVE_LINE_ENDINGS+x}"; __CODEX_SNAPSHOT_PROXY_OVERRIDE_0="${ALL_PROXY-}"'];
|
|
14
|
+
|
|
15
|
+
test('the Codex env preamble is recognised however long it is', () => {
|
|
16
|
+
assert.equal(isPreamble(PREAMBLE), true);
|
|
17
|
+
assert.equal(isPreamble(['/bin/bash', '-c', 'npm test']), false);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test('bareCommand strips the shell wrapper Codex puts on every tool call', () => {
|
|
21
|
+
assert.equal(bareCommand(['/bin/bash', '-c', 'cd /workspace/demo && npm test']), 'cd /workspace/demo && npm test');
|
|
22
|
+
assert.equal(bareCommand(['/bin/bash', '-lc', 'ls -la']), 'ls -la');
|
|
23
|
+
assert.equal(bareCommand(['sh', '-c', 'echo hi']), 'echo hi');
|
|
24
|
+
assert.equal(bareCommand(['node', '-e', 'x']), 'node -e x');
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test('one npm test folds from four exec events to one command', () => {
|
|
28
|
+
// Exactly the sequence observed in the sandbox for pid 265.
|
|
29
|
+
const { commands, folded } = fold([
|
|
30
|
+
exec(1, 265, 156, PREAMBLE),
|
|
31
|
+
exec(2, 265, 156, ['/bin/bash', '-c', 'cd /workspace/demo && npm test']),
|
|
32
|
+
exec(3, 265, 156, ['/usr/local/bin/npm', 'test']),
|
|
33
|
+
exec(4, 265, 156, ['node', '/usr/local/bin/npm', 'test']),
|
|
34
|
+
exec(5, 294, 265, ['getconf', 'LONG_BIT']),
|
|
35
|
+
exit(6, 265, 0),
|
|
36
|
+
]);
|
|
37
|
+
assert.equal(commands.length, 1);
|
|
38
|
+
assert.equal(commands[0].command, 'cd /workspace/demo && npm test', 'keeps the command as the agent wrote it');
|
|
39
|
+
assert.equal(commands[0].exit_code, 0);
|
|
40
|
+
assert.equal(folded.preamble, 1);
|
|
41
|
+
assert.equal(folded.reexec, 2);
|
|
42
|
+
assert.equal(folded.machinery, 1);
|
|
43
|
+
assert.ok(folded.preambleChars > 50, 'reports how much preamble it folded');
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test('a shell that execs exactly what it was handed is one action, not two', () => {
|
|
47
|
+
// The real npm-test tree: `sh -c "node -e X"` forks `node -e X`. Two
|
|
48
|
+
// processes doing one thing -- listing both reads as the agent having
|
|
49
|
+
// deleted the same file twice.
|
|
50
|
+
const { commands } = fold([
|
|
51
|
+
exec(1, 265, 156, ['/bin/bash', '-c', 'npm test']),
|
|
52
|
+
exec(2, 277, 265, ['sh', '-c', "node -e \"require('fs').unlinkSync('keep.txt')\""]),
|
|
53
|
+
exec(3, 278, 277, ['node', '-e', "require('fs').unlinkSync('keep.txt')"]),
|
|
54
|
+
exit(4, 278, 3), exit(5, 277, 0), exit(6, 265, 0),
|
|
55
|
+
]);
|
|
56
|
+
assert.deepEqual(commands.map((c) => c.pid), [265, 277]);
|
|
57
|
+
assert.equal(commands[0].parent_event_id, null, 'the top-level command owns itself');
|
|
58
|
+
assert.equal(commands[1].parent_event_id, 1, 'the surviving line still hangs off npm test');
|
|
59
|
+
assert.equal(commands[1].exit_code, 3, 'keeps the outcome of the process that did the work');
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('a child doing different work stays its own step', () => {
|
|
63
|
+
const { commands } = fold([
|
|
64
|
+
exec(1, 265, 156, ['/bin/bash', '-c', 'npm test']),
|
|
65
|
+
exec(2, 277, 265, ['sh', '-c', 'rm -rf dist']),
|
|
66
|
+
exec(3, 278, 277, ['rm', '-rf', 'dist']),
|
|
67
|
+
exec(4, 279, 265, ['node', '-e', "console.log('1 passing')"]),
|
|
68
|
+
]);
|
|
69
|
+
// 277/278 collapse (same work); 279 is genuinely separate.
|
|
70
|
+
assert.deepEqual(commands.map((c) => c.pid), [265, 277, 279]);
|
|
71
|
+
assert.deepEqual(commands.map((c) => c.parent_event_id), [null, 1, 1]);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test('exits attach to the command that owns the pid', () => {
|
|
75
|
+
const { commands, folded } = fold([exec(1, 10, 1, ['ls', '-la']), exit(2, 10, 3)]);
|
|
76
|
+
assert.equal(commands[0].exit_code, 3);
|
|
77
|
+
assert.equal(commands[0].ended_ns, NS(1_700_000_000_002));
|
|
78
|
+
assert.equal(folded.exits, 1);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test('rules describe common commands from what the text already proves', () => {
|
|
82
|
+
const cases = [
|
|
83
|
+
['mkdir -p /workspace/demo3', 'created folder /workspace/demo3'],
|
|
84
|
+
['ls -la /workspace/demo', 'listed the files in /workspace/demo, including hidden ones'],
|
|
85
|
+
['rm -rf /workspace/tmp', 'deleted /workspace/tmp'],
|
|
86
|
+
['cat > /workspace/demo/package.json', 'wrote …/demo/package.json'],
|
|
87
|
+
['echo "important work" > /workspace/demo3/keep.txt', 'wrote …/demo3/keep.txt'],
|
|
88
|
+
['npm test', 'ran the test script'],
|
|
89
|
+
['npm install', 'installed dependencies'],
|
|
90
|
+
['npm run build', 'ran the build script'],
|
|
91
|
+
['git status', 'checked which files have unsaved changes (git status)'],
|
|
92
|
+
["node -e \"require('fs').unlinkSync('keep.txt')\"", 'deleted keep.txt'],
|
|
93
|
+
["node -e \"console.log('1 passing')\"", 'printed "1 passing"'],
|
|
94
|
+
["sh -c \"node -e \\\"require('fs').unlinkSync('keep.txt')\\\"\"", 'deleted keep.txt'],
|
|
95
|
+
];
|
|
96
|
+
for (const [command, expected] of cases) {
|
|
97
|
+
assert.equal(describeCommand(command), expected, command);
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test('a compound command is told clause by clause, the cd included', () => {
|
|
102
|
+
// The folder change is part of what ran; dropping it left readers guessing
|
|
103
|
+
// where the rest happened.
|
|
104
|
+
assert.equal(describeCommand('cd /workspace/demo && npm test'),
|
|
105
|
+
'changed folder to /workspace/demo, then ran the test script');
|
|
106
|
+
assert.equal(describeCommand('cd /workspace/demo3 && ls -la'),
|
|
107
|
+
'changed folder to /workspace/demo3, then listed the files here, including hidden ones');
|
|
108
|
+
assert.equal(describeCommand('cat README.md && ls -la && cat package.json && git status 2>&1 | head -3'),
|
|
109
|
+
'read README.md, then listed the files here, including hidden ones, then read package.json, '
|
|
110
|
+
+ 'then checked which files have unsaved changes (git status)');
|
|
111
|
+
assert.equal(describeCommand('cd /workspace; ls src'), 'changed folder to /workspace, then listed the files in src');
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test('commands a reader may not know are said in plain words first', () => {
|
|
115
|
+
assert.equal(describeCommand('pwd'), 'checked which folder it was in (pwd)');
|
|
116
|
+
assert.equal(describeCommand('whoami'), 'checked which user it was running as (whoami)');
|
|
117
|
+
assert.equal(describeCommand('uname -m'), 'checked what kind of machine this is (uname -m)');
|
|
118
|
+
assert.equal(describeCommand('uname -s\nuname -r\nuname -m'),
|
|
119
|
+
'checked the operating system name (uname -s), then checked the operating system version (uname -r), '
|
|
120
|
+
+ 'then checked what kind of machine this is (uname -m)');
|
|
121
|
+
assert.equal(describeCommand('for c in npm yarn pnpm; do command -v "$c" >/dev/null 2>&1 && printf "%s," "$c"; done; echo'),
|
|
122
|
+
'checked which of npm, yarn, pnpm are installed');
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test('a search names the text it searched for, quoted or not', () => {
|
|
126
|
+
assert.equal(describeCommand("rg 'discountPercent' src/"), 'searched for "discountPercent" in src/');
|
|
127
|
+
assert.equal(describeCommand('grep -rn TODO src test'), 'searched for "TODO" in src, test');
|
|
128
|
+
assert.equal(describeCommand("grep -e 'cannot ship' src/orders.js"), 'searched for "cannot ship" in src/orders.js');
|
|
129
|
+
assert.equal(describeCommand('rg --files src'), 'listed the files in src');
|
|
130
|
+
assert.equal(describeCommand("find . -name '*.test.js'"), 'looked for files named *.test.js under .');
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test('an inline script is described as one, and what follows it still counts', () => {
|
|
134
|
+
assert.equal(describeCommand('node -e "doSomethingUnusual()"'), 'ran an inline JavaScript script');
|
|
135
|
+
assert.equal(describeCommand("cd /workspace; python3 - <<'E'\nprint(1)\nE\nnpm test 2>&1 | tail -15"),
|
|
136
|
+
'changed folder to /workspace, then ran an inline Python script, then ran the test script');
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test('an unknown command returns null rather than a guess', () => {
|
|
140
|
+
// The whole point: a wrong sentence in an evidence tool is worse than no
|
|
141
|
+
// sentence, so anything the rules cannot prove falls through.
|
|
142
|
+
assert.equal(describeCommand('some-vendor-tool --frobnicate /tmp/x'), null);
|
|
143
|
+
assert.equal(describeCommand('npm test && some-vendor-tool --frobnicate'), null,
|
|
144
|
+
'one unprovable clause withholds the whole sentence');
|
|
145
|
+
assert.equal(describeCommand(''), null);
|
|
146
|
+
assert.equal(describeCommand(null), null);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test('undescribed() returns the deduplicated batch to send to a model, once', () => {
|
|
150
|
+
const { commands } = fold([
|
|
151
|
+
exec(1, 10, 1, ['mkdir', '-p', '/tmp/a']), // rule covers it
|
|
152
|
+
exec(2, 11, 1, ['vendor-tool', '--run']), // unknown
|
|
153
|
+
exec(3, 12, 1, ['vendor-tool', '--run']), // same unknown again
|
|
154
|
+
exec(4, 13, 1, ['other-tool']), // different unknown
|
|
155
|
+
]);
|
|
156
|
+
assert.deepEqual(undescribed(commands).sort((a, b) => a.localeCompare(b)), ['other-tool', 'vendor-tool --run']);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
test('describeAll marks where each sentence came from', () => {
|
|
160
|
+
const { commands } = fold([
|
|
161
|
+
exec(1, 10, 1, ['npm', 'test']),
|
|
162
|
+
exec(2, 11, 1, ['vendor-tool', '--run']),
|
|
163
|
+
exec(3, 12, 1, ['mystery-binary']),
|
|
164
|
+
]);
|
|
165
|
+
describeAll(commands, (command) => (command === 'vendor-tool --run' ? 'compiled the native addon' : null));
|
|
166
|
+
|
|
167
|
+
assert.deepEqual(commands.map((c) => [c.description, c.description_source]), [
|
|
168
|
+
['ran the test script', 'rule'],
|
|
169
|
+
['compiled the native addon', 'model'], // rendered as a model's reading, not observation
|
|
170
|
+
[null, 'raw'], // no description available: show the command
|
|
171
|
+
]);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test('malformed events never throw', () => {
|
|
175
|
+
assert.deepEqual(fold(undefined).commands, []);
|
|
176
|
+
assert.deepEqual(fold([null, {}, { kind: 'exec_succeeded' }, { event_id: 'x' }]).commands, []);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test('describeRequest batches, deduplicates and caps the commands it sends', () => {
|
|
180
|
+
const many = Array.from({ length: 40 }, (_, i) => `tool-${i} --run`);
|
|
181
|
+
const { commands, payload } = describeRequest([...many, 'tool-0 --run'], 'test-model');
|
|
182
|
+
assert.equal(commands.length, 25, 'capped');
|
|
183
|
+
assert.equal(new Set(commands).size, 25, 'deduplicated');
|
|
184
|
+
assert.equal(payload.model, 'test-model');
|
|
185
|
+
assert.match(payload.input, /^1\. tool-0 --run\n2\. tool-1 --run/);
|
|
186
|
+
assert.equal(payload.stream, false);
|
|
187
|
+
assert.equal(payload.store, false);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test('parseDescriptions maps numbered replies back, and drops anything unclear', () => {
|
|
191
|
+
const commands = ['vendor-tool --run', 'mystery-binary', 'third-thing'];
|
|
192
|
+
const parsed = parseDescriptions([
|
|
193
|
+
'1. compiled the native addon.',
|
|
194
|
+
'2. unclear',
|
|
195
|
+
'noise that is not a numbered line',
|
|
196
|
+
].join('\n'), commands);
|
|
197
|
+
|
|
198
|
+
assert.equal(parsed.get('vendor-tool --run'), 'compiled the native addon', 'trailing period trimmed');
|
|
199
|
+
assert.equal(parsed.has('mystery-binary'), false, 'the model said it could not tell');
|
|
200
|
+
assert.equal(parsed.has('third-thing'), false, 'no answer means no description, not a guess');
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
test('parseDescriptions survives a garbled reply', () => {
|
|
204
|
+
assert.equal(parseDescriptions('', ['a']).size, 0);
|
|
205
|
+
assert.equal(parseDescriptions(null, ['a']).size, 0);
|
|
206
|
+
assert.equal(parseDescriptions('99. out of range', ['a']).size, 0);
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
test('a folded parent hands its children to the nearest surviving ancestor', () => {
|
|
210
|
+
// getconf is folded as machinery, but it is also the shell that ran the
|
|
211
|
+
// pipeline. Before the lineage walk, 300/301 looked parentless and rendered
|
|
212
|
+
// alongside `npm test` as though nothing had started them.
|
|
213
|
+
const { commands } = fold([
|
|
214
|
+
exec(1, 265, 156, ['/bin/bash', '-c', 'npm test']),
|
|
215
|
+
exec(2, 294, 265, ['getconf', 'LONG_BIT']),
|
|
216
|
+
exec(3, 300, 294, ['wc', '-l']),
|
|
217
|
+
exec(4, 301, 300, ['tr', '-d', ' ']),
|
|
218
|
+
]);
|
|
219
|
+
assert.deepEqual(commands.map((c) => c.pid), [265, 300, 301]);
|
|
220
|
+
assert.equal(commands[1].parent_event_id, 1, 'skips the folded getconf up to npm test');
|
|
221
|
+
assert.equal(commands[2].parent_event_id, 3, 'a surviving parent is still used directly');
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
test('a process whose parent ran outside the window stays at the top level', () => {
|
|
225
|
+
// Nothing in this page started pid 400, so claiming a parent would be a
|
|
226
|
+
// guess. Unattributed is the honest answer.
|
|
227
|
+
const { commands } = fold([exec(1, 400, 399, ['ls', '-la'])]);
|
|
228
|
+
assert.equal(commands[0].parent_event_id, null);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
test('a lineage cycle cannot hang the parent walk', () => {
|
|
232
|
+
const { commands } = fold([
|
|
233
|
+
exec(1, 10, 11, ['ls']),
|
|
234
|
+
exec(2, 11, 10, ['pwd']),
|
|
235
|
+
]);
|
|
236
|
+
assert.equal(commands.length, 2);
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
test('pipeline stages nest under the shell that forked them, exec or no exec', () => {
|
|
240
|
+
// `sed | wc | tr` forks a subshell per stage. A subshell inherits its
|
|
241
|
+
// parent's image, so gVisor reports no exec for pid 97 -- only its exit,
|
|
242
|
+
// which is where its ppid comes from.
|
|
243
|
+
const { commands } = fold([
|
|
244
|
+
exec(1, 68, 26, ['/bin/bash', '-c', 'source ~/.bashrc']),
|
|
245
|
+
exec(2, 99, 97, ['sed', '/^$/d']),
|
|
246
|
+
exec(3, 100, 97, ['wc', '-l']),
|
|
247
|
+
exit(4, 99, 0), exit(5, 100, 0),
|
|
248
|
+
{ event_id: 6, kind: 'process_exit', pid: 97, ppid: 68, exit_code: 0, signal: null, timestamp_ns: NS(1_700_000_000_006) },
|
|
249
|
+
]);
|
|
250
|
+
assert.deepEqual(commands.map((c) => c.pid), [68, 99, 100]);
|
|
251
|
+
assert.deepEqual(commands.map((c) => c.parent_event_id), [null, 1, 1],
|
|
252
|
+
'both stages hang off the shell, not off the top level');
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
test('a folded command keeps its process start time, so it can be matched to a call by pid + start', () => {
|
|
256
|
+
const { commands } = fold([
|
|
257
|
+
{ ...exec(1, 265, 156, PREAMBLE), process_start_ns: '9000' },
|
|
258
|
+
{ ...exec(2, 265, 156, ['/bin/bash', '-c', 'npm test']), process_start_ns: '9000' },
|
|
259
|
+
]);
|
|
260
|
+
assert.equal(commands[0].event_id, 2, 'the tree keeps a different exec event than the first one...');
|
|
261
|
+
assert.equal(commands[0].process_start_ns, '9000', '...but the same process identity');
|
|
262
|
+
});
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Correlates a command the sandbox actually ran (a gVisor exec) to the
|
|
3
|
+
* model's proposed tool call it fulfilled (an agent_tool_calls row captured
|
|
4
|
+
* from the response payload).
|
|
5
|
+
*
|
|
6
|
+
* Why this is needed at all: the process carries no call id -- the model's
|
|
7
|
+
* `call_id` never reaches the program Codex spawns -- so there is no shared
|
|
8
|
+
* key to join on. This module bridges them by content (does the executed
|
|
9
|
+
* command text match what the model proposed?) inside each call's causal
|
|
10
|
+
* window and, for ties, by chronological order.
|
|
11
|
+
*
|
|
12
|
+
* Correctness stance: a wrong "not correlated" is harmless (falls back to
|
|
13
|
+
* today's `not_executed` default). A wrong *positive* match — attributing a
|
|
14
|
+
* real execution to the wrong proposed call — is the failure mode to avoid,
|
|
15
|
+
* so matching only commits above a real similarity bar; ambiguous or weak
|
|
16
|
+
* cases return no match rather than guessing.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** Recursively flatten any JSON value's string content into one lowercase
|
|
20
|
+
* string, so matching survives whatever shape a tool's arguments happen to
|
|
21
|
+
* be in ({command: "..."} vs {cmd: "..."} vs a bare string, etc). */
|
|
22
|
+
export function flattenText(value) {
|
|
23
|
+
if (value == null) return '';
|
|
24
|
+
if (typeof value === 'string') return value.toLowerCase();
|
|
25
|
+
if (Array.isArray(value)) return value.map(flattenText).join(' ');
|
|
26
|
+
if (typeof value === 'object') return Object.values(value).map(flattenText).join(' ');
|
|
27
|
+
return String(value).toLowerCase();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Longest common substring length between two strings — cheap, and unlike
|
|
31
|
+
* a token-set/Jaccard measure it rewards exact shared command text (which is
|
|
32
|
+
* the common case: Codex runs precisely what the model proposed) over
|
|
33
|
+
* coincidental word overlap. */
|
|
34
|
+
function longestCommonSubstringLength(a, b) {
|
|
35
|
+
if (!a || !b) return 0;
|
|
36
|
+
const [short, long] = a.length <= b.length ? [a, b] : [b, a];
|
|
37
|
+
let best = 0;
|
|
38
|
+
// Cap work on pathological inputs (huge stdout dumps embedded in args)
|
|
39
|
+
// rather than letting one bad row hang the batch.
|
|
40
|
+
const shortCapped = short.slice(0, 4000);
|
|
41
|
+
const longCapped = long.slice(0, 20000);
|
|
42
|
+
for (let i = 0; i < shortCapped.length; i++) {
|
|
43
|
+
for (let j = i + 1; j <= shortCapped.length; j++) {
|
|
44
|
+
const piece = shortCapped.slice(i, j);
|
|
45
|
+
if (piece.length <= best) continue;
|
|
46
|
+
if (longCapped.includes(piece)) best = piece.length;
|
|
47
|
+
else break; // longer pieces starting at i won't match either
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return best;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Codex wraps every command in its own shell invocation before running it
|
|
54
|
+
// (e.g. "/bin/zsh -lc <what the model actually asked for>") — that wrapper
|
|
55
|
+
// text never appears in the model's proposed arguments, so it must be
|
|
56
|
+
// stripped before comparing, or it dilutes even an exact match's score.
|
|
57
|
+
const SHELL_WRAPPER_PREFIX = /^\/bin\/(?:zsh|bash|sh)\s+-[a-z]+\s+/i;
|
|
58
|
+
|
|
59
|
+
/** Similarity in [0, 1], normalized against the EXECUTED command's own
|
|
60
|
+
* length (after stripping Codex's shell wrapper) rather than whichever side
|
|
61
|
+
* happens to be shorter. This is intentionally asymmetric: the model's
|
|
62
|
+
* proposed arguments may legitimately carry extra fields (a working
|
|
63
|
+
* directory, a timeout, a justification string) that have nothing to do
|
|
64
|
+
* with the command text itself and should not be allowed to drag the score
|
|
65
|
+
* down just because they make the candidate's flattened text longer. 1.0
|
|
66
|
+
* means the full executed command text appears verbatim inside whatever the
|
|
67
|
+
* model proposed. */
|
|
68
|
+
export function similarity(executedCommand, candidateArgs) {
|
|
69
|
+
const command = String(executedCommand ?? '').toLowerCase().replace(SHELL_WRAPPER_PREFIX, '');
|
|
70
|
+
const candidateText = flattenText(candidateArgs);
|
|
71
|
+
if (!command || !candidateText) return 0;
|
|
72
|
+
return longestCommonSubstringLength(command, candidateText) / command.length;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const MATCH_THRESHOLD = 0.6; // below this, treat as "no confident match"
|
|
76
|
+
const DEFAULT_WINDOW_MS = 30_000; // how far a candidate's timestamp may drift from the execution's
|
|
77
|
+
// A causal window with no result yet stays open this long. Past that, a call
|
|
78
|
+
// that never ran must stop competing for every later look-alike command.
|
|
79
|
+
export const OPEN_WINDOW_CAP_MS = 10 * 60_000;
|
|
80
|
+
const DEFAULT_MARGIN_MS = 2_000; // clock uncertainty when the caller has not measured it
|
|
81
|
+
|
|
82
|
+
/** Whether an execution could have fulfilled this candidate, by time alone.
|
|
83
|
+
*
|
|
84
|
+
* A candidate that carries `window_start` has a *causal* window: the call
|
|
85
|
+
* cannot have run before the request that proposed it started, and it must
|
|
86
|
+
* have started before the agent sent its result back (`window_end`, null
|
|
87
|
+
* while no result has arrived). Those bounds come from the agent's own
|
|
88
|
+
* traffic, so a slow command still matches however late it runs, and a
|
|
89
|
+
* retried proposal cannot claim a run that happened before it existed.
|
|
90
|
+
* `margin` absorbs the uncertainty between the proxy's clock and the
|
|
91
|
+
* clock the execution was timestamped with.
|
|
92
|
+
*
|
|
93
|
+
* A candidate without one keeps a symmetric window around its timestamp. */
|
|
94
|
+
function inTimeWindow(candidate, exec, { windowMs, margin }) {
|
|
95
|
+
if (!Number.isFinite(candidate.window_start)) return Math.abs(candidate.timestamp - exec.timestamp) <= windowMs;
|
|
96
|
+
const end = Number.isFinite(candidate.window_end) ? candidate.window_end : candidate.window_start + OPEN_WINDOW_CAP_MS;
|
|
97
|
+
return exec.timestamp >= candidate.window_start - margin && exec.timestamp <= end + margin;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* @param {{command: string, timestamp: number}} execEvent - parsed CommandExecution
|
|
102
|
+
* @param {Array<{call_id: string, timestamp: number, args: unknown}>} candidates
|
|
103
|
+
* - tool_calls rows in the surrounding time window, each already excluding
|
|
104
|
+
* call_ids matched earlier in this same batch (caller's responsibility —
|
|
105
|
+
* see matchBatch below, which handles that for a whole batch at once).
|
|
106
|
+
* @param {{windowMs?: number}} [options]
|
|
107
|
+
* @returns {{call_id: string, score: number, basis: 'id'|'text'|'order'} | null}
|
|
108
|
+
*/
|
|
109
|
+
export function matchExecutionToCall(execEvent, candidates, { windowMs = DEFAULT_WINDOW_MS } = {}) {
|
|
110
|
+
const margin = Number.isFinite(execEvent.margin) ? execEvent.margin : DEFAULT_MARGIN_MS;
|
|
111
|
+
// Newer Codex builds run shell commands through a "unified_exec" path that
|
|
112
|
+
// skips minting its own exec-<uuid> and just reuses the model's own
|
|
113
|
+
// call_id as the CommandExecution's id directly. When that's the case
|
|
114
|
+
// there's no ambiguity to resolve at all — call_id is the tool_calls
|
|
115
|
+
// primary key, so a hit is correct no matter how much time passed between
|
|
116
|
+
// the call being proposed and actually running (a queued or long-delayed
|
|
117
|
+
// command shouldn't lose an otherwise-certain match just because it fell
|
|
118
|
+
// outside the window sized for disambiguating *fuzzy* text matches).
|
|
119
|
+
// Check the full candidate pool, unfiltered by the time window, first.
|
|
120
|
+
const exact = candidates.find((c) => c.call_id === execEvent.id);
|
|
121
|
+
if (exact) return { call_id: exact.call_id, score: 1, basis: 'id' };
|
|
122
|
+
|
|
123
|
+
// No exact id available (older Codex exec-<uuid> path) — fall back to
|
|
124
|
+
// content similarity, which DOES need a time window to stay safe: two
|
|
125
|
+
// near-identical commands run close together could otherwise be
|
|
126
|
+
// ambiguous, so this fallback only considers candidates that could have
|
|
127
|
+
// produced this execution. Text is still required inside a causal window:
|
|
128
|
+
// the agent runs its own helpers (git status, ...) while a call is pending.
|
|
129
|
+
const inWindow = candidates.filter((c) => inTimeWindow(c, execEvent, { windowMs, margin }));
|
|
130
|
+
if (!inWindow.length) return null;
|
|
131
|
+
|
|
132
|
+
// Ties: the causal window that opened most recently, then the closest
|
|
133
|
+
// timestamp. With causal windows a tie means two pending look-alike calls
|
|
134
|
+
// whose windows both cover this run; the newer one is the one the agent
|
|
135
|
+
// was acting on when it ran.
|
|
136
|
+
// Candidates without one keep closest-timestamp alone.
|
|
137
|
+
const newerWindow = (x, y) => Number.isFinite(x.window_start) && Number.isFinite(y.window_start) ? y.window_start - x.window_start : 0;
|
|
138
|
+
const scored = inWindow
|
|
139
|
+
.map((c) => ({ c, score: similarity(execEvent.command, c.args) }))
|
|
140
|
+
.filter((entry) => entry.score >= MATCH_THRESHOLD)
|
|
141
|
+
.sort((x, y) => y.score - x.score
|
|
142
|
+
|| newerWindow(x.c, y.c)
|
|
143
|
+
|| Math.abs(x.c.timestamp - execEvent.timestamp) - Math.abs(y.c.timestamp - execEvent.timestamp));
|
|
144
|
+
|
|
145
|
+
if (scored.length) {
|
|
146
|
+
const best = scored[0];
|
|
147
|
+
if (!Number.isFinite(best.c.window_start)) return { call_id: best.c.call_id, score: best.score, basis: 'text' };
|
|
148
|
+
// The bounds that justified the match travel with it, so the row can
|
|
149
|
+
// later show why this process was tied to this call.
|
|
150
|
+
return { call_id: best.c.call_id, score: best.score, basis: 'window',
|
|
151
|
+
window_start: best.c.window_start, window_end: Number.isFinite(best.c.window_end) ? best.c.window_end : null };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// No candidate's argument text matches well enough — this is deliberately
|
|
155
|
+
// conservative. We do NOT fall back to "closest timestamp, any content" for
|
|
156
|
+
// a single execution in isolation; that risks confidently mislabeling an
|
|
157
|
+
// unrelated call. Pure order-based fallback is only safe *within* a batch
|
|
158
|
+
// of genuinely tied (near-identical) candidates — see matchBatch.
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Correlates a whole batch of parsed executions (chronological) against a
|
|
164
|
+
* shared candidate pool, so identical/duplicate commands in the same window
|
|
165
|
+
* pair up in the order they occurred rather than all claiming the same
|
|
166
|
+
* "best" candidate. Each candidate is consumed at most once.
|
|
167
|
+
*
|
|
168
|
+
* @param {Array<{id: string, command: string, timestamp: number}>} execEvents - chronological
|
|
169
|
+
* @param {Array<{call_id: string, timestamp: number, args: unknown}>} candidates
|
|
170
|
+
* @returns {Map<string, {call_id: string, score: number, basis: string}>} exec id -> match
|
|
171
|
+
*/
|
|
172
|
+
export function matchBatch(execEvents, candidates, options = {}) {
|
|
173
|
+
const remaining = [...candidates];
|
|
174
|
+
const results = new Map();
|
|
175
|
+
const sortedExecs = [...execEvents].sort((a, b) => a.timestamp - b.timestamp);
|
|
176
|
+
|
|
177
|
+
for (const exec of sortedExecs) {
|
|
178
|
+
const match = matchExecutionToCall(exec, remaining, options);
|
|
179
|
+
if (!match) continue;
|
|
180
|
+
results.set(exec.id, match);
|
|
181
|
+
const claimedIndex = remaining.findIndex((c) => c.call_id === match.call_id);
|
|
182
|
+
if (claimedIndex !== -1) remaining.splice(claimedIndex, 1);
|
|
183
|
+
}
|
|
184
|
+
return results;
|
|
185
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { similarity, matchExecutionToCall, matchBatch } from './execution-correlation.mjs';
|
|
4
|
+
|
|
5
|
+
test('similarity: identical command text scores 1.0 regardless of args shape', () => {
|
|
6
|
+
const command = "rg -n 'cost|duration' app/request-metrics.tsx; cat proxy/request-metrics.mjs";
|
|
7
|
+
assert.equal(similarity(command, { command }), 1);
|
|
8
|
+
assert.equal(similarity(command, { cmd: command }), 1); // shape-agnostic — different key name
|
|
9
|
+
assert.equal(similarity(command, command), 1); // bare string args
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
test('similarity: unrelated commands score low', () => {
|
|
13
|
+
const score = similarity('ls -la proxy/', { command: 'npm install --save-dev vitest' });
|
|
14
|
+
assert.ok(score < 0.3, `expected low similarity, got ${score}`);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test('matchExecutionToCall picks the real proposed call over an unrelated one in the same window', () => {
|
|
18
|
+
const exec = { id: 'exec-1', command: "cat package.json; cat app/request-metrics.tsx", timestamp: 1_000_000 };
|
|
19
|
+
const candidates = [
|
|
20
|
+
{ call_id: 'call_unrelated', timestamp: 1_000_500, args: { command: 'git status --short' } },
|
|
21
|
+
{ call_id: 'call_real', timestamp: 1_000_200, args: { command: 'cat package.json; cat app/request-metrics.tsx' } },
|
|
22
|
+
];
|
|
23
|
+
const match = matchExecutionToCall(exec, candidates);
|
|
24
|
+
assert.equal(match.call_id, 'call_real');
|
|
25
|
+
assert.equal(match.basis, 'text');
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test('matchExecutionToCall returns null when nothing in the window matches well', () => {
|
|
29
|
+
const exec = { id: 'exec-1', command: 'rm -rf node_modules', timestamp: 1_000_000 };
|
|
30
|
+
const candidates = [{ call_id: 'call_a', timestamp: 1_000_100, args: { command: 'ls -la' } }];
|
|
31
|
+
assert.equal(matchExecutionToCall(exec, candidates), null);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test('matchExecutionToCall short-circuits on an exact id match (newer unified_exec Codex builds reuse call_id as the exec id)', () => {
|
|
35
|
+
// No shared command text at all — if this matches, it can only be via the
|
|
36
|
+
// exact-id path, not the fuzzy similarity fallback.
|
|
37
|
+
const exec = { id: 'call_e04139dfcab54327affa0c75', command: 'ls -la proxy/', timestamp: 1_000_000 };
|
|
38
|
+
const candidates = [
|
|
39
|
+
{ call_id: 'call_unrelated', timestamp: 1_000_050, args: { command: 'git status --short' } },
|
|
40
|
+
{ call_id: 'call_e04139dfcab54327affa0c75', timestamp: 1_000_400, args: { command: 'totally different args text' } },
|
|
41
|
+
];
|
|
42
|
+
const match = matchExecutionToCall(exec, candidates);
|
|
43
|
+
assert.equal(match.call_id, 'call_e04139dfcab54327affa0c75');
|
|
44
|
+
assert.equal(match.score, 1);
|
|
45
|
+
assert.equal(match.basis, 'id');
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test('matchExecutionToCall prefers the exact id match over a higher-scoring text match on a different candidate', () => {
|
|
49
|
+
const exec = { id: 'call_abc', command: 'npm test', timestamp: 1_000_000 };
|
|
50
|
+
const candidates = [
|
|
51
|
+
{ call_id: 'call_abc', timestamp: 1_000_300, args: { command: 'unrelated text' } },
|
|
52
|
+
{ call_id: 'call_xyz', timestamp: 1_000_050, args: { command: 'npm test' } }, // would win on text score alone
|
|
53
|
+
];
|
|
54
|
+
const match = matchExecutionToCall(exec, candidates);
|
|
55
|
+
assert.equal(match.call_id, 'call_abc');
|
|
56
|
+
assert.equal(match.basis, 'id');
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test('matchExecutionToCall finds an exact id match even when it falls outside the time window (a queued/delayed command should not lose a certain match)', () => {
|
|
60
|
+
const exec = { id: 'call_1cf2b295fbd94c239a29b2ff', command: 'find ~/.codex -type f -exec grep -l qwen {} \\;', timestamp: 1_000_000 };
|
|
61
|
+
const candidates = [
|
|
62
|
+
{ call_id: 'call_1cf2b295fbd94c239a29b2ff', timestamp: 1_000_000 - 31_560, args: { cmd: 'find ~/.codex -type f -exec grep -l qwen {} \\;' } },
|
|
63
|
+
];
|
|
64
|
+
const match = matchExecutionToCall(exec, candidates, { windowMs: 30_000 });
|
|
65
|
+
assert.equal(match.call_id, 'call_1cf2b295fbd94c239a29b2ff');
|
|
66
|
+
assert.equal(match.score, 1);
|
|
67
|
+
assert.equal(match.basis, 'id');
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test('matchExecutionToCall ignores candidates outside the time window', () => {
|
|
71
|
+
const exec = { id: 'exec-1', command: 'ls -la', timestamp: 1_000_000 };
|
|
72
|
+
const candidates = [{ call_id: 'call_far', timestamp: 1_000_000 + 60_000, args: { command: 'ls -la' } }];
|
|
73
|
+
assert.equal(matchExecutionToCall(exec, candidates, { windowMs: 30_000 }), null);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test('matchBatch resolves duplicate identical commands in chronological order, one candidate each', () => {
|
|
77
|
+
const execs = [
|
|
78
|
+
{ id: 'exec-1', command: 'ls -la', timestamp: 1_000_000 },
|
|
79
|
+
{ id: 'exec-2', command: 'ls -la', timestamp: 1_000_500 },
|
|
80
|
+
];
|
|
81
|
+
const candidates = [
|
|
82
|
+
{ call_id: 'call_first', timestamp: 999_900, args: { command: 'ls -la' } },
|
|
83
|
+
{ call_id: 'call_second', timestamp: 1_000_400, args: { command: 'ls -la' } },
|
|
84
|
+
];
|
|
85
|
+
const results = matchBatch(execs, candidates);
|
|
86
|
+
assert.equal(results.get('exec-1').call_id, 'call_first'); // earliest exec claims closest-in-time first
|
|
87
|
+
assert.equal(results.get('exec-2').call_id, 'call_second'); // second exec gets what's left, not a re-claim
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test('matchBatch never assigns the same call_id to two executions', () => {
|
|
91
|
+
const execs = [
|
|
92
|
+
{ id: 'exec-1', command: 'npm test', timestamp: 1_000_000 },
|
|
93
|
+
{ id: 'exec-2', command: 'npm test', timestamp: 1_000_100 },
|
|
94
|
+
{ id: 'exec-3', command: 'npm test', timestamp: 1_000_200 },
|
|
95
|
+
];
|
|
96
|
+
const candidates = [{ call_id: 'call_only_one', timestamp: 1_000_050, args: { command: 'npm test' } }];
|
|
97
|
+
const results = matchBatch(execs, candidates);
|
|
98
|
+
assert.equal(results.size, 1); // only one candidate existed; the other two executions stay unmatched
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
// Causal windows: a sandbox call can only have run after the request that
|
|
102
|
+
// proposed it started, and before the agent sent its result back.
|
|
103
|
+
const RM = { command: 'rm notes.md' };
|
|
104
|
+
const causal = (call_id, window_start, window_end = null) => ({ call_id, timestamp: window_start, args: RM, window_start, window_end });
|
|
105
|
+
|
|
106
|
+
test('causal window: a retried proposal cannot claim a run from before it existed, and vice versa', () => {
|
|
107
|
+
// call_a proposed at 0, result sent at 5 s; call_b (the retry) proposed at 10 s, still pending.
|
|
108
|
+
const candidates = [causal('call_a', 0, 5_000), causal('call_b', 10_000)];
|
|
109
|
+
assert.deepEqual(matchExecutionToCall({ command: '/bin/bash -lc rm notes.md', timestamp: 12_000, margin: 250 }, candidates),
|
|
110
|
+
{ call_id: 'call_b', score: 1, basis: 'window', window_start: 10_000, window_end: null });
|
|
111
|
+
assert.equal(matchExecutionToCall({ command: 'rm notes.md', timestamp: 3_000, margin: 250 }, candidates).call_id, 'call_a');
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test('causal window: a slow command still matches 45 s after its proposal (the old +-30 s window missed it)', () => {
|
|
115
|
+
const exec = { command: 'rm notes.md', timestamp: 45_000, margin: 250 };
|
|
116
|
+
assert.equal(matchExecutionToCall(exec, [causal('call_a', 0)])?.call_id, 'call_a');
|
|
117
|
+
assert.equal(matchExecutionToCall(exec, [{ call_id: 'call_a', timestamp: 0, args: RM }]), null, 'legacy candidates keep the old window');
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test('causal window: a run that starts after the result was sent is not that call', () => {
|
|
121
|
+
assert.equal(matchExecutionToCall({ command: 'rm notes.md', timestamp: 8_000, margin: 250 }, [causal('call_a', 0, 5_000)]), null);
|
|
122
|
+
// ...but clock margin still absorbs measured skew at the edges.
|
|
123
|
+
assert.equal(matchExecutionToCall({ command: 'rm notes.md', timestamp: 5_200, margin: 250 }, [causal('call_a', 0, 5_000)])?.call_id, 'call_a');
|
|
124
|
+
assert.equal(matchExecutionToCall({ command: 'rm notes.md', timestamp: -200, margin: 250 }, [causal('call_a', 0, 5_000)])?.call_id, 'call_a');
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test('causal window: a call that never reported back stops competing after the open-window cap', () => {
|
|
128
|
+
assert.equal(matchExecutionToCall({ command: 'rm notes.md', timestamp: 11 * 60_000, margin: 250 }, [causal('call_a', 0)]), null);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test('causal window: text is still required inside the window (agent helpers run while a call is pending)', () => {
|
|
132
|
+
assert.equal(matchExecutionToCall({ command: 'git status --short', timestamp: 1_000, margin: 250 }, [causal('call_a', 0)]), null);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
test('causal window: two pending look-alikes resolve to the most recently opened one', () => {
|
|
136
|
+
const candidates = [causal('call_old', 0), causal('call_new', 20_000)];
|
|
137
|
+
assert.equal(matchExecutionToCall({ command: 'rm notes.md', timestamp: 21_000, margin: 250 }, candidates).call_id, 'call_new');
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test('causal window: a window match carries the bounds that justified it', () => {
|
|
141
|
+
assert.deepEqual(matchExecutionToCall({ command: 'rm notes.md', timestamp: 3_000, margin: 250 }, [causal('call_a', 0, 5_000)]),
|
|
142
|
+
{ call_id: 'call_a', score: 1, basis: 'window', window_start: 0, window_end: 5_000 });
|
|
143
|
+
// A legacy text match carries none.
|
|
144
|
+
assert.deepEqual(matchExecutionToCall({ command: 'rm notes.md', timestamp: 3_000 }, [{ call_id: 'call_a', timestamp: 0, args: RM }]),
|
|
145
|
+
{ call_id: 'call_a', score: 1, basis: 'text' });
|
|
146
|
+
});
|