raytrace-cli 0.11.1__py3-none-win_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raytrace/__init__.py +1 -0
- raytrace/__main__.py +33 -0
- raytrace/app/LICENSE +21 -0
- raytrace/app/README.md +599 -0
- raytrace/app/bin/raytrace.mjs +297 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-BwCSEQnW.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-CW0RaeGs.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-B9HHJjqV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-Dr3UlScf.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-Aqo67rzb.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-BmdmIIQ2.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-CUSSCpQX.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-DvUfVpUG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BjWJ59Pq.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BtiwyxMk.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-BJfUCQsA.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-DR84L5F-.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-4vooXBpG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-DRtaDpgU.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-BLI3TTWz.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-CJIcxD6K.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-CTOM6hUh.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-fLZuRloM.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-9HEixskS.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-V-xxqcpd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-BgSNZQsw.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-DWFSQ4vo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-D38SheWl.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-DmB0ttJJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-D2EvbN8M.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-iLQfcSjf.woff +0 -0
- raytrace/app/dist/dashboard/assets/index-DepcI8Fi.js +38 -0
- raytrace/app/dist/dashboard/assets/index-H97VLD5l.css +1 -0
- raytrace/app/dist/dashboard/favicon.svg +6 -0
- raytrace/app/dist/dashboard/index.html +15 -0
- raytrace/app/package.json +48 -0
- raytrace/app/proxy/agent-intent-schema.test.mjs +142 -0
- raytrace/app/proxy/call-assessment.mjs +385 -0
- raytrace/app/proxy/call-assessment.test.mjs +167 -0
- raytrace/app/proxy/call-category.mjs +461 -0
- raytrace/app/proxy/capture-health.mjs +30 -0
- raytrace/app/proxy/capture-health.test.mjs +44 -0
- raytrace/app/proxy/capture.mjs +38 -0
- raytrace/app/proxy/capture.test.mjs +49 -0
- raytrace/app/proxy/claude-code-adapter.mjs +301 -0
- raytrace/app/proxy/claude-code-hook.mjs +22 -0
- raytrace/app/proxy/codex-config.mjs +36 -0
- raytrace/app/proxy/codex.mjs +14 -0
- raytrace/app/proxy/evidence-schema.test.mjs +240 -0
- raytrace/app/proxy/evidence-store.mjs +993 -0
- raytrace/app/proxy/evidence-store.test.mjs +301 -0
- raytrace/app/proxy/exec-narrative.mjs +625 -0
- raytrace/app/proxy/exec-narrative.test.mjs +262 -0
- raytrace/app/proxy/execution-correlation.mjs +185 -0
- raytrace/app/proxy/execution-correlation.test.mjs +146 -0
- raytrace/app/proxy/execution-runner.mjs +89 -0
- raytrace/app/proxy/execution-runner.test.mjs +38 -0
- raytrace/app/proxy/experiment-engine.mjs +195 -0
- raytrace/app/proxy/experiment-engine.test.mjs +93 -0
- raytrace/app/proxy/explanations.mjs +36 -0
- raytrace/app/proxy/explanations.test.mjs +20 -0
- raytrace/app/proxy/failure-signature.mjs +56 -0
- raytrace/app/proxy/gvisor-forwarder.mjs +493 -0
- raytrace/app/proxy/gvisor-forwarder.test.mjs +433 -0
- raytrace/app/proxy/init-evidence-db.mjs +227 -0
- raytrace/app/proxy/local-sandboxes.mjs +94 -0
- raytrace/app/proxy/local-token.mjs +30 -0
- raytrace/app/proxy/playground-recovery.test.mjs +130 -0
- raytrace/app/proxy/playground.mjs +439 -0
- raytrace/app/proxy/prompt-context.mjs +278 -0
- raytrace/app/proxy/prompt-traces.mjs +40 -0
- raytrace/app/proxy/prompt-traces.test.mjs +32 -0
- raytrace/app/proxy/providers.mjs +65 -0
- raytrace/app/proxy/providers.test.mjs +76 -0
- raytrace/app/proxy/proxy.integration.test.mjs +135 -0
- raytrace/app/proxy/raytace-proxy.mjs +941 -0
- raytrace/app/proxy/request-metrics.mjs +13 -0
- raytrace/app/proxy/request-metrics.test.mjs +19 -0
- raytrace/app/proxy/request-policy.mjs +45 -0
- raytrace/app/proxy/request-policy.test.mjs +47 -0
- raytrace/app/proxy/run-recap.mjs +139 -0
- raytrace/app/proxy/runtime-attribution.mjs +110 -0
- raytrace/app/proxy/sandbox-bridge.mjs +49 -0
- raytrace/app/proxy/schema/EVIDENCE_DATABASE.md +276 -0
- raytrace/app/proxy/schema/agent_intent.sql +101 -0
- raytrace/app/proxy/schema/auxiliary.sql +90 -0
- raytrace/app/proxy/schema/capture_health.sql +14 -0
- raytrace/app/proxy/schema/classification.sql +89 -0
- raytrace/app/proxy/schema/file_context.sql +53 -0
- raytrace/app/proxy/schema/runtime_evidence.sql +201 -0
- raytrace/app/proxy/schema/search.sql +66 -0
- raytrace/app/proxy/schema/shared_storage.sql +21 -0
- raytrace/app/proxy/search.mjs +418 -0
- raytrace/app/proxy/search.test.mjs +161 -0
- raytrace/app/proxy/smoke.test.mjs +93 -0
- raytrace/app/proxy/step-attribution.mjs +380 -0
- raytrace/app/proxy/step-lab.mjs +78 -0
- raytrace/app/proxy/step-lab.test.mjs +38 -0
- raytrace/app/proxy/summaries.mjs +151 -0
- raytrace/app/proxy/summaries.test.mjs +69 -0
- raytrace/app/proxy/tool-metadata.mjs +111 -0
- raytrace/app/proxy/tool-metadata.test.mjs +38 -0
- raytrace/app/proxy/trace-view.mjs +324 -0
- raytrace/app/proxy/trace-view.test.mjs +92 -0
- raytrace/app/proxy/typesafe.mjs +63 -0
- raytrace/app/proxy/typesafe.test.mjs +39 -0
- raytrace/app/src/agents.mjs +7 -0
- raytrace/app/src/api.mjs +57 -0
- raytrace/app/src/auth.mjs +205 -0
- raytrace/app/src/beta.mjs +96 -0
- raytrace/app/src/claude-code.mjs +93 -0
- raytrace/app/src/commands.mjs +285 -0
- raytrace/app/src/config.mjs +49 -0
- raytrace/app/src/export.mjs +200 -0
- raytrace/app/src/lima.mjs +176 -0
- raytrace/app/src/local-sandbox.mjs +358 -0
- raytrace/app/src/mcp.mjs +688 -0
- raytrace/app/src/paths.mjs +33 -0
- raytrace/app/src/prompt.mjs +42 -0
- raytrace/app/src/proxy-process.mjs +143 -0
- raytrace/app/src/sandbox.mjs +306 -0
- raytrace/app/src/share.mjs +86 -0
- raytrace/app/src/terminal.mjs +68 -0
- raytrace/app/src/upload.mjs +198 -0
- raytrace/app/worker/sandbox/Dockerfile +8 -0
- raytrace/app/worker/sandbox/SYSCALL_CATALOG.md +197 -0
- raytrace/app/worker/sandbox/claude-hook.mjs +364 -0
- raytrace/app/worker/sandbox/collector.py +405 -0
- raytrace/app/worker/sandbox/dns_forwarder.py +255 -0
- raytrace/app/worker/sandbox/engine.py +940 -0
- raytrace/app/worker/sandbox/file_scan.py +491 -0
- raytrace/app/worker/sandbox/lima.yaml +29 -0
- raytrace/app/worker/sandbox/network.py +90 -0
- raytrace/app/worker/sandbox/projects.py +702 -0
- raytrace/app/worker/sandbox/setup.py +159 -0
- raytrace/app/worker/sandbox/syscall_catalog.sql +985 -0
- raytrace/app/worker/sandbox/transcript-delivery.test.mjs +153 -0
- raytrace/app/worker/sandbox/viewer.py +195 -0
- raytrace/node/LICENSE +2946 -0
- raytrace/node/node.exe +4 -0
- raytrace_cli-0.11.1.dist-info/LICENSE +21 -0
- raytrace_cli-0.11.1.dist-info/METADATA +42 -0
- raytrace_cli-0.11.1.dist-info/RECORD +165 -0
- raytrace_cli-0.11.1.dist-info/WHEEL +4 -0
- raytrace_cli-0.11.1.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Splunk-style search over the derived `search_events` index
|
|
3
|
+
* (proxy/schema/search.sql, filled by evidence-store.mjs).
|
|
4
|
+
*
|
|
5
|
+
* Query shape: free text + `field:value` filters + one optional pipe.
|
|
6
|
+
* outcome:failed tool:Bash "permission denied"
|
|
7
|
+
* reported:ok observed:failed (agent said ok, sandbox saw fail)
|
|
8
|
+
* exit:127 | stats count by command
|
|
9
|
+
* failed | timechart count by model span=1d
|
|
10
|
+
*
|
|
11
|
+
* Fields: outcome (failed|ok|unknown), reported, observed, tool, model,
|
|
12
|
+
* category, kind (call|turn|proxy_error), provider, exit / exit:>=1,
|
|
13
|
+
* http / http:>=400, file:<substring>, signature:<prefix>, session,
|
|
14
|
+
* after:YYYY-MM-DD, before:YYYY-MM-DD. Bare `failed` means outcome:failed.
|
|
15
|
+
* Prefix any filter or word with `-` to negate. Trailing `*` = prefix.
|
|
16
|
+
*
|
|
17
|
+
* Outcomes never upgrade missing evidence: no result and no observed
|
|
18
|
+
* process is `unknown`, matching processOutcome/toolResultStatus.
|
|
19
|
+
*/
|
|
20
|
+
import { toolResultStatus } from './tool-metadata.mjs';
|
|
21
|
+
|
|
22
|
+
// ------------------------------------------------------------ outcomes
|
|
23
|
+
|
|
24
|
+
/** toolResultStatus label -> index outcome. A returned result with no
|
|
25
|
+
* error marker is the agent being told it worked: reported ok. */
|
|
26
|
+
export function reportedOutcome(resultItem) {
|
|
27
|
+
if (resultItem == null) return 'unknown';
|
|
28
|
+
const status = toolResultStatus(resultItem);
|
|
29
|
+
if (status === 'Failed' || status === 'Cancelled') return 'failed';
|
|
30
|
+
if (status === 'Successful' || status === 'Result received') return 'ok';
|
|
31
|
+
return 'unknown'; // Still running / No output / Output omitted
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** processOutcome status (evidence-store) -> index outcome. */
|
|
35
|
+
export function observedOutcome(processStatus) {
|
|
36
|
+
if (processStatus === 'failed') return 'failed';
|
|
37
|
+
if (processStatus === 'completed') return 'ok';
|
|
38
|
+
return 'unknown';
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Failed wins; ok needs at least one witness and no failure. */
|
|
42
|
+
export function combineOutcomes(reported, observed) {
|
|
43
|
+
if (reported === 'failed' || observed === 'failed') return 'failed';
|
|
44
|
+
if (reported === 'ok' || observed === 'ok') return 'ok';
|
|
45
|
+
return 'unknown';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// ------------------------------------------------------------ text
|
|
49
|
+
|
|
50
|
+
const MAX_INDEX_TEXT = 64_000;
|
|
51
|
+
|
|
52
|
+
/** Head+tail cap so one giant tool result cannot bloat the index. */
|
|
53
|
+
export function clipText(text, max = MAX_INDEX_TEXT) {
|
|
54
|
+
const value = typeof text === 'string' ? text : '';
|
|
55
|
+
if (value.length <= max) return value;
|
|
56
|
+
const head = Math.floor(max * 0.7);
|
|
57
|
+
return `${value.slice(0, head)}\n…[clipped]…\n${value.slice(-(max - head))}`;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Flatten a stored payload (any JSON shape) into searchable text. */
|
|
61
|
+
export function payloadToText(value, depth = 0) {
|
|
62
|
+
if (value == null || depth > 6) return '';
|
|
63
|
+
if (typeof value === 'string') {
|
|
64
|
+
// Arguments/results are often JSON strings: index their inner text too.
|
|
65
|
+
if ((value.startsWith('{') || value.startsWith('[')) && value.length > 1) {
|
|
66
|
+
try { const inner = payloadToText(JSON.parse(value), depth + 1); if (inner) return inner; } catch { /* keep raw */ }
|
|
67
|
+
}
|
|
68
|
+
return value;
|
|
69
|
+
}
|
|
70
|
+
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
|
|
71
|
+
if (Array.isArray(value)) return value.map((item) => payloadToText(item, depth + 1)).filter(Boolean).join('\n');
|
|
72
|
+
if (typeof value === 'object') {
|
|
73
|
+
const preferred = ['text', 'content', 'output', 'command', 'cmd', 'script', 'message', 'error']
|
|
74
|
+
.map((key) => payloadToText(value[key], depth + 1)).filter(Boolean);
|
|
75
|
+
if (preferred.length) return preferred.join('\n');
|
|
76
|
+
return Object.values(value).map((item) => payloadToText(item, depth + 1)).filter(Boolean).join('\n');
|
|
77
|
+
}
|
|
78
|
+
return '';
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const PATH_KEYS = new Set(['path', 'file_path', 'filepath', 'file', 'filename', 'cwd', 'workdir',
|
|
82
|
+
'directory', 'dir', 'target', 'source', 'destination', 'old_path', 'new_path', 'notebook_path']);
|
|
83
|
+
|
|
84
|
+
/** File paths named in tool arguments (for file: search). */
|
|
85
|
+
export function extractFilePaths(value, out = new Set(), depth = 0) {
|
|
86
|
+
if (value == null || depth > 6 || out.size > 40) return out;
|
|
87
|
+
if (typeof value === 'string' && (value.startsWith('{') || value.startsWith('['))) {
|
|
88
|
+
try { return extractFilePaths(JSON.parse(value), out, depth + 1); } catch { return out; }
|
|
89
|
+
}
|
|
90
|
+
if (Array.isArray(value)) { for (const item of value) extractFilePaths(item, out, depth + 1); return out; }
|
|
91
|
+
if (typeof value === 'object') {
|
|
92
|
+
for (const [key, item] of Object.entries(value)) {
|
|
93
|
+
if (typeof item === 'string' && PATH_KEYS.has(key.toLowerCase()) && item.length <= 1024) out.add(item);
|
|
94
|
+
else extractFilePaths(item, out, depth + 1);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return out;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Path-like tokens in free text (a Bash command naming src/parser.js),
|
|
101
|
+
* so file: works for shell tools too, not only file-tool arguments. */
|
|
102
|
+
export function pathsFromText(text, out = new Set()) {
|
|
103
|
+
for (const match of String(text ?? '').matchAll(/(?<![\w/.-])((?:\.?\.?\/)?(?:[\w@+~-]+\/)+[\w@+~-]+(?:\.[\w@+~-]+)+)\b/gi)) {
|
|
104
|
+
if (out.size > 40) break;
|
|
105
|
+
out.add(match[1]);
|
|
106
|
+
}
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The command a call proposed, for display + `stats by command`.
|
|
111
|
+
* Deliberately small: full rendering lives in trace-view. */
|
|
112
|
+
export function commandTextOf(args) {
|
|
113
|
+
let value = args;
|
|
114
|
+
if (typeof value === 'string') { try { value = JSON.parse(value); } catch { return value.slice(0, 400); } }
|
|
115
|
+
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
116
|
+
const cmd = value.cmd ?? value.command ?? value.script;
|
|
117
|
+
if (Array.isArray(cmd)) return cmd.map(String).join(' ').slice(0, 400);
|
|
118
|
+
if (typeof cmd === 'string' && cmd.trim()) return cmd.trim().slice(0, 400);
|
|
119
|
+
const file = value.file_path ?? value.path ?? value.file;
|
|
120
|
+
if (typeof file === 'string') return file.slice(0, 400);
|
|
121
|
+
}
|
|
122
|
+
if (Array.isArray(value)) return value.map(String).join(' ').slice(0, 400);
|
|
123
|
+
return '';
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// ------------------------------------------------------------ query parsing
|
|
127
|
+
|
|
128
|
+
const FIELD_ALIASES = { status: 'outcome', exit: 'exit_code', http: 'http_status', sig: 'signature' };
|
|
129
|
+
const KNOWN_FIELDS = new Set(['outcome', 'reported', 'observed', 'tool', 'model', 'category', 'kind',
|
|
130
|
+
'provider', 'agent', 'exit_code', 'http_status', 'signature', 'file', 'session', 'after', 'before']);
|
|
131
|
+
|
|
132
|
+
function splitPipe(input) {
|
|
133
|
+
let quoted = false;
|
|
134
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
135
|
+
if (input[i] === '"') quoted = !quoted;
|
|
136
|
+
else if (input[i] === '|' && !quoted) return [input.slice(0, i), input.slice(i + 1)];
|
|
137
|
+
}
|
|
138
|
+
return [input, null];
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function tokenize(input) {
|
|
142
|
+
const tokens = [];
|
|
143
|
+
const pattern = /(-)?"([^"]*)"|(-)?(\S+)/g;
|
|
144
|
+
let match;
|
|
145
|
+
while ((match = pattern.exec(input)) !== null) {
|
|
146
|
+
if (match[2] !== undefined) tokens.push({ value: match[2], phrase: true, negated: Boolean(match[1]) });
|
|
147
|
+
else tokens.push({ value: match[4], phrase: false, negated: Boolean(match[3]) });
|
|
148
|
+
}
|
|
149
|
+
return tokens;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function parsePipe(text, errors) {
|
|
153
|
+
if (text == null) return null;
|
|
154
|
+
const trimmed = text.trim();
|
|
155
|
+
let m = trimmed.match(/^stats\s+(count|avg\s*\(\s*duration\s*\))(?:\s+by\s+([a-z_]+))?$/i);
|
|
156
|
+
if (m) return { type: 'stats', agg: /^avg/i.test(m[1]) ? 'avg_duration' : 'count', by: m[2]?.toLowerCase() ?? null };
|
|
157
|
+
m = trimmed.match(/^timechart\s+count(?:\s+by\s+([a-z_]+))?(?:\s+span=(\d+)([mhd]))?$/i);
|
|
158
|
+
if (m) {
|
|
159
|
+
const unit = { m: 60_000, h: 3_600_000, d: 86_400_000 }[(m[3] ?? 'd').toLowerCase()];
|
|
160
|
+
return { type: 'timechart', by: m[1]?.toLowerCase() ?? null, spanMs: unit * Number(m[2] ?? 1) };
|
|
161
|
+
}
|
|
162
|
+
errors.push(`Unsupported pipe: | ${trimmed.slice(0, 60)}`);
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export function parseSearchQuery(input = '') {
|
|
167
|
+
const errors = [];
|
|
168
|
+
const [head, pipeText] = splitPipe(String(input ?? ''));
|
|
169
|
+
const text = [];
|
|
170
|
+
const filters = [];
|
|
171
|
+
for (const token of tokenize(head)) {
|
|
172
|
+
if (!token.phrase) {
|
|
173
|
+
const bare = token.value.toLowerCase();
|
|
174
|
+
if (!token.negated && (bare === 'failed' || bare === 'failure')) { filters.push({ field: 'outcome', value: 'failed', negated: false }); continue; }
|
|
175
|
+
const fieldMatch = token.value.match(/^([a-z_]+):(.*)$/i);
|
|
176
|
+
if (fieldMatch) {
|
|
177
|
+
const field = FIELD_ALIASES[fieldMatch[1].toLowerCase()] ?? fieldMatch[1].toLowerCase();
|
|
178
|
+
if (KNOWN_FIELDS.has(field)) { filters.push({ field, value: fieldMatch[2], negated: token.negated }); continue; }
|
|
179
|
+
errors.push(`Unknown field: ${fieldMatch[1]}`);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
const prefix = !token.phrase && token.value.endsWith('*') && token.value.length > 1;
|
|
183
|
+
text.push({ value: prefix ? token.value.slice(0, -1) : token.value, phrase: token.phrase, prefix, negated: token.negated });
|
|
184
|
+
}
|
|
185
|
+
return { text, filters, pipe: parsePipe(pipeText, errors), errors };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// ------------------------------------------------------------ SQL building
|
|
189
|
+
|
|
190
|
+
const EQUALITY = { outcome: 'outcome', reported: 'outcome_reported', observed: 'outcome_observed',
|
|
191
|
+
tool: 'tool_name', model: 'model', category: 'category', kind: 'kind', provider: 'provider', agent: 'agent' };
|
|
192
|
+
const NUMERIC = { exit_code: 'exit_code', http_status: 'http_status' };
|
|
193
|
+
|
|
194
|
+
const likeEscape = (value) => value.replace(/[%_\\]/g, '\\$&');
|
|
195
|
+
|
|
196
|
+
function dateBound(value, upper) {
|
|
197
|
+
const ms = /^\d+$/.test(value) ? Number(value) : Date.parse(value);
|
|
198
|
+
if (!Number.isFinite(ms)) return null;
|
|
199
|
+
// A bare date for `before:` means the end of that day.
|
|
200
|
+
return upper && /^\d{4}-\d{2}-\d{2}$/.test(value) ? ms + 86_399_999 : ms;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** WHERE clause + params for the parsed filters and time range.
|
|
204
|
+
* `exclude` drops one field's filters (for facet counts). */
|
|
205
|
+
function buildWhere(parsed, { fromMs = null, toMs = null, exclude = null } = {}) {
|
|
206
|
+
const clauses = [];
|
|
207
|
+
const params = [];
|
|
208
|
+
let from = fromMs;
|
|
209
|
+
let to = toMs;
|
|
210
|
+
for (const filter of parsed.filters) {
|
|
211
|
+
if (filter.field === exclude) continue;
|
|
212
|
+
const not = filter.negated ? 'NOT ' : '';
|
|
213
|
+
if (filter.field === 'after') { const b = dateBound(filter.value, false); if (b != null) from = from == null ? b : Math.max(from, b); continue; }
|
|
214
|
+
if (filter.field === 'before') { const b = dateBound(filter.value, true); if (b != null) to = to == null ? b : Math.min(to, b); continue; }
|
|
215
|
+
if (EQUALITY[filter.field]) {
|
|
216
|
+
const column = EQUALITY[filter.field];
|
|
217
|
+
if (filter.value.endsWith('*')) { clauses.push(`${not}(${column} LIKE ? ESCAPE '\\')`); params.push(`${likeEscape(filter.value.slice(0, -1))}%`); }
|
|
218
|
+
else { clauses.push(`${not}(LOWER(COALESCE(${column},'')) = LOWER(?))`); params.push(filter.value); }
|
|
219
|
+
} else if (NUMERIC[filter.field]) {
|
|
220
|
+
const m = filter.value.match(/^(>=|<=|>|<)?(-?\d+)$/);
|
|
221
|
+
if (m) { clauses.push(`${not}(${NUMERIC[filter.field]} ${m[1] ?? '='} ?)`); params.push(Number(m[2])); }
|
|
222
|
+
} else if (filter.field === 'file') {
|
|
223
|
+
clauses.push(`${not}(file_paths LIKE ? ESCAPE '\\')`); params.push(`%${likeEscape(filter.value.replace(/\*/g, ''))}%`);
|
|
224
|
+
} else if (filter.field === 'signature') {
|
|
225
|
+
clauses.push(`${not}(failure_signature LIKE ? ESCAPE '\\')`); params.push(`${likeEscape(filter.value.replace(/\*/g, ''))}%`);
|
|
226
|
+
} else if (filter.field === 'session') {
|
|
227
|
+
clauses.push(`${not}(session_id = ? OR session_external = ?)`); params.push(filter.value, filter.value);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
if (from != null) { clauses.push('ts_ms >= ?'); params.push(from); }
|
|
231
|
+
if (to != null) { clauses.push('ts_ms <= ?'); params.push(to); }
|
|
232
|
+
return { where: clauses.length ? clauses.join(' AND ') : '1=1', params };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const ftsTerm = (term) => `"${term.value.replace(/"/g, '""')}"${term.prefix ? '*' : ''}`;
|
|
236
|
+
|
|
237
|
+
function ftsClauses(parsed) {
|
|
238
|
+
const positives = parsed.text.filter((term) => !term.negated);
|
|
239
|
+
const negatives = parsed.text.filter((term) => term.negated);
|
|
240
|
+
return {
|
|
241
|
+
match: positives.length ? positives.map(ftsTerm).join(' AND ') : null,
|
|
242
|
+
excludeMatch: negatives.length ? negatives.map(ftsTerm).join(' OR ') : null,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function baseFrom(match) {
|
|
247
|
+
return match
|
|
248
|
+
? 'FROM search_events se JOIN search_fts ON search_fts.rowid = se.rowid AND search_fts MATCH ?'
|
|
249
|
+
: 'FROM search_events se';
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Full WHERE across the events alias, FTS match first in params. */
|
|
253
|
+
function scopedWhere(parsed, range, { exclude = null } = {}) {
|
|
254
|
+
const { match, excludeMatch } = ftsClauses(parsed);
|
|
255
|
+
const built = buildWhere(parsed, { ...range, exclude });
|
|
256
|
+
// Free-text exclusions stay out of facet-exclusion logic: always applied.
|
|
257
|
+
const extra = excludeMatch ? ' AND se.rowid NOT IN (SELECT rowid FROM search_fts WHERE search_fts MATCH ?)' : '';
|
|
258
|
+
const params = [...(match ? [match] : []), ...built.params.map((p) => p), ...(excludeMatch ? [excludeMatch] : [])];
|
|
259
|
+
// buildWhere columns are unqualified; qualify for the joined query.
|
|
260
|
+
const where = built.where.replace(/(^|[^.\w])(outcome_reported|outcome_observed|outcome|tool_name|model|category|kind|provider|agent|exit_code|http_status|file_paths|failure_signature|session_id|session_external|ts_ms)\b/g, '$1se.$2');
|
|
261
|
+
return { from: baseFrom(match), where: `${where}${extra}`, params, hasText: Boolean(match) };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const GROUP_EXPR = {
|
|
265
|
+
tool: 'se.tool_name', model: 'se.model', category: 'se.category', outcome: 'se.outcome',
|
|
266
|
+
reported: 'se.outcome_reported', observed: 'se.outcome_observed', kind: 'se.kind',
|
|
267
|
+
provider: 'se.provider', exit: 'CAST(se.exit_code AS TEXT)', exit_code: 'CAST(se.exit_code AS TEXT)',
|
|
268
|
+
http: 'CAST(se.http_status AS TEXT)', signature: 'se.failure_signature',
|
|
269
|
+
command: "substr(se.command_text, 1, instr(se.command_text || ' ', ' ') - 1)",
|
|
270
|
+
day: "date(se.ts_ms / 1000, 'unixepoch')",
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
function runStats(db, parsed, range) {
|
|
274
|
+
const { pipe } = parsed;
|
|
275
|
+
const { from, where, params } = scopedWhere(parsed, range);
|
|
276
|
+
if (pipe.type === 'stats') {
|
|
277
|
+
const expr = pipe.by ? GROUP_EXPR[pipe.by] : null;
|
|
278
|
+
if (pipe.by && !expr) return { mode: 'stats', rows: [], error: `Cannot group by ${pipe.by}` };
|
|
279
|
+
const agg = pipe.agg === 'avg_duration' ? 'ROUND(AVG(se.duration_ms))' : 'COUNT(*)';
|
|
280
|
+
const sql = expr
|
|
281
|
+
? `SELECT ${expr} AS k, ${agg} AS v, COUNT(*) AS n ${from} WHERE ${where} GROUP BY k ORDER BY ${pipe.agg === 'avg_duration' ? 'v DESC' : 'n DESC'} LIMIT 50`
|
|
282
|
+
: `SELECT NULL AS k, ${agg} AS v, COUNT(*) AS n ${from} WHERE ${where}`;
|
|
283
|
+
return { mode: 'stats', agg: pipe.agg, by: pipe.by, rows: db.prepare(sql).all(...params).map((r) => ({ key: r.k ?? (pipe.by ? '(none)' : '(all)'), value: r.v ?? 0, count: r.n })) };
|
|
284
|
+
}
|
|
285
|
+
// timechart
|
|
286
|
+
const expr = pipe.by ? GROUP_EXPR[pipe.by] : null;
|
|
287
|
+
if (pipe.by && !expr) return { mode: 'timechart', rows: [], error: `Cannot group by ${pipe.by}` };
|
|
288
|
+
const bucket = `(se.ts_ms / ${pipe.spanMs}) * ${pipe.spanMs}`;
|
|
289
|
+
const rows = db.prepare(`SELECT ${bucket} AS t, ${expr ? `${expr} AS k,` : 'NULL AS k,'} COUNT(*) AS n ${from} WHERE ${where} GROUP BY t${expr ? ', k' : ''} ORDER BY t`).all(...params);
|
|
290
|
+
const noneLabel = pipe.by ? '(none)' : '(all)';
|
|
291
|
+
const seriesTotals = new Map();
|
|
292
|
+
for (const row of rows) seriesTotals.set(row.k ?? noneLabel, (seriesTotals.get(row.k ?? noneLabel) ?? 0) + row.n);
|
|
293
|
+
const series = [...seriesTotals.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([key]) => key);
|
|
294
|
+
const byTime = new Map();
|
|
295
|
+
for (const row of rows) {
|
|
296
|
+
const key = row.k ?? noneLabel;
|
|
297
|
+
const label = series.includes(key) ? key : 'other';
|
|
298
|
+
if (!byTime.has(row.t)) byTime.set(row.t, { t: row.t });
|
|
299
|
+
byTime.get(row.t)[label] = (byTime.get(row.t)[label] ?? 0) + row.n;
|
|
300
|
+
}
|
|
301
|
+
return { mode: 'timechart', by: pipe.by, spanMs: pipe.spanMs, series: [...series, ...(seriesTotals.size > 8 ? ['other'] : [])], rows: [...byTime.values()] };
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function facet(db, parsed, range, field, column, limit) {
|
|
305
|
+
const scoped = scopedWhere(parsed, range, { exclude: field });
|
|
306
|
+
return db.prepare(`SELECT ${column} AS k, COUNT(*) AS n ${scoped.from} WHERE ${scoped.where} GROUP BY k ORDER BY n DESC LIMIT ${limit}`)
|
|
307
|
+
.all(...scoped.params).filter((row) => row.k !== null && row.k !== '').map((row) => ({ value: String(row.k), count: row.n }));
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** Materialize the matching rowids once (temp table, per connection).
|
|
311
|
+
* Summary, histogram and facets then aggregate only the matched set
|
|
312
|
+
* instead of re-evaluating the filters (FTS MATCH, LIKE) once per
|
|
313
|
+
* aggregate. Safe because runSearch is fully synchronous: no other
|
|
314
|
+
* search can interleave between the refill and the reads. */
|
|
315
|
+
function materialize(db, parsed, range) {
|
|
316
|
+
db.exec('CREATE TEMP TABLE IF NOT EXISTS _raytace_m (rid INTEGER PRIMARY KEY, ts_ms INTEGER, rank REAL)');
|
|
317
|
+
const scoped = scopedWhere(parsed, range);
|
|
318
|
+
db.exec('DELETE FROM _raytace_m');
|
|
319
|
+
db.prepare(`INSERT INTO _raytace_m (rid, ts_ms, rank)
|
|
320
|
+
SELECT se.rowid, se.ts_ms, ${scoped.hasText ? 'bm25(search_fts)' : '0'} ${scoped.from} WHERE ${scoped.where}`)
|
|
321
|
+
.run(...scoped.params);
|
|
322
|
+
return scoped;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function facetFromMatches(db, column, limit) {
|
|
326
|
+
return db.prepare(`SELECT ${column} AS k, COUNT(*) AS n FROM _raytace_m m JOIN search_events se ON se.rowid = m.rid
|
|
327
|
+
GROUP BY k ORDER BY n DESC LIMIT ${limit}`)
|
|
328
|
+
.all().filter((row) => row.k !== null && row.k !== '').map((row) => ({ value: String(row.k), count: row.n }));
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Run one search against the evidence db (node:sqlite handle).
|
|
333
|
+
* Events mode returns hits + facets + histogram + summary; a pipe
|
|
334
|
+
* returns its aggregation instead.
|
|
335
|
+
*/
|
|
336
|
+
export function runSearch(db, { q = '', fromMs = null, toMs = null, limit = 50, offset = 0 } = {}) {
|
|
337
|
+
const parsed = parseSearchQuery(q);
|
|
338
|
+
const range = { fromMs, toMs };
|
|
339
|
+
if (parsed.pipe) return { query: q, errors: parsed.errors, ...runStats(db, parsed, range) };
|
|
340
|
+
|
|
341
|
+
const scoped = materialize(db, parsed, range);
|
|
342
|
+
const MATCHED = 'FROM _raytace_m m JOIN search_events se ON se.rowid = m.rid';
|
|
343
|
+
const boundedLimit = Math.min(Math.max(Number(limit) || 50, 1), 200);
|
|
344
|
+
const boundedOffset = Math.max(Number(offset) || 0, 0);
|
|
345
|
+
const total = db.prepare('SELECT COUNT(*) AS n FROM _raytace_m').get().n;
|
|
346
|
+
const snippetJoin = scoped.hasText ? 'JOIN search_fts ON search_fts.rowid = se.rowid AND search_fts MATCH ?' : '';
|
|
347
|
+
const snippet = scoped.hasText ? `snippet(search_fts, -1, '[', ']', '…', 18) AS snippet,` : '';
|
|
348
|
+
const order = scoped.hasText ? 'm.rank, m.ts_ms DESC' : 'm.ts_ms DESC';
|
|
349
|
+
const { match } = ftsClauses(parsed);
|
|
350
|
+
const hits = db.prepare(`SELECT ${snippet} se.* ${MATCHED} ${snippetJoin} ORDER BY ${order} LIMIT ? OFFSET ?`)
|
|
351
|
+
.all(...(match ? [match] : []), boundedLimit, boundedOffset)
|
|
352
|
+
.map((full) => {
|
|
353
|
+
// Full args/result text stays server-side; hits carry excerpts only.
|
|
354
|
+
const { prompt_text, args_text, result_text, ...row } = full;
|
|
355
|
+
void args_text; void result_text;
|
|
356
|
+
return {
|
|
357
|
+
...row,
|
|
358
|
+
snippet: row.snippet ?? null,
|
|
359
|
+
ts: new Date(row.ts_ms).toISOString(),
|
|
360
|
+
prompt_excerpt: (prompt_text ?? '').slice(0, 220) || null,
|
|
361
|
+
error_excerpt: (row.error_text ?? '').slice(0, 400) || null,
|
|
362
|
+
};
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
const summaryRow = db.prepare(`SELECT COUNT(*) AS total,
|
|
366
|
+
SUM(CASE WHEN se.outcome = 'failed' THEN 1 ELSE 0 END) AS failed,
|
|
367
|
+
SUM(CASE WHEN se.outcome_reported != 'unknown' AND se.outcome_observed != 'unknown'
|
|
368
|
+
AND se.outcome_reported != se.outcome_observed THEN 1 ELSE 0 END) AS mismatched,
|
|
369
|
+
SUM(CASE WHEN se.http_status >= 400 THEN 1 ELSE 0 END) AS http_errors
|
|
370
|
+
${MATCHED}`).get();
|
|
371
|
+
|
|
372
|
+
const span = (toMs ?? Date.now()) - (fromMs ?? (hits.at(-1)?.ts_ms ?? 0));
|
|
373
|
+
const bucketMs = span <= 2 * 86_400_000 ? 3_600_000 : span <= 120 * 86_400_000 ? 86_400_000 : 7 * 86_400_000;
|
|
374
|
+
const histogram = db.prepare(`SELECT (se.ts_ms / ${bucketMs}) * ${bucketMs} AS t, COUNT(*) AS total,
|
|
375
|
+
SUM(CASE WHEN se.outcome = 'failed' THEN 1 ELSE 0 END) AS failed
|
|
376
|
+
${MATCHED} GROUP BY t ORDER BY t`).all();
|
|
377
|
+
|
|
378
|
+
// Facets from the matched set, except a facet whose own field is
|
|
379
|
+
// filtered: that one re-runs without its filter so the other values
|
|
380
|
+
// stay visible (standard faceted-search behavior).
|
|
381
|
+
const filteredFields = new Set(parsed.filters.map((filter) => filter.field));
|
|
382
|
+
const facetFor = (field, column, limit) => (filteredFields.has(field)
|
|
383
|
+
? facet(db, parsed, range, field, column, limit)
|
|
384
|
+
: facetFromMatches(db, column, limit));
|
|
385
|
+
|
|
386
|
+
return {
|
|
387
|
+
mode: 'events', query: q, errors: parsed.errors, total,
|
|
388
|
+
limit: boundedLimit, offset: boundedOffset, hits,
|
|
389
|
+
summary: { total: summaryRow.total, failed: summaryRow.failed ?? 0, mismatched: summaryRow.mismatched ?? 0, http_errors: summaryRow.http_errors ?? 0 },
|
|
390
|
+
histogram: histogram.map((row) => ({ t: row.t, total: row.total, failed: row.failed ?? 0 })),
|
|
391
|
+
facets: {
|
|
392
|
+
outcome: facetFor('outcome', 'se.outcome', 5),
|
|
393
|
+
tool: facetFor('tool', 'se.tool_name', 15),
|
|
394
|
+
model: facetFor('model', 'se.model', 10),
|
|
395
|
+
category: facetFor('category', 'se.category', 15),
|
|
396
|
+
exit_code: facetFor('exit_code', 'se.exit_code', 10),
|
|
397
|
+
},
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/** Top failure clusters (the failures home). */
|
|
402
|
+
export function topFailureSignatures(db, { fromMs = null, toMs = null, limit = 20 } = {}) {
|
|
403
|
+
const clauses = ["outcome = 'failed'", 'failure_signature IS NOT NULL'];
|
|
404
|
+
const params = [];
|
|
405
|
+
if (fromMs != null) { clauses.push('ts_ms >= ?'); params.push(fromMs); }
|
|
406
|
+
if (toMs != null) { clauses.push('ts_ms <= ?'); params.push(toMs); }
|
|
407
|
+
const rows = db.prepare(`SELECT failure_signature AS signature, signature_label AS label,
|
|
408
|
+
COUNT(*) AS count, MIN(ts_ms) AS first_ms, MAX(ts_ms) AS last_ms,
|
|
409
|
+
GROUP_CONCAT(DISTINCT tool_name) AS tools, GROUP_CONCAT(DISTINCT exit_code) AS exit_codes,
|
|
410
|
+
(SELECT turn_id FROM search_events s2 WHERE s2.failure_signature = se.failure_signature ORDER BY s2.ts_ms DESC LIMIT 1) AS sample_turn,
|
|
411
|
+
(SELECT external_call_id FROM search_events s2 WHERE s2.failure_signature = se.failure_signature ORDER BY s2.ts_ms DESC LIMIT 1) AS sample_call
|
|
412
|
+
FROM search_events se WHERE ${clauses.join(' AND ')} GROUP BY failure_signature ORDER BY count DESC LIMIT ?`)
|
|
413
|
+
.all(...params, Math.min(Math.max(Number(limit) || 20, 1), 100));
|
|
414
|
+
return rows.map((row) => ({ ...row,
|
|
415
|
+
tools: (row.tools ?? '').split(',').filter(Boolean).slice(0, 4),
|
|
416
|
+
exit_codes: (row.exit_codes ?? '').split(',').filter(Boolean).slice(0, 4),
|
|
417
|
+
first: new Date(row.first_ms).toISOString(), last: new Date(row.last_ms).toISOString() }));
|
|
418
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { mkdtemp, rm } from 'node:fs/promises';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { openEvidenceStore } from './evidence-store.mjs';
|
|
7
|
+
import { parseSearchQuery, reportedOutcome, combineOutcomes, pathsFromText } from './search.mjs';
|
|
8
|
+
import { failureSignature } from './failure-signature.mjs';
|
|
9
|
+
|
|
10
|
+
const SESSION = 'rtp-' + 'f'.repeat(32);
|
|
11
|
+
const CONTAINER = 'c'.repeat(64);
|
|
12
|
+
const PROMPT = { role: 'user', type: 'message', content: 'In /workspace/demo, run npm test and fix notes.md.' };
|
|
13
|
+
const CALL = { type: 'function_call', call_id: 'call_1', name: 'exec_command', arguments: '{"cmd":"cd /workspace/demo && npm test"}' };
|
|
14
|
+
const EDIT = { type: 'tool_use', call_id: 'call_2', name: 'Edit', input: { file_path: '/workspace/demo/notes.md', old_string: 'a', new_string: 'b' } };
|
|
15
|
+
const RESULT = (call_id, output) => ({ type: 'function_call_output', call_id, output });
|
|
16
|
+
const HEADER = (code, body) => `Wall time: 0.4 seconds\nProcess exited with code ${code}\nOutput:\n${body}`;
|
|
17
|
+
|
|
18
|
+
let clock = 0;
|
|
19
|
+
const exchange = (id, input, output, { status = 200 } = {}) => ({
|
|
20
|
+
event_type: 'model.exchange', span_id: id, trace_id: 't', session_id: SESSION, session_started_at: new Date(0).toISOString(),
|
|
21
|
+
provider: 'openrouter', route: '/v1/responses', method: 'POST',
|
|
22
|
+
timestamp: new Date(1_790_000_000_000 + (clock += 1000)).toISOString(),
|
|
23
|
+
completed_at: new Date(1_790_000_000_000 + clock + 400).toISOString(), metrics: { cost: 0.001 },
|
|
24
|
+
request: { headers: {}, bytes: 10, sha256: 'req', payload: { model: 'qwen', tools: [], input } },
|
|
25
|
+
response: { status, headers: {}, bytes: 20, sha256: 'res', payload: { output } },
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
async function withStore(run) {
|
|
29
|
+
const dir = await mkdtemp(join(tmpdir(), 'raytace-search-'));
|
|
30
|
+
const file = join(dir, 'evidence.db');
|
|
31
|
+
const store = openEvidenceStore(file);
|
|
32
|
+
try { await run(store, file); } finally { store.close(); await rm(dir, { recursive: true, force: true }); }
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const FAILING = HEADER(1, "Error: Cannot find module './missing.js'");
|
|
36
|
+
|
|
37
|
+
function captureFailing(store, reported = FAILING) {
|
|
38
|
+
store.recordExchange(exchange('e1', [PROMPT], [CALL, EDIT]));
|
|
39
|
+
store.recordExchange(exchange('e2', [PROMPT, CALL, RESULT('call_1', reported), EDIT, { type: 'tool_result', call_id: 'call_2', content: 'edited' }],
|
|
40
|
+
[{ type: 'message', content: [{ type: 'output_text', text: 'Done.' }] }]));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
test('parser: fields, phrases, negation, pipes, unknown fields', () => {
|
|
44
|
+
const parsed = parseSearchQuery('outcome:failed tool:Bash "permission denied" -flaky exit:>=1 bogus:x');
|
|
45
|
+
assert.deepEqual(parsed.filters, [
|
|
46
|
+
{ field: 'outcome', value: 'failed', negated: false },
|
|
47
|
+
{ field: 'tool', value: 'Bash', negated: false },
|
|
48
|
+
{ field: 'exit_code', value: '>=1', negated: false },
|
|
49
|
+
]);
|
|
50
|
+
assert.equal(parsed.text.length, 3); // phrase, -flaky, bogus:x as text
|
|
51
|
+
assert.deepEqual(parsed.errors, ['Unknown field: bogus']);
|
|
52
|
+
assert.deepEqual(parseSearchQuery('failed | stats count by tool').pipe, { type: 'stats', agg: 'count', by: 'tool' });
|
|
53
|
+
assert.deepEqual(parseSearchQuery('x | timechart count by model span=1d').pipe, { type: 'timechart', by: 'model', spanMs: 86_400_000 });
|
|
54
|
+
assert.equal(parseSearchQuery('perm*').text[0].prefix, true);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('outcome truth table: missing evidence stays unknown', () => {
|
|
58
|
+
assert.equal(combineOutcomes('ok', 'failed'), 'failed');
|
|
59
|
+
assert.equal(combineOutcomes('failed', 'ok'), 'failed');
|
|
60
|
+
assert.equal(combineOutcomes('ok', 'unknown'), 'ok');
|
|
61
|
+
assert.equal(combineOutcomes('unknown', 'ok'), 'ok');
|
|
62
|
+
assert.equal(combineOutcomes('unknown', 'unknown'), 'unknown');
|
|
63
|
+
assert.equal(reportedOutcome(null), 'unknown');
|
|
64
|
+
assert.equal(reportedOutcome({ type: 'function_call_output', output: HEADER(2, 'boom') }), 'failed');
|
|
65
|
+
assert.equal(reportedOutcome({ type: 'function_call_output', output: HEADER(0, 'fine') }), 'ok');
|
|
66
|
+
assert.equal(reportedOutcome({ type: 'tool_result', is_error: true, content: 'no' }), 'failed');
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test('failure signatures cluster volatile parts, split distinct errors', () => {
|
|
70
|
+
const a = failureSignature({ errorText: 'Error: ENOENT open /Users/a/proj/src/index.js', exitCode: 1, toolName: 'Bash' });
|
|
71
|
+
const b = failureSignature({ errorText: 'Error: ENOENT open /home/b/other/src/index.js', exitCode: 1, toolName: 'Bash' });
|
|
72
|
+
const c = failureSignature({ errorText: 'TypeError: x is not a function', exitCode: 1, toolName: 'Bash' });
|
|
73
|
+
assert.equal(a.signature, b.signature);
|
|
74
|
+
assert.notEqual(a.signature, c.signature);
|
|
75
|
+
const headed = failureSignature({ errorText: FAILING, exitCode: 1, toolName: 'exec_command' });
|
|
76
|
+
assert.match(headed.label, /cannot find module/);
|
|
77
|
+
assert.equal(failureSignature({ errorText: '' }), null);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test('captured failure is indexed and searchable by outcome, text and file', () => withStore((store) => {
|
|
81
|
+
captureFailing(store);
|
|
82
|
+
const failed = store.search({ q: 'outcome:failed kind:call' });
|
|
83
|
+
assert.equal(failed.mode, 'events');
|
|
84
|
+
assert.equal(failed.total, 1);
|
|
85
|
+
const [hit] = failed.hits;
|
|
86
|
+
assert.equal(hit.tool_name, 'exec_command');
|
|
87
|
+
assert.equal(hit.outcome_reported, 'failed');
|
|
88
|
+
assert.equal(hit.outcome_observed, 'unknown');
|
|
89
|
+
assert.equal(hit.exit_code, 1);
|
|
90
|
+
assert.ok(hit.failure_signature);
|
|
91
|
+
assert.equal(hit.turn_id.length > 0, true);
|
|
92
|
+
assert.equal(hit.external_call_id, 'call_1');
|
|
93
|
+
|
|
94
|
+
assert.equal(store.search({ q: '"cannot find module"' }).total, 1);
|
|
95
|
+
assert.equal(store.search({ q: 'missing' }).total >= 1, true);
|
|
96
|
+
assert.equal(store.search({ q: 'missing -module' }).total, 0);
|
|
97
|
+
assert.equal(store.search({ q: 'tool:Edit' }).total, 1);
|
|
98
|
+
assert.equal(store.search({ q: 'file:notes.md' }).total, 1);
|
|
99
|
+
assert.deepEqual([...pathsFromText('node src/parser.js && cat ./a/b.test.ts; echo v1.2 done')], ['src/parser.js', './a/b.test.ts']);
|
|
100
|
+
assert.equal(store.search({ q: 'exit:1' }).total, 1);
|
|
101
|
+
assert.equal(store.search({ q: 'exit:>=2' }).total, 0);
|
|
102
|
+
assert.equal(store.search({ q: 'kind:turn outcome:failed' }).total, 1, 'turn inherits its failed call');
|
|
103
|
+
assert.equal(failed.summary.failed, 1);
|
|
104
|
+
assert.ok(failed.facets.tool.some((f) => f.value === 'exec_command'));
|
|
105
|
+
assert.ok(failed.histogram.length >= 1);
|
|
106
|
+
}));
|
|
107
|
+
|
|
108
|
+
test('agent reported ok but sandbox saw failure is findable as a mismatch', () => withStore((store) => {
|
|
109
|
+
captureFailing(store, HEADER(0, '1 passing'));
|
|
110
|
+
const [candidate] = store.candidatesInWindow(0, Number.MAX_SAFE_INTEGER, { sessionId: SESSION });
|
|
111
|
+
const t = candidate.window_start + 100;
|
|
112
|
+
const proc = store.recordProcessStart({ containerId: CONTAINER, projectId: SESSION,
|
|
113
|
+
raw: { event_id: 1, kind: 'exec_succeeded', pid: 108, ppid: 10, argv: ['/bin/bash', '-c', 'npm test'], container_id: CONTAINER, process_start_ns: '1000', timestamp_ns: String(BigInt(t) * 1_000_000n) }, startedMs: t });
|
|
114
|
+
store.attribute({ toolCallId: candidate.call_id, eventId: proc.eventId, method: 'window', score: 1, windowStartMs: candidate.window_start, windowEndMs: candidate.window_end });
|
|
115
|
+
store.recordProcessExit({ containerId: CONTAINER,
|
|
116
|
+
raw: { event_id: 2, kind: 'process_exit', pid: 108, container_id: CONTAINER, process_start_ns: '1000', exit_code: 1, timestamp_ns: String(BigInt(t + 300) * 1_000_000n) }, endedMs: t + 300, exit_code: 1 });
|
|
117
|
+
|
|
118
|
+
const mismatch = store.search({ q: 'reported:ok observed:failed' });
|
|
119
|
+
assert.equal(mismatch.total, 1);
|
|
120
|
+
assert.equal(mismatch.hits[0].outcome, 'failed');
|
|
121
|
+
assert.equal(mismatch.hits[0].exit_code, 1);
|
|
122
|
+
assert.equal(mismatch.summary.mismatched, 1);
|
|
123
|
+
}));
|
|
124
|
+
|
|
125
|
+
test('unanswered call with no process stays unknown, pipes aggregate', () => withStore((store) => {
|
|
126
|
+
store.recordExchange(exchange('e1', [PROMPT], [CALL]));
|
|
127
|
+
const unknown = store.search({ q: 'outcome:unknown kind:call' });
|
|
128
|
+
assert.equal(unknown.total, 1);
|
|
129
|
+
assert.equal(unknown.hits[0].failure_signature, null);
|
|
130
|
+
|
|
131
|
+
captureFailing(store);
|
|
132
|
+
const stats = store.search({ q: '| stats count by outcome' });
|
|
133
|
+
assert.equal(stats.mode, 'stats');
|
|
134
|
+
assert.ok(stats.rows.some((row) => row.key === 'failed' && row.count >= 1));
|
|
135
|
+
const chart = store.search({ q: 'failed | timechart count by tool span=1d' });
|
|
136
|
+
assert.equal(chart.mode, 'timechart');
|
|
137
|
+
assert.ok(chart.rows.length >= 1);
|
|
138
|
+
|
|
139
|
+
const clusters = store.failureSignatures({});
|
|
140
|
+
assert.equal(clusters.length, 1);
|
|
141
|
+
assert.match(clusters[0].label, /cannot find module/);
|
|
142
|
+
}));
|
|
143
|
+
|
|
144
|
+
test('backfill fills the index for captures that predate it; reindex rebuilds', async () => {
|
|
145
|
+
const dir = await mkdtemp(join(tmpdir(), 'raytace-search-backfill-'));
|
|
146
|
+
const file = join(dir, 'evidence.db');
|
|
147
|
+
try {
|
|
148
|
+
const first = openEvidenceStore(file);
|
|
149
|
+
captureFailing(first);
|
|
150
|
+
first.db.exec('DELETE FROM search_events');
|
|
151
|
+
assert.equal(first.search({ q: 'outcome:failed' }).total, 0);
|
|
152
|
+
first.close();
|
|
153
|
+
const reopened = openEvidenceStore(file);
|
|
154
|
+
try {
|
|
155
|
+
assert.equal(reopened.search({ q: 'outcome:failed kind:call' }).total, 1, 'reopen backfills');
|
|
156
|
+
reopened.db.exec('DELETE FROM search_events');
|
|
157
|
+
assert.ok(reopened.reindexSearch() >= 3);
|
|
158
|
+
assert.equal(reopened.search({ q: 'outcome:failed kind:call' }).total, 1);
|
|
159
|
+
} finally { reopened.close(); }
|
|
160
|
+
} finally { await rm(dir, { recursive: true, force: true }); }
|
|
161
|
+
});
|