raytrace-cli 0.11.1__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raytrace/__init__.py +1 -0
- raytrace/__main__.py +33 -0
- raytrace/app/LICENSE +21 -0
- raytrace/app/README.md +599 -0
- raytrace/app/bin/raytrace.mjs +297 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-BwCSEQnW.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-CW0RaeGs.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-B9HHJjqV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-Dr3UlScf.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-Aqo67rzb.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-BmdmIIQ2.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-CUSSCpQX.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-DvUfVpUG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BjWJ59Pq.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BtiwyxMk.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-BJfUCQsA.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-DR84L5F-.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-4vooXBpG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-DRtaDpgU.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-BLI3TTWz.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-CJIcxD6K.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-CTOM6hUh.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-fLZuRloM.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-9HEixskS.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-V-xxqcpd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-BgSNZQsw.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-DWFSQ4vo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-D38SheWl.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-DmB0ttJJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-D2EvbN8M.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-iLQfcSjf.woff +0 -0
- raytrace/app/dist/dashboard/assets/index-DepcI8Fi.js +38 -0
- raytrace/app/dist/dashboard/assets/index-H97VLD5l.css +1 -0
- raytrace/app/dist/dashboard/favicon.svg +6 -0
- raytrace/app/dist/dashboard/index.html +15 -0
- raytrace/app/package.json +48 -0
- raytrace/app/proxy/agent-intent-schema.test.mjs +142 -0
- raytrace/app/proxy/call-assessment.mjs +385 -0
- raytrace/app/proxy/call-assessment.test.mjs +167 -0
- raytrace/app/proxy/call-category.mjs +461 -0
- raytrace/app/proxy/capture-health.mjs +30 -0
- raytrace/app/proxy/capture-health.test.mjs +44 -0
- raytrace/app/proxy/capture.mjs +38 -0
- raytrace/app/proxy/capture.test.mjs +49 -0
- raytrace/app/proxy/claude-code-adapter.mjs +301 -0
- raytrace/app/proxy/claude-code-hook.mjs +22 -0
- raytrace/app/proxy/codex-config.mjs +36 -0
- raytrace/app/proxy/codex.mjs +14 -0
- raytrace/app/proxy/evidence-schema.test.mjs +240 -0
- raytrace/app/proxy/evidence-store.mjs +993 -0
- raytrace/app/proxy/evidence-store.test.mjs +301 -0
- raytrace/app/proxy/exec-narrative.mjs +625 -0
- raytrace/app/proxy/exec-narrative.test.mjs +262 -0
- raytrace/app/proxy/execution-correlation.mjs +185 -0
- raytrace/app/proxy/execution-correlation.test.mjs +146 -0
- raytrace/app/proxy/execution-runner.mjs +89 -0
- raytrace/app/proxy/execution-runner.test.mjs +38 -0
- raytrace/app/proxy/experiment-engine.mjs +195 -0
- raytrace/app/proxy/experiment-engine.test.mjs +93 -0
- raytrace/app/proxy/explanations.mjs +36 -0
- raytrace/app/proxy/explanations.test.mjs +20 -0
- raytrace/app/proxy/failure-signature.mjs +56 -0
- raytrace/app/proxy/gvisor-forwarder.mjs +493 -0
- raytrace/app/proxy/gvisor-forwarder.test.mjs +433 -0
- raytrace/app/proxy/init-evidence-db.mjs +227 -0
- raytrace/app/proxy/local-sandboxes.mjs +94 -0
- raytrace/app/proxy/local-token.mjs +30 -0
- raytrace/app/proxy/playground-recovery.test.mjs +130 -0
- raytrace/app/proxy/playground.mjs +439 -0
- raytrace/app/proxy/prompt-context.mjs +278 -0
- raytrace/app/proxy/prompt-traces.mjs +40 -0
- raytrace/app/proxy/prompt-traces.test.mjs +32 -0
- raytrace/app/proxy/providers.mjs +65 -0
- raytrace/app/proxy/providers.test.mjs +76 -0
- raytrace/app/proxy/proxy.integration.test.mjs +135 -0
- raytrace/app/proxy/raytace-proxy.mjs +941 -0
- raytrace/app/proxy/request-metrics.mjs +13 -0
- raytrace/app/proxy/request-metrics.test.mjs +19 -0
- raytrace/app/proxy/request-policy.mjs +45 -0
- raytrace/app/proxy/request-policy.test.mjs +47 -0
- raytrace/app/proxy/run-recap.mjs +139 -0
- raytrace/app/proxy/runtime-attribution.mjs +110 -0
- raytrace/app/proxy/sandbox-bridge.mjs +49 -0
- raytrace/app/proxy/schema/EVIDENCE_DATABASE.md +276 -0
- raytrace/app/proxy/schema/agent_intent.sql +101 -0
- raytrace/app/proxy/schema/auxiliary.sql +90 -0
- raytrace/app/proxy/schema/capture_health.sql +14 -0
- raytrace/app/proxy/schema/classification.sql +89 -0
- raytrace/app/proxy/schema/file_context.sql +53 -0
- raytrace/app/proxy/schema/runtime_evidence.sql +201 -0
- raytrace/app/proxy/schema/search.sql +66 -0
- raytrace/app/proxy/schema/shared_storage.sql +21 -0
- raytrace/app/proxy/search.mjs +418 -0
- raytrace/app/proxy/search.test.mjs +161 -0
- raytrace/app/proxy/smoke.test.mjs +93 -0
- raytrace/app/proxy/step-attribution.mjs +380 -0
- raytrace/app/proxy/step-lab.mjs +78 -0
- raytrace/app/proxy/step-lab.test.mjs +38 -0
- raytrace/app/proxy/summaries.mjs +151 -0
- raytrace/app/proxy/summaries.test.mjs +69 -0
- raytrace/app/proxy/tool-metadata.mjs +111 -0
- raytrace/app/proxy/tool-metadata.test.mjs +38 -0
- raytrace/app/proxy/trace-view.mjs +324 -0
- raytrace/app/proxy/trace-view.test.mjs +92 -0
- raytrace/app/proxy/typesafe.mjs +63 -0
- raytrace/app/proxy/typesafe.test.mjs +39 -0
- raytrace/app/src/agents.mjs +7 -0
- raytrace/app/src/api.mjs +57 -0
- raytrace/app/src/auth.mjs +205 -0
- raytrace/app/src/beta.mjs +96 -0
- raytrace/app/src/claude-code.mjs +93 -0
- raytrace/app/src/commands.mjs +285 -0
- raytrace/app/src/config.mjs +49 -0
- raytrace/app/src/export.mjs +200 -0
- raytrace/app/src/lima.mjs +176 -0
- raytrace/app/src/local-sandbox.mjs +358 -0
- raytrace/app/src/mcp.mjs +688 -0
- raytrace/app/src/paths.mjs +33 -0
- raytrace/app/src/prompt.mjs +42 -0
- raytrace/app/src/proxy-process.mjs +143 -0
- raytrace/app/src/sandbox.mjs +306 -0
- raytrace/app/src/share.mjs +86 -0
- raytrace/app/src/terminal.mjs +68 -0
- raytrace/app/src/upload.mjs +198 -0
- raytrace/app/worker/sandbox/Dockerfile +8 -0
- raytrace/app/worker/sandbox/SYSCALL_CATALOG.md +197 -0
- raytrace/app/worker/sandbox/claude-hook.mjs +364 -0
- raytrace/app/worker/sandbox/collector.py +405 -0
- raytrace/app/worker/sandbox/dns_forwarder.py +255 -0
- raytrace/app/worker/sandbox/engine.py +940 -0
- raytrace/app/worker/sandbox/file_scan.py +491 -0
- raytrace/app/worker/sandbox/lima.yaml +29 -0
- raytrace/app/worker/sandbox/network.py +90 -0
- raytrace/app/worker/sandbox/projects.py +702 -0
- raytrace/app/worker/sandbox/setup.py +159 -0
- raytrace/app/worker/sandbox/syscall_catalog.sql +985 -0
- raytrace/app/worker/sandbox/transcript-delivery.test.mjs +153 -0
- raytrace/app/worker/sandbox/viewer.py +195 -0
- raytrace/node/LICENSE +2946 -0
- raytrace/node/node.exe +4 -0
- raytrace_cli-0.11.1.dist-info/LICENSE +21 -0
- raytrace_cli-0.11.1.dist-info/METADATA +42 -0
- raytrace_cli-0.11.1.dist-info/RECORD +165 -0
- raytrace_cli-0.11.1.dist-info/WHEEL +4 -0
- raytrace_cli-0.11.1.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This machine's local sandboxes, for the proxy: what raytace's sandbox
|
|
3
|
+
* manager answers there, from the CLI's own records instead. Each sandbox is
|
|
4
|
+
* one the CLI made (`raytrace sandbox create --local`, kept in
|
|
5
|
+
* ~/.raytrace/sandboxes.json); its container and state come from the Lima VM
|
|
6
|
+
* (worker/sandbox/projects.py list), and what the kernel saw it do from the
|
|
7
|
+
* fact engine's records there (projects.py activity).
|
|
8
|
+
*
|
|
9
|
+
* limactl is the one the CLI resolved (src/lima.mjs): the user's own, or the
|
|
10
|
+
* one RayTrace downloaded.
|
|
11
|
+
*/
|
|
12
|
+
import { spawn } from 'node:child_process';
|
|
13
|
+
import { readFile } from 'node:fs/promises';
|
|
14
|
+
import { join } from 'node:path';
|
|
15
|
+
import { findLima } from '../src/lima.mjs';
|
|
16
|
+
|
|
17
|
+
export const VM = process.env.RAYTRACE_VM || 'raytrace';
|
|
18
|
+
const SANDBOX = /^rtp-[a-f0-9]{32}$/;
|
|
19
|
+
// Listing goes through the VM, a few hundred milliseconds: the forwarder and
|
|
20
|
+
// the dashboard both poll it, so one answer serves for a few seconds.
|
|
21
|
+
const LIST_TTL_MS = 4000;
|
|
22
|
+
|
|
23
|
+
export const limactl = () => process.env.RAYTRACE_LIMACTL || findLima()?.binary || 'limactl';
|
|
24
|
+
|
|
25
|
+
function run(args, { input } = {}) {
|
|
26
|
+
return new Promise((resolve, reject) => {
|
|
27
|
+
const child = spawn(limactl(), args, { stdio: ['pipe', 'pipe', 'pipe'] });
|
|
28
|
+
const out = []; const err = [];
|
|
29
|
+
child.stdout.on('data', (chunk) => out.push(chunk));
|
|
30
|
+
child.stderr.on('data', (chunk) => err.push(chunk));
|
|
31
|
+
child.on('error', reject);
|
|
32
|
+
child.on('close', (code) => (code === 0 ? resolve(Buffer.concat(out).toString('utf8'))
|
|
33
|
+
: reject(new Error(Buffer.concat(err).toString('utf8').trim().split('\n').at(-1) || `limactl exited ${code}`))));
|
|
34
|
+
child.stdin.end(input ?? '');
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
const projects = (args) => run(['shell', '--workdir', '/', VM, 'sudo', 'python3', '/opt/raytace/projects.py', ...args]);
|
|
38
|
+
|
|
39
|
+
async function vmRunning() {
|
|
40
|
+
const listed = await run(['list', '--format', '{{.Name}} {{.Status}}']);
|
|
41
|
+
return listed.split('\n').some((line) => line === `${VM} Running`);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function localSandboxes({ dataDir }) {
|
|
45
|
+
const registryFile = join(dataDir, 'sandboxes.json');
|
|
46
|
+
const registry = async () => { try { return JSON.parse(await readFile(registryFile, 'utf8')); } catch { return {}; } };
|
|
47
|
+
let cached = null;
|
|
48
|
+
|
|
49
|
+
/** Every local sandbox: {id, name, status, created, container_id}, and its
|
|
50
|
+
* Playground forks (with fork_of). Throws
|
|
51
|
+
* while the VM is stopped or unreachable, which says nothing about whether
|
|
52
|
+
* the sandboxes exist (the forwarder must not take them for removed). */
|
|
53
|
+
async function list() {
|
|
54
|
+
if (cached && Date.now() - cached.at < LIST_TTL_MS) return cached.value;
|
|
55
|
+
const boxes = await registry();
|
|
56
|
+
if (!Object.keys(boxes).length) return [];
|
|
57
|
+
if (!(await vmRunning())) throw new Error(`the ${VM} VM is not running`);
|
|
58
|
+
const containers = new Map(JSON.parse((await projects(['list'])) || '[]').map((box) => [box.id, box]));
|
|
59
|
+
const value = Object.entries(boxes).map(([id, box]) => {
|
|
60
|
+
const container = containers.get(id);
|
|
61
|
+
return { id, name: box.name, root: box.root ?? null, created: box.created, status: container?.status ?? 'missing',
|
|
62
|
+
container_id: container?.container_id ?? null, size_bytes: container?.size_bytes ?? null };
|
|
63
|
+
});
|
|
64
|
+
// Playground forks of these sandboxes: their own containers, which the
|
|
65
|
+
// forwarder tails like any sandbox. Not sandboxes of the user's: `fork_of` says so.
|
|
66
|
+
for (const box of containers.values()) {
|
|
67
|
+
if (/^rtf-[a-f0-9]{32}$/.test(box.id) && boxes[box.fork_of]) {
|
|
68
|
+
value.push({ id: box.id, name: null, fork_of: box.fork_of, created: box.created, status: box.status,
|
|
69
|
+
container_id: box.container_id ?? null, size_bytes: box.size_bytes ?? null });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
cached = { at: Date.now(), value };
|
|
73
|
+
return value;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The registry alone, for a listing that must answer with the VM stopped. */
|
|
77
|
+
async function known() {
|
|
78
|
+
const boxes = await registry();
|
|
79
|
+
return Object.entries(boxes).map(([id, box]) => ({ id, name: box.name, root: box.root ?? null, created: box.created }));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** What the kernel saw a sandbox do, by kind (files, network, lookups, ...);
|
|
83
|
+
* since/until are epoch ms, to scope it to one prompt. */
|
|
84
|
+
async function activity(id, { since = 0, until = 0 } = {}) {
|
|
85
|
+
if (!SANDBOX.test(id)) throw new Error('Invalid sandbox id');
|
|
86
|
+
const window = [...(since ? ['--since', String(since)] : []), ...(until ? ['--until', String(until)] : [])];
|
|
87
|
+
return JSON.parse(await projects(['activity', id, ...window]));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** When a sandbox was created, for its session's start (the CLI's record). */
|
|
91
|
+
async function createdAt(id) { return (await registry())[id]?.created ?? null; }
|
|
92
|
+
|
|
93
|
+
return { list, known, activity, createdAt, invalidate: () => { cached = null; } };
|
|
94
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local API's credential: a random token in the data folder, readable only
|
|
3
|
+
* by the user who runs RayTrace. Binding to 127.0.0.1 keeps the network out and
|
|
4
|
+
* the Origin and Host checks keep web pages out, but neither says anything
|
|
5
|
+
* about another account on the same machine; this file's mode does.
|
|
6
|
+
*/
|
|
7
|
+
import { randomBytes } from 'node:crypto';
|
|
8
|
+
import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
9
|
+
import { homedir } from 'node:os';
|
|
10
|
+
import { join } from 'node:path';
|
|
11
|
+
|
|
12
|
+
const FILE = 'api-token';
|
|
13
|
+
|
|
14
|
+
/** The data folder of a caller the CLI did not start, such as the Claude Code hook. */
|
|
15
|
+
export const dataFolder = () => process.env.RAYTACE_DATA_DIR || process.env.RAYTRACE_HOME || join(homedir(), '.raytrace');
|
|
16
|
+
|
|
17
|
+
export function readToken(folder = dataFolder()) {
|
|
18
|
+
try { return readFileSync(join(folder, FILE), 'utf8').trim() || null; } catch { return null; }
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** The token, made on first use. Also closes the folder: what is recorded in it
|
|
22
|
+
* (prompts, tool output) is as private as the token. */
|
|
23
|
+
export function ensureToken(folder = dataFolder()) {
|
|
24
|
+
mkdirSync(folder, { recursive: true, mode: 0o700 });
|
|
25
|
+
try { chmodSync(folder, 0o700); } catch { /* not ours to change, or a filesystem without modes */ }
|
|
26
|
+
try { writeFileSync(join(folder, FILE), `${randomBytes(32).toString('hex')}\n`, { mode: 0o600, flag: 'wx' }); }
|
|
27
|
+
catch (error) { if (error.code !== 'EEXIST') throw error; }
|
|
28
|
+
chmodSync(join(folder, FILE), 0o600);
|
|
29
|
+
return readToken(folder);
|
|
30
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import test from 'node:test';
|
|
3
|
+
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { startPlayground } from './playground.mjs';
|
|
7
|
+
|
|
8
|
+
const id = (letter) => `rtf-${letter.repeat(32)}`;
|
|
9
|
+
const job = (letter, status, extra = {}) => ({ id: id(letter), kind: 'fork', status,
|
|
10
|
+
created_at: new Date().toISOString(), check: { status: 'pending' }, ...extra });
|
|
11
|
+
|
|
12
|
+
async function fixture(t, records, initial = {}) {
|
|
13
|
+
const directory = await mkdtemp(join(tmpdir(), 'raytrace-recovery-test-'));
|
|
14
|
+
const statePath = join(directory, 'state.json');
|
|
15
|
+
const before = { path: process.env.PATH, ssh: process.env.RAYTACE_WORKER_SSH, lima: process.env.RAYTRACE_LIMACTL };
|
|
16
|
+
const saveHost = (state) => writeFile(statePath, JSON.stringify(state));
|
|
17
|
+
const host = async () => JSON.parse(await readFile(statePath, 'utf8'));
|
|
18
|
+
await saveHost({ calls: [], boxes: records.filter((r) => r.status !== 'queued').map((r) => ({ id: r.id, running: true })),
|
|
19
|
+
failStop: [], failList: false, ...initial });
|
|
20
|
+
// A fake host only changes an in-memory-fixture JSON file. No VM/provider is used.
|
|
21
|
+
await writeFile(join(directory, 'limactl'), `#!${process.execPath}
|
|
22
|
+
const fs = require('node:fs');
|
|
23
|
+
const file = ${JSON.stringify(statePath)};
|
|
24
|
+
const state = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
25
|
+
const [action, id] = process.argv.slice(process.argv.indexOf('/opt/raytace/projects.py') + 1);
|
|
26
|
+
state.calls.push([action, id]);
|
|
27
|
+
fs.writeFileSync(file, JSON.stringify(state));
|
|
28
|
+
if (action === 'fork-stop' && state.stopDelayMs) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, state.stopDelayMs);
|
|
29
|
+
let code = 0;
|
|
30
|
+
if (action === 'fork-stop') {
|
|
31
|
+
const box = state.boxes.find((box) => box.id === id);
|
|
32
|
+
if (!box || state.failStop.includes(id)) code = 1;
|
|
33
|
+
else box.running = false;
|
|
34
|
+
} else if (action === 'list' && state.failList) code = 1;
|
|
35
|
+
else if (action === 'fork-delete') throw new Error('Recovery must retain recent fork data');
|
|
36
|
+
fs.writeFileSync(file, JSON.stringify(state));
|
|
37
|
+
if (code) console.error('Simulated unavailable worker or absent container');
|
|
38
|
+
else console.log(JSON.stringify(action === 'list' ? state.boxes : {}));
|
|
39
|
+
process.exitCode = code;
|
|
40
|
+
`, { mode: 0o700 });
|
|
41
|
+
process.env.PATH = directory + ':' + before.path;
|
|
42
|
+
process.env.RAYTRACE_LIMACTL = join(directory, 'limactl');
|
|
43
|
+
delete process.env.RAYTACE_WORKER_SSH;
|
|
44
|
+
t.after(async () => {
|
|
45
|
+
process.env.PATH = before.path;
|
|
46
|
+
if (before.ssh === undefined) delete process.env.RAYTACE_WORKER_SSH;
|
|
47
|
+
else process.env.RAYTACE_WORKER_SSH = before.ssh;
|
|
48
|
+
if (before.lima === undefined) delete process.env.RAYTRACE_LIMACTL;
|
|
49
|
+
else process.env.RAYTRACE_LIMACTL = before.lima;
|
|
50
|
+
await rm(directory, { recursive: true, force: true });
|
|
51
|
+
});
|
|
52
|
+
const saves = [];
|
|
53
|
+
const playground = startPlayground({ db: { loadExperiments: () => records,
|
|
54
|
+
saveExperiment(record) { saves.push(structuredClone(record)); } }, transcriptsDir: directory,
|
|
55
|
+
hookSource: () => '', orphans: false, log() {} });
|
|
56
|
+
return { playground, host, saveHost, saves };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
test('recovery stops each known active fork before saving it interrupted, retaining kept data', async (t) => {
|
|
60
|
+
const records = ['creating', 'running', 'collecting', 'checking'].map((status, index) => job('abcd'[index], status, { keep: true }));
|
|
61
|
+
const { playground, host, saves } = await fixture(t, records);
|
|
62
|
+
await playground.recover();
|
|
63
|
+
const state = await host();
|
|
64
|
+
assert.equal(state.boxes.every((box) => !box.running), true);
|
|
65
|
+
assert.deepEqual(state.calls, records.map((record) => ['fork-stop', record.id]));
|
|
66
|
+
assert.equal(saves.length, 4);
|
|
67
|
+
for (const record of records) {
|
|
68
|
+
assert.equal(playground.get(record.id).status, 'interrupted');
|
|
69
|
+
assert.equal(playground.get(record.id).deleted_at, undefined);
|
|
70
|
+
assert.equal(playground.get(record.id).keep, true);
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test('queued, completed and deleted jobs do not stop unrelated workers', async (t) => {
|
|
75
|
+
const records = [job('a', 'queued'), job('b', 'completed'), job('c', 'running', { deleted_at: 'already deleted' })];
|
|
76
|
+
const { playground, host } = await fixture(t, records, { boxes: [{ id: id('d'), running: true }] });
|
|
77
|
+
await playground.recover();
|
|
78
|
+
assert.equal(playground.get(id('a')).status, 'interrupted');
|
|
79
|
+
assert.ok(playground.get(id('a')).deleted_at);
|
|
80
|
+
assert.equal(playground.get(id('b')).status, 'completed');
|
|
81
|
+
assert.deepEqual((await host()).calls, []);
|
|
82
|
+
assert.equal((await host()).boxes[0].running, true);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test('failed stop remains active and blocks new runs until a successful recovery retry', async (t) => {
|
|
86
|
+
const record = job('a', 'checking');
|
|
87
|
+
const { playground, host, saveHost } = await fixture(t, [record], { failStop: [record.id] });
|
|
88
|
+
await playground.recover();
|
|
89
|
+
assert.equal(playground.get(record.id).status, 'checking');
|
|
90
|
+
assert.equal(playground.get(record.id).recovery_stop_pending, true);
|
|
91
|
+
assert.match(playground.get(record.id).error, /Could not confirm.*Restart RayTrace/);
|
|
92
|
+
assert.equal((await host()).boxes[0].running, true);
|
|
93
|
+
await assert.rejects(playground.create({}), /previous fork could not be stopped/);
|
|
94
|
+
await saveHost({ ...await host(), failStop: [] });
|
|
95
|
+
await playground.recover();
|
|
96
|
+
assert.equal(playground.get(record.id).status, 'interrupted');
|
|
97
|
+
assert.equal(playground.get(record.id).recovery_stop_pending, undefined);
|
|
98
|
+
assert.equal((await host()).boxes[0].running, false);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test('a creating job confirmed absent does not permanently block new runs', async (t) => {
|
|
102
|
+
const record = job('a', 'creating');
|
|
103
|
+
const { playground, host } = await fixture(t, [record], { boxes: [{ id: id('b'), running: true }] });
|
|
104
|
+
await playground.recover();
|
|
105
|
+
assert.equal(playground.get(record.id).status, 'interrupted');
|
|
106
|
+
assert.equal(playground.get(record.id).recovery_stop_pending, undefined);
|
|
107
|
+
assert.deepEqual((await host()).calls, [['fork-stop', record.id], ['list', null]]);
|
|
108
|
+
assert.equal((await host()).boxes[0].running, true);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
test('failed host inventory cannot establish a missing fork is stopped', async (t) => {
|
|
112
|
+
const record = job('a', 'creating');
|
|
113
|
+
const { playground } = await fixture(t, [record], { boxes: [], failList: true });
|
|
114
|
+
await playground.recover();
|
|
115
|
+
assert.equal(playground.get(record.id).status, 'creating');
|
|
116
|
+
assert.equal(playground.get(record.id).recovery_stop_pending, true);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test('new fork creation is blocked while restart recovery is still stopping old work', async (t) => {
|
|
120
|
+
const record = job('a', 'running');
|
|
121
|
+
const { playground, host } = await fixture(t, [record], { stopDelayMs: 300 });
|
|
122
|
+
const recovery = playground.recover();
|
|
123
|
+
try {
|
|
124
|
+
await assert.rejects(playground.create({}), /previous fork could not be stopped/);
|
|
125
|
+
} finally {
|
|
126
|
+
await recovery;
|
|
127
|
+
}
|
|
128
|
+
assert.equal(playground.get(record.id).status, 'interrupted');
|
|
129
|
+
assert.equal((await host()).boxes[0].running, false);
|
|
130
|
+
});
|