raytrace-cli 0.11.1__py3-none-win_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. raytrace/__init__.py +1 -0
  2. raytrace/__main__.py +33 -0
  3. raytrace/app/LICENSE +21 -0
  4. raytrace/app/README.md +599 -0
  5. raytrace/app/bin/raytrace.mjs +297 -0
  6. raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-BwCSEQnW.woff +0 -0
  7. raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-CW0RaeGs.woff2 +0 -0
  8. raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-B9HHJjqV.woff2 +0 -0
  9. raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-Dr3UlScf.woff +0 -0
  10. raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-Aqo67rzb.woff2 +0 -0
  11. raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-BmdmIIQ2.woff +0 -0
  12. raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-CUSSCpQX.woff +0 -0
  13. raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-DvUfVpUG.woff2 +0 -0
  14. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BjWJ59Pq.woff +0 -0
  15. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BtiwyxMk.woff2 +0 -0
  16. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-BJfUCQsA.woff2 +0 -0
  17. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-DR84L5F-.woff +0 -0
  18. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-4vooXBpG.woff2 +0 -0
  19. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-DRtaDpgU.woff +0 -0
  20. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-BLI3TTWz.woff +0 -0
  21. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-CJIcxD6K.woff2 +0 -0
  22. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
  23. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
  24. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
  25. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
  26. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-CTOM6hUh.woff2 +0 -0
  27. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-fLZuRloM.woff +0 -0
  28. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
  29. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
  30. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
  31. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
  32. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-9HEixskS.woff +0 -0
  33. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-V-xxqcpd.woff2 +0 -0
  34. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
  35. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
  36. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
  37. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
  38. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-BgSNZQsw.woff2 +0 -0
  39. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-DWFSQ4vo.woff +0 -0
  40. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
  41. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
  42. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
  43. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
  44. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-D38SheWl.woff2 +0 -0
  45. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-DmB0ttJJ.woff +0 -0
  46. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
  47. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
  48. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
  49. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
  50. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-D2EvbN8M.woff2 +0 -0
  51. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-iLQfcSjf.woff +0 -0
  52. raytrace/app/dist/dashboard/assets/index-DepcI8Fi.js +38 -0
  53. raytrace/app/dist/dashboard/assets/index-H97VLD5l.css +1 -0
  54. raytrace/app/dist/dashboard/favicon.svg +6 -0
  55. raytrace/app/dist/dashboard/index.html +15 -0
  56. raytrace/app/package.json +48 -0
  57. raytrace/app/proxy/agent-intent-schema.test.mjs +142 -0
  58. raytrace/app/proxy/call-assessment.mjs +385 -0
  59. raytrace/app/proxy/call-assessment.test.mjs +167 -0
  60. raytrace/app/proxy/call-category.mjs +461 -0
  61. raytrace/app/proxy/capture-health.mjs +30 -0
  62. raytrace/app/proxy/capture-health.test.mjs +44 -0
  63. raytrace/app/proxy/capture.mjs +38 -0
  64. raytrace/app/proxy/capture.test.mjs +49 -0
  65. raytrace/app/proxy/claude-code-adapter.mjs +301 -0
  66. raytrace/app/proxy/claude-code-hook.mjs +22 -0
  67. raytrace/app/proxy/codex-config.mjs +36 -0
  68. raytrace/app/proxy/codex.mjs +14 -0
  69. raytrace/app/proxy/evidence-schema.test.mjs +240 -0
  70. raytrace/app/proxy/evidence-store.mjs +993 -0
  71. raytrace/app/proxy/evidence-store.test.mjs +301 -0
  72. raytrace/app/proxy/exec-narrative.mjs +625 -0
  73. raytrace/app/proxy/exec-narrative.test.mjs +262 -0
  74. raytrace/app/proxy/execution-correlation.mjs +185 -0
  75. raytrace/app/proxy/execution-correlation.test.mjs +146 -0
  76. raytrace/app/proxy/execution-runner.mjs +89 -0
  77. raytrace/app/proxy/execution-runner.test.mjs +38 -0
  78. raytrace/app/proxy/experiment-engine.mjs +195 -0
  79. raytrace/app/proxy/experiment-engine.test.mjs +93 -0
  80. raytrace/app/proxy/explanations.mjs +36 -0
  81. raytrace/app/proxy/explanations.test.mjs +20 -0
  82. raytrace/app/proxy/failure-signature.mjs +56 -0
  83. raytrace/app/proxy/gvisor-forwarder.mjs +493 -0
  84. raytrace/app/proxy/gvisor-forwarder.test.mjs +433 -0
  85. raytrace/app/proxy/init-evidence-db.mjs +227 -0
  86. raytrace/app/proxy/local-sandboxes.mjs +94 -0
  87. raytrace/app/proxy/local-token.mjs +30 -0
  88. raytrace/app/proxy/playground-recovery.test.mjs +130 -0
  89. raytrace/app/proxy/playground.mjs +439 -0
  90. raytrace/app/proxy/prompt-context.mjs +278 -0
  91. raytrace/app/proxy/prompt-traces.mjs +40 -0
  92. raytrace/app/proxy/prompt-traces.test.mjs +32 -0
  93. raytrace/app/proxy/providers.mjs +65 -0
  94. raytrace/app/proxy/providers.test.mjs +76 -0
  95. raytrace/app/proxy/proxy.integration.test.mjs +135 -0
  96. raytrace/app/proxy/raytace-proxy.mjs +941 -0
  97. raytrace/app/proxy/request-metrics.mjs +13 -0
  98. raytrace/app/proxy/request-metrics.test.mjs +19 -0
  99. raytrace/app/proxy/request-policy.mjs +45 -0
  100. raytrace/app/proxy/request-policy.test.mjs +47 -0
  101. raytrace/app/proxy/run-recap.mjs +139 -0
  102. raytrace/app/proxy/runtime-attribution.mjs +110 -0
  103. raytrace/app/proxy/sandbox-bridge.mjs +49 -0
  104. raytrace/app/proxy/schema/EVIDENCE_DATABASE.md +276 -0
  105. raytrace/app/proxy/schema/agent_intent.sql +101 -0
  106. raytrace/app/proxy/schema/auxiliary.sql +90 -0
  107. raytrace/app/proxy/schema/capture_health.sql +14 -0
  108. raytrace/app/proxy/schema/classification.sql +89 -0
  109. raytrace/app/proxy/schema/file_context.sql +53 -0
  110. raytrace/app/proxy/schema/runtime_evidence.sql +201 -0
  111. raytrace/app/proxy/schema/search.sql +66 -0
  112. raytrace/app/proxy/schema/shared_storage.sql +21 -0
  113. raytrace/app/proxy/search.mjs +418 -0
  114. raytrace/app/proxy/search.test.mjs +161 -0
  115. raytrace/app/proxy/smoke.test.mjs +93 -0
  116. raytrace/app/proxy/step-attribution.mjs +380 -0
  117. raytrace/app/proxy/step-lab.mjs +78 -0
  118. raytrace/app/proxy/step-lab.test.mjs +38 -0
  119. raytrace/app/proxy/summaries.mjs +151 -0
  120. raytrace/app/proxy/summaries.test.mjs +69 -0
  121. raytrace/app/proxy/tool-metadata.mjs +111 -0
  122. raytrace/app/proxy/tool-metadata.test.mjs +38 -0
  123. raytrace/app/proxy/trace-view.mjs +324 -0
  124. raytrace/app/proxy/trace-view.test.mjs +92 -0
  125. raytrace/app/proxy/typesafe.mjs +63 -0
  126. raytrace/app/proxy/typesafe.test.mjs +39 -0
  127. raytrace/app/src/agents.mjs +7 -0
  128. raytrace/app/src/api.mjs +57 -0
  129. raytrace/app/src/auth.mjs +205 -0
  130. raytrace/app/src/beta.mjs +96 -0
  131. raytrace/app/src/claude-code.mjs +93 -0
  132. raytrace/app/src/commands.mjs +285 -0
  133. raytrace/app/src/config.mjs +49 -0
  134. raytrace/app/src/export.mjs +200 -0
  135. raytrace/app/src/lima.mjs +176 -0
  136. raytrace/app/src/local-sandbox.mjs +358 -0
  137. raytrace/app/src/mcp.mjs +688 -0
  138. raytrace/app/src/paths.mjs +33 -0
  139. raytrace/app/src/prompt.mjs +42 -0
  140. raytrace/app/src/proxy-process.mjs +143 -0
  141. raytrace/app/src/sandbox.mjs +306 -0
  142. raytrace/app/src/share.mjs +86 -0
  143. raytrace/app/src/terminal.mjs +68 -0
  144. raytrace/app/src/upload.mjs +198 -0
  145. raytrace/app/worker/sandbox/Dockerfile +8 -0
  146. raytrace/app/worker/sandbox/SYSCALL_CATALOG.md +197 -0
  147. raytrace/app/worker/sandbox/claude-hook.mjs +364 -0
  148. raytrace/app/worker/sandbox/collector.py +405 -0
  149. raytrace/app/worker/sandbox/dns_forwarder.py +255 -0
  150. raytrace/app/worker/sandbox/engine.py +940 -0
  151. raytrace/app/worker/sandbox/file_scan.py +491 -0
  152. raytrace/app/worker/sandbox/lima.yaml +29 -0
  153. raytrace/app/worker/sandbox/network.py +90 -0
  154. raytrace/app/worker/sandbox/projects.py +702 -0
  155. raytrace/app/worker/sandbox/setup.py +159 -0
  156. raytrace/app/worker/sandbox/syscall_catalog.sql +985 -0
  157. raytrace/app/worker/sandbox/transcript-delivery.test.mjs +153 -0
  158. raytrace/app/worker/sandbox/viewer.py +195 -0
  159. raytrace/node/LICENSE +2946 -0
  160. raytrace/node/node.exe +4 -0
  161. raytrace_cli-0.11.1.dist-info/LICENSE +21 -0
  162. raytrace_cli-0.11.1.dist-info/METADATA +42 -0
  163. raytrace_cli-0.11.1.dist-info/RECORD +165 -0
  164. raytrace_cli-0.11.1.dist-info/WHEEL +4 -0
  165. raytrace_cli-0.11.1.dist-info/entry_points.txt +2 -0
@@ -0,0 +1,493 @@
1
+ /**
2
+ * Turns gVisor process evidence from a sandbox into RayTrace evidence rows
3
+ * (runtime_processes, runtime_events, runtime_attributions; see
4
+ * proxy/evidence-store.mjs), so a sandboxed Codex session gets a real
5
+ * per-tool-call verdict instead of every call defaulting to `not_executed`.
6
+ *
7
+ * gVisor is the only execution witness: its SecCheck stream reports every
8
+ * exec and every process exit from the layer that actually mediated the
9
+ * syscall, collected outside the workload into a SQLite file in the VM and
10
+ * served on the evidence viewer (:8798). This module tails that viewer.
11
+ *
12
+ * Correctness stance is the same as execution-correlation.mjs: a missed
13
+ * match only leaves a call `not_executed`; a wrong match mislabels real
14
+ * data. Three things keep this conservative:
15
+ * 1. Candidates are scoped to the sandbox's own session id (the rtp- id
16
+ * the broker stamps on every exchange), never the whole database.
17
+ * 2. Only *top-level* execs are matched. Codex runs each tool call as
18
+ * `/bin/bash -lc <cmd>`; that bash then forks `git`, `ls`, `node`...
19
+ * Children of an already-matched command inherit its call instead of
20
+ * being matched, and so does a re-exec in the same pid (bash exec()ing
21
+ * its last command in place), so a child's argv can never claim a
22
+ * *different* pending call.
23
+ * 3. Events are processed one at a time, in event-id order, so a parent
24
+ * always claims its candidate before its children are even considered.
25
+ *
26
+ * Outcome arrives in two steps because gVisor reports it in two events:
27
+ * `exec_succeeded` records the process, and its `process_exit` closes it
28
+ * with the real exit code. Until the exit arrives the UI keeps showing its
29
+ * text heuristic for that call rather than asserting an outcome it lacks.
30
+ */
31
+ import { readFileSync } from 'node:fs';
32
+
33
+ // rtp- a project's sandbox, rtf- a Playground fork of one (worker/gvisor/projects.py).
34
+ export const SANDBOX_ID = /^rt[pf]-[a-f0-9]{32}$/;
35
+ export const CONTAINER_ID = /^[a-f0-9]{64}$/;
36
+
37
+ /** Mapping from the sandbox VM's clock to the proxy's. gVisor stamps events
38
+ * with the VM's realtime clock; exchanges are stamped by the proxy on the
39
+ * Mac. The two drift (notably after the Mac sleeps), so exec times are
40
+ * shifted by a measured offset before they are compared with exchange
41
+ * times, and `marginMs` is how uncertain that shift is. Unmeasured, the
42
+ * offset is assumed zero with a margin wide enough for ordinary drift. */
43
+ export const UNMEASURED_CLOCK = Object.freeze({ offsetMs: 0, marginMs: 2000, measured: false });
44
+
45
+ /** Picks the clock mapping from recent `/time` samples, NTP style: the
46
+ * sample with the shortest round trip has the least room for error. */
47
+ export function clockFromSamples(samples) {
48
+ const usable = (samples ?? []).filter((s) => Number.isFinite(s?.offsetMs) && Number.isFinite(s?.rttMs) && s.rttMs >= 0);
49
+ if (!usable.length) return UNMEASURED_CLOCK;
50
+ const best = usable.reduce((a, b) => (b.rttMs < a.rttMs ? b : a));
51
+ return { offsetMs: best.offsetMs, marginMs: best.rttMs / 2 + 250, measured: true };
52
+ }
53
+
54
+ /** `gvisor-<container12>-`: the id prefix every row from one sandbox shares,
55
+ * which is how a pid (reused across sandboxes) is scoped back to its own. */
56
+ export const rowPrefix = (containerId) => `gvisor-${typeof containerId === 'string' && containerId ? containerId.slice(0, 12) : 'unknown'}-`;
57
+
58
+ const eventRowId = event => `${rowPrefix(event.container_id)}${event.source_instance_id && event.source_connection_id ? `${event.source_instance_id}:${event.source_connection_id}:` : ''}${Number(event.event_id)}`;
59
+
60
+ /** ns-since-epoch (as the collector's string) -> ms, or null if unparseable. */
61
+ export function msFromNs(ns) {
62
+ if (ns == null || ns === '') return null;
63
+ try {
64
+ const value = Number(BigInt(String(ns)) / 1_000_000n);
65
+ return Number.isFinite(value) && value > 0 ? value : null;
66
+ } catch { return null; }
67
+ }
68
+
69
+ /** The call id a command's own text exports, or null. */
70
+ export function commandMarker(command) {
71
+ return /(?:^|[\s;'])export RAYTRACE_CALL_ID=([A-Za-z0-9_-]{1,128})(?=[\s;]|$)/.exec(String(command ?? ''))?.[1] ?? null;
72
+ }
73
+
74
+ /** One collector `exec_succeeded` event -> the shape matchBatch() consumes,
75
+ * or null if it isn't one (other kinds, empty argv, no usable timestamp).
76
+ * The id is the collector's own autoincrement event id, prefixed so it can
77
+ * never collide with Codex's exec-<uuid> / call_id-reused ids and so
78
+ * re-ingesting the same event twice stays a no-op (recordExecution is
79
+ * INSERT OR REPLACE on id). */
80
+ export function toExecutionEvent(event, clock = UNMEASURED_CLOCK) {
81
+ if (!event || event.kind !== 'exec_succeeded') return null;
82
+ const argv = Array.isArray(event.argv) ? event.argv.filter((part) => typeof part === 'string') : [];
83
+ const command = argv.join(' ').trim();
84
+ const observed = msFromNs(event.timestamp_ns);
85
+ const timestamp = observed == null ? null : observed - clock.offsetMs;
86
+ const pid = Number(event.pid);
87
+ const eventId = Number(event.event_id);
88
+ if (!command || timestamp == null || !Number.isFinite(pid) || !Number.isFinite(eventId)) return null;
89
+ // `markers` is the collector's allowlisted RAYTRACE_* env (see
90
+ // worker/gvisor/collector.py MARKER_KEYS). When RAYTRACE_CALL_ID is present
91
+ // it IS the answer -- the call id travelled with the process -- and the
92
+ // text matcher is skipped entirely. Absent on every event until something
93
+ // in the spawn path stamps it, which is why the matcher stays the default.
94
+ // A sandboxed Claude Code stamps it in the command itself (`export
95
+ // RAYTRACE_CALL_ID=...`, worker/gvisor/claude-hook.mjs): the shell that
96
+ // runs the command carries it in argv, the programs it starts in env.
97
+ const marker = (event.markers && typeof event.markers === 'object' ? event.markers.RAYTRACE_CALL_ID : null)
98
+ ?? commandMarker(command);
99
+ return {
100
+ id: eventRowId(event),
101
+ pid,
102
+ ppid: Number.isFinite(Number(event.ppid)) ? Number(event.ppid) : null,
103
+ command,
104
+ argv,
105
+ cwd: typeof event.cwd === 'string' ? event.cwd : null,
106
+ timestamp,
107
+ start_time_ns: typeof event.process_start_ns === 'string' ? event.process_start_ns : null,
108
+ call_marker: typeof marker === 'string' && marker ? marker : null,
109
+ margin: clock.marginMs,
110
+ // The collector event itself, for the evidence store: it is recorded
111
+ // as observed, alongside the process it describes.
112
+ container_id: typeof event.container_id === 'string' ? event.container_id : null,
113
+ raw: event,
114
+ };
115
+ }
116
+
117
+ /** One collector `exec_failed` event (an execve that returned an error, so
118
+ * the program never ran and the calling process carried on) -> a row that
119
+ * is already finished as failed, or null if unusable. */
120
+ export function toFailedExec(event, clock = UNMEASURED_CLOCK) {
121
+ if (!event || event.kind !== 'exec_failed') return null;
122
+ const argv = Array.isArray(event.argv) ? event.argv.filter((part) => typeof part === 'string') : [];
123
+ if (!argv.length && typeof event.executable === 'string' && event.executable) argv.push(event.executable);
124
+ const observed = msFromNs(event.timestamp_ns);
125
+ const pid = Number(event.pid);
126
+ const eventId = Number(event.event_id);
127
+ if (!argv.length || observed == null || !Number.isFinite(pid) || !Number.isFinite(eventId)) return null;
128
+ const at = new Date(observed - clock.offsetMs).toISOString();
129
+ return {
130
+ id: eventRowId(event),
131
+ pid,
132
+ ppid: Number.isFinite(Number(event.ppid)) ? Number(event.ppid) : null,
133
+ command: argv.join(' ').trim(),
134
+ argv,
135
+ timestamp: observed - clock.offsetMs,
136
+ start_time_ns: typeof event.process_start_ns === 'string' ? event.process_start_ns : null,
137
+ status: 'failed',
138
+ error: `exec failed: errno ${Number.isFinite(Number(event.errno)) ? Number(event.errno) : 'unknown'}`,
139
+ ended_at: at,
140
+ container_id: typeof event.container_id === 'string' ? event.container_id : null,
141
+ raw: event,
142
+ };
143
+ }
144
+
145
+ /** One collector `process_exit` event -> the status/error to close the row
146
+ * with. Never guesses: an exit we can't read is 'unknown', not 'completed'. */
147
+ export function exitOutcome(event) {
148
+ if (typeof event?.signal === 'number' && event.signal > 0) return { status: 'failed', error: `killed by signal ${event.signal}`, exit_code: null };
149
+ if (typeof event?.exit_code === 'number') {
150
+ return event.exit_code === 0 ? { status: 'completed', error: null, exit_code: 0 } : { status: 'failed', error: `exit ${event.exit_code}`, exit_code: event.exit_code };
151
+ }
152
+ return { status: 'unknown', error: null, exit_code: null };
153
+ }
154
+
155
+ /** Per-sandbox tailing state: how far into the collector's event ids we've
156
+ * read, and which pids currently belong to a matched, still-running command.
157
+ * Each entry is `{ execId, callId }` -- `execId` is that pid's OWN row (so
158
+ * its exit closes the right one) and `callId` is the proposed call the whole
159
+ * subtree belongs to (so a child inherits attribution without re-matching).
160
+ * Same-pid re-execs still collapse onto the first entry: bash exec()ing its
161
+ * last command in place is one process doing one job, not a new one. */
162
+ export class SandboxTracker {
163
+ constructor() { this.cursor = 0; this.live = new Map(); }
164
+ }
165
+
166
+ /**
167
+ * Feeds a page of collector events (any order) through the matcher. Pure
168
+ * apart from the injected `ingest`/`finish`, so it's unit-testable without
169
+ * a proxy, a viewer, or a database.
170
+ *
171
+ * @param {Array<object>} events - raw viewer events for ONE container
172
+ * @param {object} deps
173
+ * @param {string} deps.sessionId - the sandbox's rtp- id, used to scope candidates
174
+ * @param {SandboxTracker} deps.tracker
175
+ * @param {(events: Array, opts: {sessionId: string}) => Promise<Map<string, object>>|Map<string, object>} deps.ingest
176
+ * - resolves executions to call_ids and records them; returns exec id -> match for the ones that matched
177
+ * @param {(row: {id: string, ended_at: string|null, status: string, error: string|null}) => Promise<boolean>|boolean} deps.finish
178
+ * @param {(row: {exec: object, parent_call_id: string|null, tier: string}) => Promise<void>|void} [deps.descend]
179
+ * - records a process that ran underneath a call, or underneath nothing.
180
+ * Defaults to a no-op so existing callers keep their behaviour.
181
+ * @param {(row: object) => Promise<boolean>|boolean} [deps.finishByProcess]
182
+ * - closes a process by its identity (pid + start time) when `tracker.live`
183
+ * holds no link to it (after a proxy restart, or an unattributed process,
184
+ * never tracked). Same row shape as `finish`.
185
+ * @param {(row: {container_id: string, session_id: string, reason: string, raw?: object, ended?: boolean}) => Promise<number>|number} [deps.lose]
186
+ * - marks a sandbox's open rows `unknown` once their exits can no longer arrive.
187
+ * @param {{offsetMs: number, marginMs: number}} [deps.clock] - VM->proxy clock mapping
188
+ * @returns {Promise<{forwarded: number, matched: number, finished: number, descendants: number, unattributed: number, failedExecs: number, lost: number}>}
189
+ */
190
+ export async function processEvents(events, { sessionId, tracker, ingest, finish, descend = () => {}, finishByProcess = () => false, lose = () => 0, recordCapture = async () => {}, clock = UNMEASURED_CLOCK }) {
191
+ const counts = { forwarded: 0, matched: 0, finished: 0, descendants: 0, unattributed: 0, failedExecs: 0, lost: 0 };
192
+ const ordered = (Array.isArray(events) ? events : [])
193
+ .filter((event) => Number.isFinite(Number(event?.event_id)) && Number(event.event_id) > tracker.cursor)
194
+ .sort((a, b) => Number(a.event_id) - Number(b.event_id));
195
+
196
+ for (const sourceEvent of ordered) {
197
+ const event=clock.measured && Number.isFinite(sourceEvent.received_ms) ? {...sourceEvent,
198
+ recorded_at_ms:Math.round(sourceEvent.received_ms-clock.offsetMs),
199
+ ...(Number.isFinite(sourceEvent.gap_from_ms)?{gap_from_ms:Math.round(sourceEvent.gap_from_ms-clock.offsetMs)}:{})} : sourceEvent;
200
+ const previousLive = new Map(tracker.live);
201
+ const restoreLive = () => { tracker.live.clear(); for (const [pid,entry] of previousLive) tracker.live.set(pid,entry); };
202
+ let failed = false;
203
+ try {
204
+ if (event.kind === 'exec_succeeded') {
205
+ const exec = toExecutionEvent(event, clock);
206
+ if (!exec) continue;
207
+ // Same pid already carrying a matched command: bash exec()ing its last
208
+ // command in place. Still one process doing one job -- recording it
209
+ // again would both double-count and re-point that pid's exit at the
210
+ // wrong row, so it stays collapsed onto the first entry.
211
+ if (tracker.live.has(exec.pid)) continue;
212
+
213
+ // A child of a command we already attributed. Before migration 005 this
214
+ // was `continue` -- the event was seen, recognised, and dropped, which
215
+ // is how a file deleted by an npm pretest hook ended up in gVisor's
216
+ // stream and nowhere in RayTrace. It now gets its own row hanging off
217
+ // the owning call, and joins `live` so ITS children inherit too (the
218
+ // deletion is a grandchild; a one-level check never reached it).
219
+ const owner = exec.ppid != null ? tracker.live.get(exec.ppid) : null;
220
+ if (owner) {
221
+ await descend({ exec, parent_call_id: owner.callId, tier: owner.callId ? 'corroborated' : 'unverified', session_id: sessionId });
222
+ tracker.live.set(exec.pid, { execId: exec.id, callId: owner.callId });
223
+ counts.descendants += 1;
224
+ continue;
225
+ }
226
+
227
+ counts.forwarded += 1;
228
+ const matches = await ingest([exec], { sessionId });
229
+ const match = matches && matches.get ? matches.get(exec.id) : null;
230
+ if (match) {
231
+ counts.matched += 1;
232
+ tracker.live.set(exec.pid, { execId: exec.id, callId: match.call_id });
233
+ } else {
234
+ // Ran inside the agent's own sandbox, matched no proposed call. Either
235
+ // the correlator failed or nothing proposed it; both are worth seeing
236
+ // and neither was storable before 005. Deliberately NOT tracked in
237
+ // `live`: leaving its children to face the matcher keeps every
238
+ // existing match outcome identical, so this path only adds rows.
239
+ await descend({ exec, parent_call_id: null, tier: 'unverified', session_id: sessionId });
240
+ counts.unattributed += 1;
241
+ }
242
+ } else if (event.kind === 'exec_failed') {
243
+ // The program never started, so there is no exec_succeeded and no row
244
+ // for it -- a forked child whose execve failed used to vanish. The
245
+ // calling process carries on (bash reports the error and exits), so
246
+ // its own row still closes normally. Not added to `live`: nothing ran.
247
+ const failed = toFailedExec(event, clock);
248
+ if (!failed) continue;
249
+ const owner = tracker.live.get(failed.pid) ?? (failed.ppid != null ? tracker.live.get(failed.ppid) : null);
250
+ await descend({ exec: failed, parent_call_id: owner?.callId ?? null, tier: owner?.callId ? 'corroborated' : 'unverified', session_id: sessionId });
251
+ counts.failedExecs += 1;
252
+ } else if (event.kind === 'process_exit') {
253
+ const pid = Number(event.pid);
254
+ const endedMs = msFromNs(event.timestamp_ns);
255
+ const ended_at = endedMs == null ? null : new Date(endedMs - clock.offsetMs).toISOString();
256
+ const { status, error, exit_code } = exitOutcome(event);
257
+ const exitRow = { ended_at, ended_ms: endedMs == null ? null : endedMs - clock.offsetMs, status, error, exit_code,
258
+ signal: typeof event.signal === 'number' && event.signal > 0 ? event.signal : null,
259
+ pid, start_time_ns: typeof event.process_start_ns === 'string' ? event.process_start_ns : null,
260
+ container_id: typeof event.container_id === 'string' ? event.container_id : null, session_id: sessionId, raw: event };
261
+ const entry = tracker.live.get(pid);
262
+ if (entry) {
263
+ tracker.live.delete(pid);
264
+ await finish({ id: entry.execId, ...exitRow });
265
+ counts.finished += 1;
266
+ } else if (exitRow.start_time_ns) {
267
+ // No in-memory link: the process was seen before a proxy restart, or
268
+ // was never tracked (unattributed). Found again by pid + start time.
269
+ const closed = await finishByProcess(exitRow);
270
+ if (closed) counts.finished += 1;
271
+ }
272
+ } else if (event.kind === 'collector_disconnected' && typeof event.container_id === 'string' && event.container_id) {
273
+ // This sandbox's evidence stream ended. Exits for processes still
274
+ // open can no longer arrive, so they become unknown -- not failed.
275
+ // A later exit on a new connection still closes them (see
276
+ // finishExecutionByProcess), since real evidence beats "lost".
277
+ counts.lost += Number(await lose({ container_id: event.container_id, session_id: sessionId, reason: 'collector disconnected', raw: event })) || 0;
278
+ tracker.live.clear();
279
+ }
280
+ } catch (error) { failed = true; restoreLive(); throw error; }
281
+ finally {
282
+ if (!failed) {
283
+ try { await recordCapture({containerId:event.container_id,projectId:sessionId,raw:event}); }
284
+ catch (error) { restoreLive(); throw error; }
285
+ tracker.cursor = Number(event.event_id);
286
+ }
287
+ }
288
+ }
289
+ return counts;
290
+ }
291
+
292
+ /** What a request to the viewer carries (worker/gvisor/viewer.py): the token
293
+ * from the control plane's env, or the one in `file`, which sandbox setup makes.
294
+ * Read each time, as setup may come after this process starts. */
295
+ export function viewerAuth(file) {
296
+ let token = process.env.RAYTACE_VIEWER_TOKEN;
297
+ if (!token && file) try { token = readFileSync(file, 'utf8').trim(); } catch { /* no sandboxes set up yet */ }
298
+ return token ? { authorization: `Bearer ${token}` } : {};
299
+ }
300
+
301
+ /**
302
+ * Discovers sandboxes from the Mac-side manager (:8799, `/api/projects`
303
+ * maps each rtp- id to its container id), tails each container's events
304
+ * from the evidence viewer (:8798), and runs them through processEvents().
305
+ * Best-effort throughout: the manager or viewer being down just means no
306
+ * gVisor evidence for now -- logged once per outage, never fatal to the
307
+ * proxy, never a wrong row.
308
+ */
309
+ export function startGvisorForwarder({
310
+ listProjects,
311
+ managerBase = 'http://127.0.0.1:8799',
312
+ viewerBase = 'http://127.0.0.1:8798',
313
+ ingest,
314
+ finish,
315
+ descend,
316
+ finishByProcess,
317
+ lose,
318
+ recordCapture,
319
+ loadCheckpoint = async () => 0,
320
+ saveProgress = async () => {},
321
+ log = (message) => console.error(message),
322
+ fetchImpl = globalThis.fetch,
323
+ viewerHeaders = () => ({}),
324
+ pollMs = 1500,
325
+ discoverMs = 10_000,
326
+ // First discovery is deferred a little so a proxy that starts with no
327
+ // manager running prints its "listening" line before, not interleaved
328
+ // with, the one-time "sandbox manager not reachable" notice.
329
+ initialDelayMs = 2000,
330
+ } = {}) {
331
+ if (typeof ingest !== 'function' || typeof finish !== 'function') throw new TypeError('startGvisorForwarder needs ingest() and finish()');
332
+ const sandboxes = new Map(); // container_id -> { sessionId, tracker }
333
+ let managerDown = false;
334
+ let viewerDown = false;
335
+ let clockDown = false;
336
+ let clock = UNMEASURED_CLOCK;
337
+ const clockSamples = []; // most recent last; bounded
338
+ let discovering = null; // in-flight promise, so a concurrent call awaits it instead of racing
339
+ let polling = null;
340
+ let stopped = false;
341
+
342
+ const getJson = async (url, headers = {}) => {
343
+ const res = await fetchImpl(url, { headers, signal: AbortSignal.timeout(8000) });
344
+ if (!res.ok) throw Object.assign(new Error(`${res.status} from ${url}`),{status:res.status});
345
+ return res.json();
346
+ };
347
+
348
+ const discover = () => {
349
+ if (stopped) return Promise.resolve();
350
+ if (discovering) return discovering;
351
+ discovering = discoverOnce().finally(() => { discovering = null; });
352
+ return discovering;
353
+ };
354
+ /** One NTP-style sample of the VM clock against ours. Keeps the last few;
355
+ * a failed sample leaves the previous mapping in place, and with none at
356
+ * all the forwarder runs unmeasured (zero offset, wide margin). */
357
+ const sampleClock = async () => {
358
+ try {
359
+ const sent = Date.now();
360
+ const body = await getJson(`${viewerBase}/time`, viewerHeaders());
361
+ const received = Date.now();
362
+ const vmMs = msFromNs(body?.realtime_ns);
363
+ if (vmMs == null) throw new Error('no realtime_ns in /time response');
364
+ clockSamples.push({ offsetMs: vmMs - (sent + received) / 2, rttMs: received - sent });
365
+ if (clockSamples.length > 5) clockSamples.shift();
366
+ clock = clockFromSamples(clockSamples);
367
+ if (clockDown) { clockDown = false; log('[raytace][gvisor] sandbox clock measurable again'); }
368
+ } catch (error) {
369
+ if (!clockDown) {
370
+ clockDown = true;
371
+ log(`[raytace][gvisor] could not read the sandbox clock at ${viewerBase}/time (${error.message}) -- matching with a ${clock.marginMs} ms margin until it can`);
372
+ }
373
+ }
374
+ };
375
+
376
+ const discoverOnce = async () => {
377
+ try {
378
+ const projects = typeof listProjects==='function' ? await listProjects() : await getJson(`${managerBase}/api/projects`);
379
+ await sampleClock();
380
+ const present = new Set();
381
+ for (const project of Array.isArray(projects) ? projects : []) {
382
+ const sessionId = project?.id;
383
+ const containerId = project?.container_id;
384
+ if (!SANDBOX_ID.test(sessionId || '') || !CONTAINER_ID.test(containerId || '')) continue;
385
+ // The manager keeps a deleted sandbox's record, but its container is
386
+ // gone: treat it as removed rather than tail it for good.
387
+ if (project?.status === 'deleted') continue;
388
+ present.add(containerId);
389
+ if (!sandboxes.has(containerId)) {
390
+ const tracker = new SandboxTracker();
391
+ // Source identity and the durable checkpoint are loaded on poll.
392
+ tracker.cursor = 0; // Backfill safely until a durable source checkpoint is loaded.
393
+ sandboxes.set(containerId, { sessionId, tracker });
394
+ log(`[raytace][gvisor] forwarding evidence for sandbox ${sessionId} (container ${containerId.slice(0, 12)}) from event ${tracker.cursor}`);
395
+ }
396
+ }
397
+ // A sandbox the manager no longer lists is gone, and so is any exit
398
+ // its open rows were waiting for. Only on a successful listing: an
399
+ // unreachable manager proves nothing about the sandboxes.
400
+ for (const [containerId, { sessionId }] of sandboxes) {
401
+ if (present.has(containerId)) continue;
402
+ sandboxes.delete(containerId);
403
+ const lost = typeof lose === 'function' ? Number(await lose({ container_id: containerId, session_id: sessionId, reason: 'sandbox removed', ended: true })) || 0 : 0;
404
+ log(`[raytace][gvisor] sandbox ${sessionId} is gone; stopped forwarding${lost ? `, ${lost} open execution(s) marked unknown` : ''}`);
405
+ }
406
+ if (managerDown) { managerDown = false; log('[raytace][gvisor] sandbox manager reachable again'); }
407
+ } catch (error) {
408
+ if (!managerDown) {
409
+ managerDown = true;
410
+ log(`[raytace][gvisor] sandbox manager not reachable at ${managerBase} (${error.message}) -- gVisor evidence forwarding idle until it is. Start it with: npm run sandbox:manager`);
411
+ }
412
+ }
413
+ };
414
+
415
+ const poll = () => {
416
+ if (stopped || !sandboxes.size) return Promise.resolve();
417
+ if (polling) return polling;
418
+ polling = pollOnce().catch(error=>log(`[raytace][gvisor] recording progress could not be saved: ${error.message}`)).finally(() => { polling = null; });
419
+ return polling;
420
+ };
421
+ const pollOnce = async () => {
422
+ for (const [containerId, { sessionId, tracker }] of sandboxes) {
423
+ let coverage;
424
+ try { coverage = await getJson(`${viewerBase}/coverage?container=${containerId}`, viewerHeaders()); }
425
+ catch(error) {
426
+ if(error.status!==404) {
427
+ if(tracker.sourceId) await saveProgress({containerId,projectId:sessionId,sourceId:tracker.sourceId,cursor:tracker.cursor,
428
+ coverage:{streams:[],source_instance_id:tracker.sourceId,unavailable:true,checked_at_ms:Date.now()}});
429
+ continue;
430
+ }
431
+ coverage={streams:[],source_instance_id:'legacy',latest:0,checked_at_ms:Date.now()};
432
+ }
433
+ const sourceId=coverage.source_instance_id || 'legacy';
434
+ if(tracker.sourceId !== sourceId) {
435
+ tracker.cursor=Number(await loadCheckpoint(containerId,sourceId,sessionId)) || 0;
436
+ tracker.sourceId=sourceId; tracker.live.clear();
437
+ }
438
+ const url = `${viewerBase}/events?container=${containerId}&after=${tracker.cursor}&limit=500&processes=1`;
439
+ let page;
440
+ try { page = await getJson(url, viewerHeaders()); } catch (error) {
441
+ await saveProgress({containerId,projectId:sessionId,sourceId,cursor:tracker.cursor,
442
+ coverage:{...coverage,streams:[],unavailable:true,checked_at_ms:Date.now()}});
443
+ if (!viewerDown) { viewerDown = true; log(`[raytace][gvisor] evidence viewer not reachable at ${viewerBase} (${error.message})`); }
444
+ continue;
445
+ }
446
+ if(sourceId==='legacy' && page?.coverage?.source_instance_id) coverage=page.coverage;
447
+ coverage={...coverage,checked_at_ms:Date.now(),clock_measured:clock.measured,
448
+ heartbeat_ms:clock.measured && coverage.heartbeat_ms ? Math.round(coverage.heartbeat_ms-clock.offsetMs) : null,
449
+ streams:(coverage.streams??[]).map(stream=>({...stream,clock_measured:clock.measured,
450
+ started_ms:Math.round(stream.started_ms-clock.offsetMs),
451
+ ended_ms:stream.ended_ms==null?null:Math.round(stream.ended_ms-clock.offsetMs),
452
+ heartbeat_ms:Math.round(stream.heartbeat_ms-clock.offsetMs)}))};
453
+ if (viewerDown) { viewerDown = false; log('[raytace][gvisor] evidence viewer reachable again'); }
454
+ // An ingest/finish failure (e.g. a DB error) must not stall the loop
455
+ // or leave the cursor wedged; log it and move on to the next poll.
456
+ try {
457
+ const counts = await processEvents(page?.events, { sessionId, tracker, ingest, finish, descend,
458
+ ...(typeof finishByProcess === 'function' ? { finishByProcess } : {}),
459
+ ...(typeof lose === 'function' ? { lose } : {}),
460
+ ...(typeof recordCapture === 'function' ? {recordCapture} : {}), clock });
461
+ // Exhausted selected events: acknowledge excluded source rows too,
462
+ // otherwise a non-process event would keep us catching up forever.
463
+ if(!page.next_after) {
464
+ tracker.cursor=Math.max(tracker.cursor,Number(page.coverage?.latest??coverage.latest)||0);
465
+ if(clock.measured)coverage.forwarded_through_ms=coverage.heartbeat_ms;
466
+ }
467
+ await saveProgress({containerId,projectId:sessionId,sourceId,cursor:tracker.cursor,coverage});
468
+ if (counts.matched || counts.finished || counts.descendants || counts.unattributed || counts.failedExecs || counts.lost) {
469
+ log(`[raytace][gvisor] ${sessionId.slice(0, 12)}: ${counts.matched} exec(s) matched to proposed calls, `
470
+ + `${counts.descendants} descendant(s), ${counts.unattributed} unattributed, ${counts.failedExecs} failed exec(s), `
471
+ + `${counts.finished} closed out, ${counts.lost} marked unknown, ${counts.forwarded} top-level exec(s) considered`);
472
+ }
473
+ } catch (error) {
474
+ log(`[raytace][gvisor] ${sessionId.slice(0, 12)}: failed to record evidence: ${error.message}`);
475
+ }
476
+ }
477
+ };
478
+
479
+ const firstDiscover = setTimeout(discover, initialDelayMs);
480
+ const discoverTimer = setInterval(discover, discoverMs);
481
+ const pollTimer = setInterval(poll, pollMs);
482
+ firstDiscover.unref?.(); discoverTimer.unref?.(); pollTimer.unref?.();
483
+
484
+ return {
485
+ stop() { stopped = true; clearTimeout(firstDiscover); clearInterval(discoverTimer); clearInterval(pollTimer); },
486
+ /** Test/inspection hook: the sandboxes currently being tailed. */
487
+ sandboxes,
488
+ discover,
489
+ poll,
490
+ /** Test/inspection hook: the VM->proxy clock mapping in use. */
491
+ clock: () => clock,
492
+ };
493
+ }