raytrace-cli 0.11.1__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. raytrace/__init__.py +1 -0
  2. raytrace/__main__.py +33 -0
  3. raytrace/app/LICENSE +21 -0
  4. raytrace/app/README.md +599 -0
  5. raytrace/app/bin/raytrace.mjs +297 -0
  6. raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-BwCSEQnW.woff +0 -0
  7. raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-CW0RaeGs.woff2 +0 -0
  8. raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-B9HHJjqV.woff2 +0 -0
  9. raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-Dr3UlScf.woff +0 -0
  10. raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-Aqo67rzb.woff2 +0 -0
  11. raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-BmdmIIQ2.woff +0 -0
  12. raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-CUSSCpQX.woff +0 -0
  13. raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-DvUfVpUG.woff2 +0 -0
  14. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BjWJ59Pq.woff +0 -0
  15. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BtiwyxMk.woff2 +0 -0
  16. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-BJfUCQsA.woff2 +0 -0
  17. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-DR84L5F-.woff +0 -0
  18. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-4vooXBpG.woff2 +0 -0
  19. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-DRtaDpgU.woff +0 -0
  20. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-BLI3TTWz.woff +0 -0
  21. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-CJIcxD6K.woff2 +0 -0
  22. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
  23. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
  24. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
  25. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
  26. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-CTOM6hUh.woff2 +0 -0
  27. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-fLZuRloM.woff +0 -0
  28. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
  29. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
  30. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
  31. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
  32. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-9HEixskS.woff +0 -0
  33. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-V-xxqcpd.woff2 +0 -0
  34. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
  35. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
  36. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
  37. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
  38. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-BgSNZQsw.woff2 +0 -0
  39. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-DWFSQ4vo.woff +0 -0
  40. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
  41. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
  42. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
  43. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
  44. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-D38SheWl.woff2 +0 -0
  45. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-DmB0ttJJ.woff +0 -0
  46. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
  47. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
  48. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
  49. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
  50. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-D2EvbN8M.woff2 +0 -0
  51. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-iLQfcSjf.woff +0 -0
  52. raytrace/app/dist/dashboard/assets/index-DepcI8Fi.js +38 -0
  53. raytrace/app/dist/dashboard/assets/index-H97VLD5l.css +1 -0
  54. raytrace/app/dist/dashboard/favicon.svg +6 -0
  55. raytrace/app/dist/dashboard/index.html +15 -0
  56. raytrace/app/package.json +48 -0
  57. raytrace/app/proxy/agent-intent-schema.test.mjs +142 -0
  58. raytrace/app/proxy/call-assessment.mjs +385 -0
  59. raytrace/app/proxy/call-assessment.test.mjs +167 -0
  60. raytrace/app/proxy/call-category.mjs +461 -0
  61. raytrace/app/proxy/capture-health.mjs +30 -0
  62. raytrace/app/proxy/capture-health.test.mjs +44 -0
  63. raytrace/app/proxy/capture.mjs +38 -0
  64. raytrace/app/proxy/capture.test.mjs +49 -0
  65. raytrace/app/proxy/claude-code-adapter.mjs +301 -0
  66. raytrace/app/proxy/claude-code-hook.mjs +22 -0
  67. raytrace/app/proxy/codex-config.mjs +36 -0
  68. raytrace/app/proxy/codex.mjs +14 -0
  69. raytrace/app/proxy/evidence-schema.test.mjs +240 -0
  70. raytrace/app/proxy/evidence-store.mjs +993 -0
  71. raytrace/app/proxy/evidence-store.test.mjs +301 -0
  72. raytrace/app/proxy/exec-narrative.mjs +625 -0
  73. raytrace/app/proxy/exec-narrative.test.mjs +262 -0
  74. raytrace/app/proxy/execution-correlation.mjs +185 -0
  75. raytrace/app/proxy/execution-correlation.test.mjs +146 -0
  76. raytrace/app/proxy/execution-runner.mjs +89 -0
  77. raytrace/app/proxy/execution-runner.test.mjs +38 -0
  78. raytrace/app/proxy/experiment-engine.mjs +195 -0
  79. raytrace/app/proxy/experiment-engine.test.mjs +93 -0
  80. raytrace/app/proxy/explanations.mjs +36 -0
  81. raytrace/app/proxy/explanations.test.mjs +20 -0
  82. raytrace/app/proxy/failure-signature.mjs +56 -0
  83. raytrace/app/proxy/gvisor-forwarder.mjs +493 -0
  84. raytrace/app/proxy/gvisor-forwarder.test.mjs +433 -0
  85. raytrace/app/proxy/init-evidence-db.mjs +227 -0
  86. raytrace/app/proxy/local-sandboxes.mjs +94 -0
  87. raytrace/app/proxy/local-token.mjs +30 -0
  88. raytrace/app/proxy/playground-recovery.test.mjs +130 -0
  89. raytrace/app/proxy/playground.mjs +439 -0
  90. raytrace/app/proxy/prompt-context.mjs +278 -0
  91. raytrace/app/proxy/prompt-traces.mjs +40 -0
  92. raytrace/app/proxy/prompt-traces.test.mjs +32 -0
  93. raytrace/app/proxy/providers.mjs +65 -0
  94. raytrace/app/proxy/providers.test.mjs +76 -0
  95. raytrace/app/proxy/proxy.integration.test.mjs +135 -0
  96. raytrace/app/proxy/raytace-proxy.mjs +941 -0
  97. raytrace/app/proxy/request-metrics.mjs +13 -0
  98. raytrace/app/proxy/request-metrics.test.mjs +19 -0
  99. raytrace/app/proxy/request-policy.mjs +45 -0
  100. raytrace/app/proxy/request-policy.test.mjs +47 -0
  101. raytrace/app/proxy/run-recap.mjs +139 -0
  102. raytrace/app/proxy/runtime-attribution.mjs +110 -0
  103. raytrace/app/proxy/sandbox-bridge.mjs +49 -0
  104. raytrace/app/proxy/schema/EVIDENCE_DATABASE.md +276 -0
  105. raytrace/app/proxy/schema/agent_intent.sql +101 -0
  106. raytrace/app/proxy/schema/auxiliary.sql +90 -0
  107. raytrace/app/proxy/schema/capture_health.sql +14 -0
  108. raytrace/app/proxy/schema/classification.sql +89 -0
  109. raytrace/app/proxy/schema/file_context.sql +53 -0
  110. raytrace/app/proxy/schema/runtime_evidence.sql +201 -0
  111. raytrace/app/proxy/schema/search.sql +66 -0
  112. raytrace/app/proxy/schema/shared_storage.sql +21 -0
  113. raytrace/app/proxy/search.mjs +418 -0
  114. raytrace/app/proxy/search.test.mjs +161 -0
  115. raytrace/app/proxy/smoke.test.mjs +93 -0
  116. raytrace/app/proxy/step-attribution.mjs +380 -0
  117. raytrace/app/proxy/step-lab.mjs +78 -0
  118. raytrace/app/proxy/step-lab.test.mjs +38 -0
  119. raytrace/app/proxy/summaries.mjs +151 -0
  120. raytrace/app/proxy/summaries.test.mjs +69 -0
  121. raytrace/app/proxy/tool-metadata.mjs +111 -0
  122. raytrace/app/proxy/tool-metadata.test.mjs +38 -0
  123. raytrace/app/proxy/trace-view.mjs +324 -0
  124. raytrace/app/proxy/trace-view.test.mjs +92 -0
  125. raytrace/app/proxy/typesafe.mjs +63 -0
  126. raytrace/app/proxy/typesafe.test.mjs +39 -0
  127. raytrace/app/src/agents.mjs +7 -0
  128. raytrace/app/src/api.mjs +57 -0
  129. raytrace/app/src/auth.mjs +205 -0
  130. raytrace/app/src/beta.mjs +96 -0
  131. raytrace/app/src/claude-code.mjs +93 -0
  132. raytrace/app/src/commands.mjs +285 -0
  133. raytrace/app/src/config.mjs +49 -0
  134. raytrace/app/src/export.mjs +200 -0
  135. raytrace/app/src/lima.mjs +176 -0
  136. raytrace/app/src/local-sandbox.mjs +358 -0
  137. raytrace/app/src/mcp.mjs +688 -0
  138. raytrace/app/src/paths.mjs +33 -0
  139. raytrace/app/src/prompt.mjs +42 -0
  140. raytrace/app/src/proxy-process.mjs +143 -0
  141. raytrace/app/src/sandbox.mjs +306 -0
  142. raytrace/app/src/share.mjs +86 -0
  143. raytrace/app/src/terminal.mjs +68 -0
  144. raytrace/app/src/upload.mjs +198 -0
  145. raytrace/app/worker/sandbox/Dockerfile +8 -0
  146. raytrace/app/worker/sandbox/SYSCALL_CATALOG.md +197 -0
  147. raytrace/app/worker/sandbox/claude-hook.mjs +364 -0
  148. raytrace/app/worker/sandbox/collector.py +405 -0
  149. raytrace/app/worker/sandbox/dns_forwarder.py +255 -0
  150. raytrace/app/worker/sandbox/engine.py +940 -0
  151. raytrace/app/worker/sandbox/file_scan.py +491 -0
  152. raytrace/app/worker/sandbox/lima.yaml +29 -0
  153. raytrace/app/worker/sandbox/network.py +90 -0
  154. raytrace/app/worker/sandbox/projects.py +702 -0
  155. raytrace/app/worker/sandbox/setup.py +159 -0
  156. raytrace/app/worker/sandbox/syscall_catalog.sql +985 -0
  157. raytrace/app/worker/sandbox/transcript-delivery.test.mjs +153 -0
  158. raytrace/app/worker/sandbox/viewer.py +195 -0
  159. raytrace/node/LICENSE +2946 -0
  160. raytrace/node/node.exe +4 -0
  161. raytrace_cli-0.11.1.dist-info/LICENSE +21 -0
  162. raytrace_cli-0.11.1.dist-info/METADATA +42 -0
  163. raytrace_cli-0.11.1.dist-info/RECORD +165 -0
  164. raytrace_cli-0.11.1.dist-info/WHEEL +4 -0
  165. raytrace_cli-0.11.1.dist-info/entry_points.txt +2 -0
@@ -0,0 +1,940 @@
1
+ """The sandbox host's fact engine. Python stdlib only; never mount into guests.
2
+
3
+ gVisor reports every open, connect, delete and rename a sandbox makes. That is
4
+ far too much to keep and most of it concerns nobody: one `npm install` opens
5
+ tens of thousands of files. The engine looks at each event once, in memory,
6
+ keeps what matters as a fact, counts the rest and stores no event at all.
7
+
8
+ What matters:
9
+
10
+ files a file the project's owner restricted or watches, and a new file
11
+ whose path marks it sensitive (a .env, a key): read, created,
12
+ changed, emptied, appended, deleted, renamed, or an attempt
13
+ the kernel refused
14
+ network every connection out of the sandbox, one fact for each
15
+ destination a step reached, and whether the firewall refuses it
16
+ packages what a package manager left installed once it had run
17
+ identity a change of user or group id, which a sandbox should never make
18
+
19
+ A fact combines three witnesses, and says which it had (`basis`, `certainty`):
20
+
21
+ gVisor who asked for what, and when (collector.py). For an open it
22
+ gives the path and flags; for a delete or rename only the
23
+ number of the call and whether it was refused.
24
+ the host what the file is before and after: size, time, inode, SHA-256,
25
+ read from the sandbox's volume by this process, not the agent.
26
+ a watcher inotify on the folders of those files, which names the file a
27
+ delete or rename hit; it is matched to gVisor's call by time.
28
+
29
+ confirmed a call was seen and the file shows its effect
30
+ attempted the kernel refused the call
31
+ possible opened for writing, and the file is as it was
32
+ one_of more than one process made such a call at that moment
33
+ unattributed the file changed and no call was seen for it
34
+ reported read from files the agent could have written (packages)
35
+
36
+ Each fact names its process and, when the process carries one, the tool call
37
+ (RAYTRACE_CALL_ID, as the forwarder reads it). Facts go to the `facts` table of
38
+ the collector's database, where viewer.py serves them and the control plane
39
+ (facts.py) collects them. Which calls arrive at all is syscall_catalog.sql's
40
+ `capture` column, through setup.py.
41
+ """
42
+ import collections
43
+ import ctypes
44
+ import hashlib
45
+ import ipaddress
46
+ import json
47
+ import os
48
+ from pathlib import Path, PurePosixPath
49
+ import platform
50
+ import posixpath
51
+ import queue
52
+ import re
53
+ import sqlite3
54
+ import stat
55
+ import struct
56
+ import threading
57
+ import time
58
+
59
+ import file_scan
60
+ import network
61
+
62
+ CATALOG = Path(__file__).with_name('syscall_catalog.sql')
63
+ # Which sandbox a container is, and where its volumes are (projects.py writes one file each).
64
+ CONTAINERS = Path('/var/lib/raytace/containers')
65
+ # Each sandbox's restricted and watched files (projects.py guardrails()).
66
+ GUARDRAILS = Path('/var/lib/raytace/guardrails')
67
+ BRIDGE = Path('/etc/raytace/bridge')
68
+ WORKSPACE = '/workspace/'
69
+ # Large, regenerable folders: the same ones the sandbox hook leaves out of its snapshots.
70
+ SKIPPED = frozenset(['node_modules', '.git', '.next', '.vinext', 'dist', 'build', 'out', 'coverage', '.cache', '.turbo',
71
+ '.venv', 'venv', '__pycache__', '.pytest_cache', '.mypy_cache', 'target', '.gradle', '.wrangler'])
72
+ SCHEMA = '''
73
+ CREATE TABLE IF NOT EXISTS facts (id INTEGER PRIMARY KEY AUTOINCREMENT, fact_id TEXT NOT NULL UNIQUE,
74
+ container_id TEXT NOT NULL, kind TEXT NOT NULL, at_ns TEXT NOT NULL, received_ms INTEGER NOT NULL, body TEXT NOT NULL);
75
+ CREATE INDEX IF NOT EXISTS facts_container ON facts(container_id, id);
76
+ CREATE TABLE IF NOT EXISTS dns_cache (client TEXT NOT NULL, ip TEXT NOT NULL, name TEXT NOT NULL, seen_ns INTEGER NOT NULL,
77
+ PRIMARY KEY (client, ip));
78
+ '''
79
+ AT_FDCWD = -100
80
+ O_ACCMODE, O_CREAT, O_TRUNC, O_APPEND = 3, 0o100, 0o1000, 0o2000
81
+ REFUSED = {1: 'EPERM', 13: 'EACCES', 30: 'EROFS'}
82
+ ERRNO = {**REFUSED, 2: 'ENOENT', 101: 'ENETUNREACH', 110: 'ETIMEDOUT', 111: 'ECONNREFUSED', 113: 'EHOSTUNREACH', 115: 'EINPROGRESS'}
83
+ EINPROGRESS = 115
84
+ MAX_HASHED = 5 * 1024 * 1024 # a larger file is compared by size, time and inode
85
+ HASH_LIMIT = 2000 # with more files than this to keep, none is hashed up front
86
+ SENSITIVE_FROM = 4 # file_scan's level from which a new file's path alone is worth a fact
87
+ SETTLE = 0.3 # seconds between a change seen on disk and looking: gVisor's call may still be on its way
88
+ WRITER_LIFE = 900 # how long an open for writing may explain a later change
89
+ RAW_WINDOW = (2_000_000_000, 1_000_000_000) # ns a delete or rename call may precede, or follow, the watcher's notice
90
+ BULK = 200 # opens from which a step is worth one summary line
91
+ RETAIN_DAYS, RETAIN_ROWS = 14, 50_000
92
+ PACKAGE_TOOLS = frozenset(['npm', 'npx', 'pnpm', 'yarn', 'bun', 'pip', 'pip3', 'uv', 'poetry'])
93
+ REGISTRIES = frozenset(['registry.npmjs.org', 'registry.yarnpkg.com', 'pypi.org', 'files.pythonhosted.org'])
94
+ MAX_PACKAGES = 400
95
+ REJECTED = [ipaddress.ip_network(block) for block in network.REJECTED]
96
+ OWN_KINDS = frozenset(['open', 'connect', 'listen', 'setid'])
97
+ PROCESS_KINDS = frozenset(['exec_succeeded', 'clone', 'process_exit', 'collection_gap', 'collector_disconnected'])
98
+ CALL_MARKER = re.compile(r'(?:^|[\s;])export RAYTRACE_CALL_ID=([A-Za-z0-9_-]{1,128})(?=[\s;]|$)')
99
+ SANDBOX_ID = re.compile(r'rt[pf]-[a-f0-9]{32}')
100
+
101
+
102
+ def open_inside(root, path, directory=False):
103
+ """A descriptor for a file below `root`, every step of the way opened
104
+ without following links: the agent names its own folders, and a link out
105
+ of the project is never read."""
106
+ at = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
107
+ try:
108
+ parts = PurePosixPath(path).parts
109
+ for index, part in enumerate(parts):
110
+ last = index == len(parts) - 1
111
+ below = os.open(part, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | (os.O_DIRECTORY if directory or not last else 0), dir_fd=at)
112
+ os.close(at)
113
+ at = below
114
+ return at
115
+ except BaseException:
116
+ os.close(at)
117
+ raise
118
+
119
+
120
+ def photo(root, path, hashed=True):
121
+ """What a project file is now: None when it is not there (or is no regular file)."""
122
+ try:
123
+ fd = open_inside(root, path)
124
+ except (OSError, ValueError):
125
+ return None
126
+ try:
127
+ info = os.fstat(fd)
128
+ if not stat.S_ISREG(info.st_mode):
129
+ return None
130
+ found = {'size': info.st_size, 'mtime_ns': info.st_mtime_ns, 'inode': info.st_ino, 'mode': stat.S_IMODE(info.st_mode), 'sha256': None}
131
+ if hashed and info.st_size <= MAX_HASHED:
132
+ digest = hashlib.sha256()
133
+ while chunk := os.read(fd, 1024 * 1024):
134
+ digest.update(chunk)
135
+ found['sha256'] = digest.hexdigest()
136
+ return found
137
+ except OSError:
138
+ return None
139
+ finally:
140
+ os.close(fd)
141
+
142
+
143
+ def read_inside(root, path, limit):
144
+ """A file's bytes, or None when it is absent, no regular file or over `limit`."""
145
+ try:
146
+ fd = open_inside(root, path)
147
+ except (OSError, ValueError):
148
+ return None
149
+ try:
150
+ info = os.fstat(fd)
151
+ if not stat.S_ISREG(info.st_mode) or info.st_size > limit:
152
+ return None
153
+ return b''.join(iter(lambda: os.read(fd, 1024 * 1024), b''))
154
+ except OSError:
155
+ return None
156
+ finally:
157
+ os.close(fd)
158
+
159
+
160
+ def names_inside(root, path):
161
+ try:
162
+ fd = open_inside(root, path, directory=True)
163
+ except (OSError, ValueError):
164
+ return []
165
+ try:
166
+ return os.listdir(fd)
167
+ except OSError:
168
+ return []
169
+ finally:
170
+ os.close(fd)
171
+
172
+
173
+ def change(before, after, events, flags):
174
+ """What happened to a file between two photos, or None when nothing did."""
175
+ if after is None:
176
+ return None if before is None else 'renamed' if 'moved_from' in events else 'deleted'
177
+ if before is None:
178
+ return 'created'
179
+ if before['sha256'] and after['sha256']:
180
+ same = before['sha256'] == after['sha256']
181
+ else:
182
+ same = (before['size'], before['mtime_ns'], before['inode']) == (after['size'], after['mtime_ns'], after['inode'])
183
+ if same:
184
+ return 'permissions_changed' if before['mode'] != after['mode'] else None
185
+ if after['size'] == 0:
186
+ return 'emptied'
187
+ if flags & O_APPEND and after['size'] > before['size']:
188
+ return 'appended'
189
+ return 'changed'
190
+
191
+
192
+ def project_path(event):
193
+ """The project file an open names, relative to the project, or None when
194
+ it is outside it. Worked out from the text of the path: links inside the
195
+ project are not followed, which a fact says as `resolution: lexical`."""
196
+ name = event.get('pathname') or ''
197
+ if not name:
198
+ return None
199
+ if not name.startswith('/'):
200
+ base = event.get('cwd') if event.get('dirfd', AT_FDCWD) == AT_FDCWD else event.get('fd_path')
201
+ if not base or not base.startswith('/'):
202
+ return None
203
+ name = base + '/' + name
204
+ name = posixpath.normpath(name)
205
+ return name[len(WORKSPACE):] if name.startswith(WORKSPACE) else None
206
+
207
+
208
+ def installed(workspace, home):
209
+ """{(manager, name, version): where it came from} as the package managers'
210
+ own records on the volume have it. The agent can write those files, so a
211
+ fact drawn from them is `reported`, not seen."""
212
+ found = {}
213
+ roots = [''] + [name for name in names_inside(workspace, '') if name not in SKIPPED and not name.startswith('.')][:50]
214
+ for root in roots:
215
+ modules = posixpath.join(root, 'node_modules')
216
+ lock = read_inside(workspace, posixpath.join(modules, '.package-lock.json'), 32 * 1024 * 1024)
217
+ try:
218
+ listed = json.loads(lock)['packages'] if lock else None
219
+ except (ValueError, KeyError, TypeError):
220
+ listed = None
221
+ if isinstance(listed, dict):
222
+ for key, entry in listed.items():
223
+ if isinstance(key, str) and isinstance(entry, dict) and isinstance(entry.get('version'), str):
224
+ found[('npm', key.rsplit('node_modules/', 1)[-1][:200], entry['version'][:80])] = str(entry.get('resolved') or '')[:300] or None
225
+ continue
226
+ # No npm record (yarn, pnpm, bun): the packages at the top of node_modules.
227
+ for name in names_inside(workspace, modules)[:1000]:
228
+ for package in ([posixpath.join(name, inner) for inner in names_inside(workspace, posixpath.join(modules, name))[:200]]
229
+ if name.startswith('@') else [name]):
230
+ text = read_inside(workspace, posixpath.join(modules, package, 'package.json'), 1024 * 1024)
231
+ try:
232
+ version = json.loads(text)['version'] if text else None
233
+ except (ValueError, KeyError, TypeError):
234
+ version = None
235
+ if isinstance(version, str):
236
+ found[('node', package[:200], version[:80])] = None
237
+ sites = [(workspace, posixpath.join(env, 'lib')) for env in ('.venv', 'venv')] + ([(home, '.local/lib')] if home else [])
238
+ for volume, lib in sites:
239
+ for python in names_inside(volume, lib)[:10]:
240
+ for name in names_inside(volume, posixpath.join(lib, python, 'site-packages')):
241
+ if name.endswith('.dist-info') and '-' in name:
242
+ package, _, version = name[:-len('.dist-info')].rpartition('-')
243
+ found[('pip', package[:200], version[:80])] = None
244
+ return found
245
+
246
+
247
+ class Inotify:
248
+ """The host kernel's file notifications, for the folders of the files
249
+ worth a fact. It names the file a delete or rename hit, which gVisor does not."""
250
+ CLOSE_WRITE, MODIFY, ATTRIB, CREATE, DELETE, MOVED_FROM, MOVED_TO = 0x8, 0x2, 0x4, 0x100, 0x200, 0x40, 0x80
251
+ DELETE_SELF, MOVE_SELF, OVERFLOW, IGNORED, ONLYDIR = 0x400, 0x800, 0x4000, 0x8000, 0x01000000
252
+ WORDS = [(CLOSE_WRITE, 'close_write'), (MODIFY, 'modify'), (ATTRIB, 'attrib'), (CREATE, 'create'), (DELETE, 'delete'),
253
+ (MOVED_FROM, 'moved_from'), (MOVED_TO, 'moved_to')]
254
+ FOLDER = CLOSE_WRITE | MODIFY | ATTRIB | CREATE | DELETE | MOVED_FROM | MOVED_TO | DELETE_SELF | MOVE_SELF | ONLYDIR
255
+
256
+ def __init__(self):
257
+ self.libc = ctypes.CDLL(None, use_errno=True)
258
+ self.fd = self.libc.inotify_init1(os.O_NONBLOCK | os.O_CLOEXEC) # AttributeError where there is no inotify
259
+ if self.fd < 0:
260
+ raise OSError(ctypes.get_errno(), 'inotify_init1')
261
+
262
+ def add(self, root, folder):
263
+ """Watches a folder of a project, reached without following links."""
264
+ fd = open_inside(root, folder, directory=True)
265
+ try:
266
+ watch = self.libc.inotify_add_watch(self.fd, f'/proc/self/fd/{fd}'.encode(), self.FOLDER)
267
+ if watch < 0:
268
+ raise OSError(ctypes.get_errno(), 'inotify_add_watch')
269
+ return watch
270
+ finally:
271
+ os.close(fd)
272
+
273
+ def remove(self, watch):
274
+ self.libc.inotify_rm_watch(self.fd, watch)
275
+
276
+ def read(self):
277
+ try:
278
+ data = os.read(self.fd, 256 * 1024)
279
+ except BlockingIOError:
280
+ return []
281
+ events, at = [], 0
282
+ while at + 16 <= len(data):
283
+ watch, mask, cookie, size = struct.unpack_from('iIII', data, at)
284
+ events.append((watch, mask, cookie, os.fsdecode(data[at + 16:at + 16 + size].split(b'\0', 1)[0])))
285
+ at += 16 + size
286
+ return events
287
+
288
+
289
+ class Sandbox:
290
+ """What the engine holds of one container, all of it in memory."""
291
+ def __init__(self, container):
292
+ self.container = container
293
+ self.sid = self.workspace = self.home = self.ip = None
294
+ self.fork = False
295
+ self.mapped = self.guarded_at = None # the two files' times, as last read
296
+ self.restricted, self.watched, self.sensitive = set(), set(), set()
297
+ self.guarded = {} # path -> restricted | watched | sensitive
298
+ self.hashing = True
299
+ self.photos = {} # path -> its last photo; None: it was not there
300
+ self.known = None # project paths that exist, to tell a new file from an old one
301
+ self.processes = {} # pid -> {start_ns, ppid, name, argv, call_id}
302
+ self.writers = {} # path -> opens for writing that may explain a change
303
+ self.due = {} # path -> when to look at it next (monotonic)
304
+ self.final = {} # path -> when to stop waiting for a change
305
+ self.causes = {} # path -> what the watcher saw of it since the last look
306
+ self.raw = collections.deque(maxlen=512) # recent delete, rename, resize and mode calls: they carry no path
307
+ self.watches = {} # folder -> inotify watch
308
+ self.counts = {} # step -> opens that were only counted
309
+ self.packages = None # what was installed when last looked
310
+ self.package_due = self.package_who = None
311
+ self.registry_at = float('-inf') # when a package registry was last connected to
312
+ self.gap_at = float('-inf') # when events were last lost
313
+ self.names = {} # ip -> (host name, when read)
314
+
315
+
316
+ class Engine(threading.Thread):
317
+ def __init__(self, database, arch=None, watch=True, clock=time.monotonic):
318
+ super().__init__(daemon=True, name='raytace-engine')
319
+ self.database = database
320
+ self.clock = clock
321
+ catalog = sqlite3.connect(':memory:')
322
+ try:
323
+ catalog.executescript(CATALOG.read_text())
324
+ # number -> (name, rule) for the calls the catalog captures on this architecture.
325
+ self.calls = {number: (name, rule) for number, name, rule in catalog.execute(
326
+ "SELECT n.sysno, c.syscall_name, c.effect_rule FROM syscall_catalog AS c JOIN syscall_numbers AS n "
327
+ "ON n.syscall_name = c.syscall_name AND n.arch = ? WHERE c.capture IN ('yes', 'watcher')",
328
+ (arch or {'arm64': 'aarch64'}.get(platform.machine(), platform.machine()),))}
329
+ finally:
330
+ catalog.close()
331
+ # Calls gVisor reports by number only: taken from the collector whole.
332
+ self.numbered = frozenset(number for number, (_, rule) in self.calls.items() if rule in ('remove', 'rename', 'resize', 'mkdir', 'metadata'))
333
+ self.inbox = queue.Queue(maxsize=200_000)
334
+ self.overflowed = float('-inf')
335
+ self.bridge = ('', float('-inf')) # RayTrace's own bridge as sandboxes reach it, and when that was read
336
+ self.boxes = {}
337
+ self.facts = {} # fact id -> the fact, while it may still grow
338
+ self.dirty = {} # fact ids to write
339
+ self.inotify = None
340
+ self.watched = {} # inotify watch -> (sandbox, folder)
341
+ if watch:
342
+ try:
343
+ self.inotify = Inotify()
344
+ except (AttributeError, OSError) as error:
345
+ print(f'RayTrace engine: no file watcher ({error}); deletes and renames will not be named', flush=True)
346
+ self.db = None
347
+ self.ticked = self.flushed = self.swept = 0.0
348
+ self.errors = 0
349
+
350
+ # ------------------------------------------------------------ from the collector
351
+
352
+ def accept(self, event):
353
+ """Called on a sandbox's reader thread for every event: queues it and
354
+ answers whether it is the engine's own (a file or network call), which
355
+ the collector then does not write. Never blocks and never raises."""
356
+ try:
357
+ kind = event.get('kind')
358
+ own = kind in OWN_KINDS or (kind in ('syscall_return', 'syscall_enter') and event.get('sysno') in self.numbered)
359
+ if own or kind in PROCESS_KINDS:
360
+ try:
361
+ self.inbox.put_nowait(event)
362
+ except queue.Full:
363
+ self.overflowed = self.clock() # said on every fact for a minute: coverage has a gap
364
+ return own
365
+ except Exception:
366
+ return False
367
+
368
+ def run(self):
369
+ self.db = sqlite3.connect(self.database, timeout=10)
370
+ self.db.execute('PRAGMA journal_mode=WAL')
371
+ self.db.executescript(SCHEMA)
372
+ while True:
373
+ try:
374
+ busy = self.turn()
375
+ except Exception as error:
376
+ busy = False
377
+ self.errors += 1
378
+ if self.errors <= 20:
379
+ print(f'RayTrace engine: {type(error).__name__}: {error}', flush=True)
380
+ if not busy:
381
+ time.sleep(0.02)
382
+
383
+ def turn(self):
384
+ """One pass: the events waiting, the watcher, then whatever is due."""
385
+ handled = 0
386
+ while handled < 5000:
387
+ try:
388
+ event = self.inbox.get_nowait()
389
+ except queue.Empty:
390
+ break
391
+ handled += 1
392
+ try:
393
+ self.handle(event)
394
+ except Exception as error:
395
+ self.errors += 1
396
+ if self.errors <= 20:
397
+ print(f'RayTrace engine: {event.get("kind")}: {type(error).__name__}: {error}', flush=True)
398
+ seen = self.watcher_events()
399
+ now = self.clock()
400
+ for box in list(self.boxes.values()):
401
+ for path in [path for path, at in box.due.items() if at <= now]:
402
+ del box.due[path]
403
+ self.look(box, path)
404
+ for path in [path for path, at in box.final.items() if at <= now and path not in box.due]:
405
+ del box.final[path]
406
+ self.look(box, path, final=True)
407
+ if box.package_due and box.package_due <= now:
408
+ box.package_due = None
409
+ self.packages(box)
410
+ if now - self.ticked >= 1:
411
+ self.ticked = now
412
+ for box in list(self.boxes.values()):
413
+ self.refresh(box)
414
+ self.summaries(box, now)
415
+ if now - self.flushed >= 1:
416
+ self.flushed = now
417
+ self.flush()
418
+ if now - self.swept >= 3600:
419
+ self.swept = now
420
+ self.sweep()
421
+ return bool(handled or seen)
422
+
423
+ def handle(self, event):
424
+ container = event.get('container_id')
425
+ if not container:
426
+ return
427
+ kind = event['kind']
428
+ if kind == 'collector_disconnected':
429
+ box = self.boxes.pop(container, None)
430
+ if box:
431
+ self.close(box)
432
+ return
433
+ box = self.boxes.get(container)
434
+ if box is None:
435
+ box = self.boxes[container] = Sandbox(container)
436
+ self.refresh(box)
437
+ if kind == 'open':
438
+ self.opened(box, event)
439
+ elif kind in ('connect', 'listen'):
440
+ self.connected(box, event)
441
+ elif kind == 'setid':
442
+ self.identity(box, event)
443
+ elif kind == 'syscall_return':
444
+ self.numbered_call(box, event)
445
+ elif kind == 'exec_succeeded':
446
+ self.executed(box, event)
447
+ elif kind == 'clone':
448
+ child, parent = event.get('child_pid'), box.processes.get(event.get('pid'))
449
+ if child and child != event.get('pid'):
450
+ box.processes[child] = {**(parent or {'name': event.get('process_name'), 'argv': [], 'call_id': None}),
451
+ 'ppid': event.get('pid'), 'start_ns': event.get('child_start_ns')}
452
+ elif kind == 'process_exit':
453
+ self.exited(box, event)
454
+ elif kind == 'collection_gap':
455
+ box.gap_at = self.clock()
456
+
457
+ # ------------------------------------------------------------ sandboxes and their lists
458
+
459
+ def refresh(self, box):
460
+ """Reads again, when either changed, which sandbox the container is and
461
+ which of its files to keep facts of."""
462
+ mapping = CONTAINERS / (box.container + '.json')
463
+ try:
464
+ mapped = mapping.stat().st_mtime_ns
465
+ except OSError:
466
+ mapped = None
467
+ if mapped != box.mapped:
468
+ box.mapped = mapped
469
+ try:
470
+ found = json.loads(mapping.read_text()) if mapped else {}
471
+ except (OSError, ValueError):
472
+ found = {}
473
+ sid = found.get('id') if SANDBOX_ID.fullmatch(str(found.get('id') or '')) else None
474
+ workspace = found.get('workspace') if sid and os.path.isdir(str(found.get('workspace') or '')) else None
475
+ fresh = workspace != box.workspace
476
+ box.sid, box.workspace, box.home, box.ip = sid, workspace, found.get('home') or None, found.get('ip') or None
477
+ box.fork = bool(found.get('fork_of'))
478
+ if fresh:
479
+ box.guarded_at, box.known, box.packages = None, None, None
480
+ if workspace:
481
+ box.known = set()
482
+ for folder, folders, files in os.walk(workspace):
483
+ folders[:] = [name for name in folders if name not in SKIPPED]
484
+ inside = os.path.relpath(folder, workspace)
485
+ box.known.update(name if inside == '.' else inside + '/' + name for name in files)
486
+ if len(box.known) > 200_000:
487
+ break
488
+ box.packages = installed(workspace, box.home)
489
+ if not box.workspace:
490
+ return
491
+ lists = GUARDRAILS / (box.sid + '.json')
492
+ try:
493
+ at = lists.stat().st_mtime_ns
494
+ except OSError:
495
+ at = 0
496
+ if at != box.guarded_at:
497
+ box.guarded_at = at
498
+ try:
499
+ kept = json.loads(lists.read_text()) if at else {}
500
+ except (OSError, ValueError):
501
+ kept = {}
502
+ box.restricted = {path for path in kept.get('restricted') or [] if isinstance(path, str)}
503
+ box.watched = {path for path in kept.get('watched') or [] if isinstance(path, str)}
504
+ self.guard(box)
505
+ self.arm(box)
506
+
507
+ def guard(self, box):
508
+ """The files to keep facts of, each with a first photo to compare later ones with."""
509
+ guarded = {path: 'sensitive' for path in box.sensitive}
510
+ guarded.update({path: 'watched' for path in box.watched})
511
+ guarded.update({path: 'restricted' for path in box.restricted})
512
+ box.hashing = len(guarded) <= HASH_LIMIT
513
+ for path in guarded:
514
+ if path not in box.photos:
515
+ box.photos[path] = photo(box.workspace, path, box.hashing)
516
+ for path in [path for path in box.photos if path not in guarded]:
517
+ del box.photos[path]
518
+ box.guarded = guarded
519
+
520
+ def arm(self, box):
521
+ """Watches the folder of every file to keep facts of; one that is not
522
+ there yet is tried again on the next pass."""
523
+ if not self.inotify:
524
+ return
525
+ wanted = {posixpath.dirname(path) for path in box.guarded}
526
+ for folder in [folder for folder in box.watches if folder not in wanted]:
527
+ self.unwatch(box, folder)
528
+ for folder in wanted - box.watches.keys():
529
+ try:
530
+ watch = self.inotify.add(box.workspace, folder)
531
+ except (OSError, ValueError):
532
+ continue
533
+ box.watches[folder] = watch
534
+ self.watched[watch] = (box, folder)
535
+
536
+ def unwatch(self, box, folder):
537
+ watch = box.watches.pop(folder, None)
538
+ if watch is not None and self.watched.pop(watch, None):
539
+ self.inotify.remove(watch)
540
+
541
+ def close(self, box):
542
+ """A sandbox's stream ended: settle what was pending and let it go."""
543
+ for path in set(box.due) | set(box.final):
544
+ self.look(box, path, final=True)
545
+ self.summaries(box, self.clock(), closing=True)
546
+ for folder in list(box.watches):
547
+ self.unwatch(box, folder)
548
+
549
+ # ------------------------------------------------------------ processes
550
+
551
+ def executed(self, box, event):
552
+ pid, argv = event.get('pid'), [str(part) for part in event.get('argv') or []]
553
+ known = box.processes.get(pid) or {}
554
+ parent = box.processes.get(event.get('ppid')) or {}
555
+ # The call id travels in the environment, and for the shell that runs a
556
+ # command in its own text (claude-hook.mjs): the forwarder reads both the same way.
557
+ marker = (event.get('markers') or {}).get('RAYTRACE_CALL_ID') or (CALL_MARKER.search(' '.join(argv)) or [None, None])[1]
558
+ box.processes[pid] = {'start_ns': event.get('process_start_ns'), 'ppid': event.get('ppid'),
559
+ 'name': posixpath.basename(event.get('executable') or (argv[0] if argv else '')) or event.get('process_name'),
560
+ 'argv': [part[:400] for part in argv[:24]],
561
+ 'call_id': marker or known.get('call_id') or parent.get('call_id')}
562
+ if len(box.processes) > 20_000: # exits that never arrived: forget the oldest half
563
+ for stale in list(box.processes)[:10_000]:
564
+ del box.processes[stale]
565
+
566
+ def exited(self, box, event):
567
+ gone = box.processes.pop(event.get('pid'), None)
568
+ if gone and box.workspace and PACKAGE_TOOLS & {posixpath.basename(part) for part in gone['argv'][:4]}:
569
+ # A package manager finished: see what it left, once it has settled.
570
+ box.package_due = self.clock() + 1
571
+ box.package_who = self.who(box, event, gone)
572
+
573
+ def who(self, box, event, known=None):
574
+ """The process an event is from, and the tool call it belongs to."""
575
+ pid = event.get('pid')
576
+ known = known or box.processes.get(pid) or {}
577
+ if known.get('start_ns') and event.get('process_start_ns') and known['start_ns'] != event['process_start_ns']:
578
+ known = {} # the pid was used again
579
+ parent = box.processes.get(event.get('ppid')) or {}
580
+ return {'pid': pid, 'start_ns': known.get('start_ns') or event.get('process_start_ns'),
581
+ 'name': known.get('name') or event.get('process_name') or None, 'argv': known.get('argv') or [],
582
+ 'call_id': known.get('call_id') or parent.get('call_id')}
583
+
584
+ # ------------------------------------------------------------ facts
585
+
586
+ def fact(self, box, kind, key, who, body, at_ns):
587
+ """The fact these say, begun or added to: one fact for the same thing
588
+ done by the same step, with a count, not one for each time."""
589
+ step = who.get('call_id') or (f"{who.get('pid')}:{who.get('start_ns')}" if who.get('pid') else '')
590
+ fact_id = hashlib.sha256('\0'.join([box.container, kind, step, *map(str, key)]).encode()).hexdigest()[:32]
591
+ fact = self.facts.get(fact_id)
592
+ if fact is None:
593
+ fact = self.facts[fact_id] = {
594
+ 'fact_id': fact_id, 'container_id': box.container, 'kind': kind, 'count': 0, 'first_ns': str(at_ns),
595
+ 'pid': who.get('pid'), 'process_start_ns': who.get('start_ns'), 'process': who.get('name'),
596
+ 'command': ' '.join(who.get('argv') or [])[:160] or None, 'call_id': who.get('call_id'), **body}
597
+ if box.fork:
598
+ fact['fork'] = True
599
+ fact['count'] += 1
600
+ fact['last_ns'] = str(at_ns)
601
+ now = self.clock()
602
+ if now - box.gap_at < 60 or now - self.overflowed < 60:
603
+ fact['coverage'] = 'gaps' # events were lost about now: this may not be all of it
604
+ fact['touched'] = now
605
+ self.dirty[fact_id] = True
606
+ return fact
607
+
608
+ def flush(self):
609
+ if self.dirty:
610
+ now = int(time.time() * 1000)
611
+ rows = []
612
+ for fact_id in self.dirty:
613
+ fact = {key: value for key, value in self.facts[fact_id].items() if key != 'touched'}
614
+ rows.append((fact_id, fact['container_id'], fact['kind'], fact['last_ns'], now, json.dumps(fact)))
615
+ # Replaced, not updated: a fact that grew gets a new row id, so
616
+ # whoever reads by "newer than" sees it again.
617
+ self.db.executemany('INSERT OR REPLACE INTO facts (fact_id, container_id, kind, at_ns, received_ms, body) VALUES (?,?,?,?,?,?)', rows)
618
+ self.db.commit()
619
+ self.dirty.clear()
620
+ idle = self.clock() - 600
621
+ for fact_id in [fact_id for fact_id, fact in self.facts.items() if fact['touched'] < idle]:
622
+ del self.facts[fact_id]
623
+
624
+ def sweep(self):
625
+ """Retention on this host: facts are kept two weeks, and no sandbox keeps more than its newest 50,000."""
626
+ self.db.execute('DELETE FROM facts WHERE received_ms < ?', (int(time.time() * 1000) - RETAIN_DAYS * 86_400_000,))
627
+ for (container,) in self.db.execute('SELECT container_id FROM facts GROUP BY container_id HAVING count(*) > ?', (RETAIN_ROWS,)).fetchall():
628
+ self.db.execute('DELETE FROM facts WHERE container_id = ? AND id NOT IN (SELECT id FROM facts WHERE container_id = ? ORDER BY id DESC LIMIT ?)',
629
+ (container, container, RETAIN_ROWS))
630
+ self.db.execute('DELETE FROM dns_cache WHERE seen_ns < ?', (time.time_ns() - 86_400_000_000_000,))
631
+ self.db.commit()
632
+
633
+ def counted(self, box, who, top, at_ns):
634
+ """An open that is nobody's business: counted for its step, and gone."""
635
+ step = who.get('call_id') or f"{who.get('pid')}:{who.get('start_ns')}"
636
+ count = box.counts.get(step)
637
+ if count is None:
638
+ count = box.counts[step] = {'who': who, 'opens': 0, 'tops': {}, 'first_ns': at_ns, 'written': 0}
639
+ count['opens'] += 1
640
+ count['tops'][top] = count['tops'].get(top, 0) + 1
641
+ count['last_ns'], count['touched'] = at_ns, self.clock()
642
+
643
+ def summaries(self, box, now, closing=False):
644
+ """One line for a step that opened files in bulk: how many, and where."""
645
+ for step, count in list(box.counts.items()):
646
+ idle = now - count['touched']
647
+ if count['opens'] >= BULK and count['opens'] != count['written'] and (closing or idle > 3 or now - count.get('wrote_at', 0) > 15):
648
+ # A tool call's count is of all its processes: it names the call, not whichever opened first.
649
+ who = {'call_id': count['who']['call_id']} if count['who'].get('call_id') else count['who']
650
+ fact = self.fact(box, 'summary', ('opens',), who, {'effect': 'opened'}, count['last_ns'])
651
+ fact.update(count=1, first_ns=str(count['first_ns']), opens=count['opens'],
652
+ folders=dict(sorted(count['tops'].items(), key=lambda item: -item[1])[:5]))
653
+ count['written'], count['wrote_at'] = count['opens'], now
654
+ if closing or idle > 120:
655
+ del box.counts[step]
656
+
657
+ # ------------------------------------------------------------ files
658
+
659
+ def opened(self, box, event):
660
+ at_ns = int(event.get('timestamp_ns') or 0) or time.time_ns()
661
+ who = self.who(box, event)
662
+ path = project_path(event)
663
+ if path is None or not box.workspace:
664
+ return self.counted(box, who, '(outside the project)', at_ns)
665
+ flags, errno = event.get('flags', 0), event.get('errno', 0)
666
+ writing = (flags & O_ACCMODE) != 0 or bool(flags & (O_CREAT | O_TRUNC))
667
+ guard = box.guarded.get(path)
668
+ top = path.split('/', 1)[0] if '/' in path else '(project root)'
669
+ if guard is None:
670
+ # A file that was not there and now is: worth a fact when its path alone marks it (a .env, a key).
671
+ if flags & O_CREAT and not errno and box.known is not None and path not in box.known and not SKIPPED.intersection(path.split('/')):
672
+ box.known.add(path)
673
+ if file_scan.created_entry(path)['level'] >= SENSITIVE_FROM:
674
+ box.sensitive.add(path)
675
+ box.photos[path] = None
676
+ self.guard(box)
677
+ self.arm(box)
678
+ guard = 'sensitive'
679
+ if guard is None:
680
+ return self.counted(box, who, top, at_ns)
681
+ call = self.calls.get(event.get('sysno'), ('open', 'open'))[0]
682
+ basis = {'syscall': call, 'flags': flags, 'resolution': 'lexical'}
683
+ if errno:
684
+ if errno in REFUSED:
685
+ self.fact(box, 'file', (path, 'attempted', 'write' if writing else 'read'), who,
686
+ {'path': path, 'guard': guard, 'effect': 'attempted', 'certainty': 'attempted', 'intent': 'write' if writing else 'read',
687
+ 'basis': {**basis, 'errno': REFUSED[errno]}}, at_ns)
688
+ return
689
+ if not writing:
690
+ self.fact(box, 'file', (path, 'read'), who, {'path': path, 'guard': guard, 'effect': 'read', 'certainty': 'confirmed',
691
+ 'basis': {**basis, 'note': 'opened for reading; the bytes read are not captured'}}, at_ns)
692
+ return
693
+ now = self.clock()
694
+ box.writers.setdefault(path, []).append({'who': who, 'at_ns': at_ns, 'flags': flags, 'syscall': call, 'credited': False, 'seen': now})
695
+ box.due[path] = min(box.due.get(path, now + 1), now + 1)
696
+ box.final[path] = now + 30
697
+
698
+ def numbered_call(self, box, event):
699
+ """A delete, rename, resize, mkdir or mode change: gVisor gives its
700
+ number and result, not its path. Kept a moment, for the watcher to say
701
+ which file; a refusal is a fact at once."""
702
+ name, rule = self.calls.get(event.get('sysno'), (None, None))
703
+ if not name:
704
+ return
705
+ at_ns = int(event.get('timestamp_ns') or 0) or time.time_ns()
706
+ who, errno = self.who(box, event), event.get('errno', 0)
707
+ box.raw.append({'rule': rule, 'syscall': name, 'errno': errno, 'at_ns': at_ns, 'who': who})
708
+ if errno not in REFUSED or rule == 'mkdir':
709
+ return
710
+ # The process's own arguments, as gVisor reported them at exec, may name
711
+ # the file: `rm notes.txt`. Text that only contains the name is weaker.
712
+ named = self.named(box, who, event.get('cwd'))
713
+ basis = {'syscall': name, 'errno': REFUSED[errno], 'note': 'gVisor does not report the path of this call'}
714
+ for path, how in named or [(None, None)]:
715
+ self.fact(box, 'file', (path or '', 'attempted', name), who,
716
+ {'path': path, 'guard': box.guarded.get(path) if path else None, 'effect': 'attempted', 'certainty': 'attempted',
717
+ 'intent': {'remove': 'delete', 'rename': 'rename', 'resize': 'write', 'metadata': 'permissions'}[rule],
718
+ 'basis': {**basis, **({'path_from': how} if how else {})}}, at_ns)
719
+
720
+ def named(self, box, who, cwd):
721
+ """Restricted or watched files a process's arguments name: [(path, how)]."""
722
+ found = {}
723
+ for part in who.get('argv') or []:
724
+ if not part or part.startswith('-'):
725
+ continue
726
+ name = posixpath.normpath(part if part.startswith('/') else (cwd or WORKSPACE) + '/' + part)
727
+ if name + '/' != WORKSPACE and not name.startswith(WORKSPACE):
728
+ continue
729
+ at = name[len(WORKSPACE):] # empty for the project folder itself: `rm -r .` names everything
730
+ if at in box.guarded:
731
+ found[at] = 'argv'
732
+ for path in box.guarded: # `rm -r src` reaches every file under src
733
+ if not at or path.startswith(at + '/'):
734
+ found.setdefault(path, 'argv_folder')
735
+ if not found:
736
+ text = ' '.join(who.get('argv') or [])
737
+ found = {path: 'argv_text' for path in box.guarded if path in text}
738
+ return sorted(found.items())[:5]
739
+
740
+ def actors(self, box, rule, at_ns, path):
741
+ """The processes whose delete, rename, resize or mode call succeeded
742
+ about when the watcher saw `path` change, likeliest first."""
743
+ found = {}
744
+ for call in box.raw:
745
+ if call['rule'] == rule and not call['errno'] and at_ns - RAW_WINDOW[0] <= call['at_ns'] <= at_ns + RAW_WINDOW[1]:
746
+ found.setdefault((call['who']['pid'], call['who']['start_ns']), call)
747
+ names = lambda call: any(posixpath.basename(path) == posixpath.basename(part) for part in call['who']['argv'])
748
+ ranked = sorted(found.values(), key=lambda call: (not names(call), abs(call['at_ns'] - at_ns)))
749
+ own = [call for call in ranked if names(call)]
750
+ return [(call['who'], call['syscall']) for call in (own if len(own) == 1 else ranked)[:5]]
751
+
752
+ def watcher_events(self):
753
+ if not self.inotify:
754
+ return 0
755
+ events = self.inotify.read()
756
+ moved = {cookie: posixpath.join(self.watched[watch][1], name) for watch, mask, cookie, name in events
757
+ if mask & Inotify.MOVED_TO and watch in self.watched}
758
+ for watch, mask, cookie, name in events:
759
+ if mask & Inotify.OVERFLOW: # the kernel dropped notices: look at everything again
760
+ for box in self.boxes.values():
761
+ box.gap_at = self.clock()
762
+ for path in box.guarded:
763
+ self.noticed(box, path, [])
764
+ continue
765
+ target = self.watched.get(watch)
766
+ if not target:
767
+ continue
768
+ box, folder = target
769
+ if mask & (Inotify.IGNORED | Inotify.DELETE_SELF | Inotify.MOVE_SELF):
770
+ # The folder itself went: its files went with it, and nothing named them.
771
+ if mask & Inotify.IGNORED:
772
+ box.watches.pop(folder, None)
773
+ self.watched.pop(watch, None)
774
+ for path in box.guarded:
775
+ if posixpath.dirname(path) == folder:
776
+ self.noticed(box, path, [])
777
+ continue
778
+ path = posixpath.join(folder, name)
779
+ if path in box.guarded:
780
+ self.noticed(box, path, [word for bit, word in Inotify.WORDS if mask & bit],
781
+ moved.get(cookie) if mask & Inotify.MOVED_FROM else None)
782
+ return len(events)
783
+
784
+ def noticed(self, box, path, words, moved_to=None):
785
+ """The watcher saw something of a file: look at it once things settle."""
786
+ cause = box.causes.setdefault(path, {'events': set(), 'at_ns': time.time_ns()})
787
+ cause['events'].update(words)
788
+ if moved_to:
789
+ cause['to'] = moved_to
790
+ now = self.clock()
791
+ box.due[path] = min(box.due.get(path, now + SETTLE), now + SETTLE)
792
+
793
+ def look(self, box, path, final=False):
794
+ """Compares a file with its last photo and says what happened to it, and who did it."""
795
+ guard = box.guarded.get(path)
796
+ if guard is None or not box.workspace:
797
+ box.writers.pop(path, None)
798
+ return
799
+ cause = box.causes.pop(path, None) or {'events': set(), 'at_ns': time.time_ns()}
800
+ now = self.clock()
801
+ writers = box.writers[path] = [writer for writer in box.writers.get(path, []) if now - writer['seen'] < WRITER_LIFE]
802
+ before = box.photos.get(path)
803
+ after = photo(box.workspace, path, box.hashing)
804
+ flags = 0
805
+ for writer in writers:
806
+ flags |= writer['flags']
807
+ effect = change(before, after, cause['events'], flags)
808
+ if effect:
809
+ box.photos[path] = after
810
+ if after is None and box.known is not None:
811
+ box.known.discard(path)
812
+ if effect in ('deleted', 'renamed'):
813
+ found = self.actors(box, 'remove' if effect == 'deleted' else 'rename', cause['at_ns'], path)
814
+ elif effect == 'permissions_changed':
815
+ found = self.actors(box, 'metadata', cause['at_ns'], path)
816
+ else:
817
+ found, seen = [], set()
818
+ for writer in reversed(writers): # the latest to open it first
819
+ key = (writer['who']['pid'], writer['who']['start_ns'])
820
+ if key not in seen:
821
+ seen.add(key)
822
+ found.append((writer['who'], writer['syscall']))
823
+ writer['credited'] = True
824
+ if 'moved_to' in cause['events']: # written elsewhere and moved over it, as editors and `sed -i` do
825
+ found += self.actors(box, 'rename', cause['at_ns'], path)
826
+ if not found:
827
+ found = self.actors(box, 'resize', cause['at_ns'], path)
828
+ who, call = found[0] if found else ({}, None)
829
+ certainty = 'unattributed' if not found else 'confirmed' if len(found) == 1 else 'one_of'
830
+ basis = {'syscall': call, 'watcher': sorted(cause['events']), 'before': before, 'after': after, 'resolution': 'lexical'}
831
+ if len(found) > 1:
832
+ basis['candidates'] = [{'pid': other['pid'], 'process': other['name']} for other, _ in found[:5]]
833
+ fact = self.fact(box, 'file', (path, effect, certainty), who,
834
+ {'path': path, 'guard': guard, 'effect': effect, 'certainty': certainty, 'basis': basis}, cause['at_ns'])
835
+ fact['basis']['after'] = after
836
+ if cause.get('to'):
837
+ fact['to'] = cause['to']
838
+ elif final:
839
+ for writer in writers:
840
+ if not writer['credited']:
841
+ writer['credited'] = True
842
+ self.fact(box, 'file', (path, 'opened_for_writing'), writer['who'],
843
+ {'path': path, 'guard': guard, 'effect': 'opened_for_writing', 'certainty': 'possible',
844
+ 'basis': {'syscall': writer['syscall'], 'flags': writer['flags'], 'before': before, 'after': after,
845
+ 'resolution': 'lexical', 'note': 'opened for writing; the file is as it was'}}, writer['at_ns'])
846
+ if not writers:
847
+ box.writers.pop(path, None)
848
+
849
+ # ------------------------------------------------------------ network, identity, packages
850
+
851
+ def host_name(self, box, ip):
852
+ """The name last looked up, from this sandbox, that answered with this
853
+ address (dns_forwarder.py). A best match: an address can serve many names."""
854
+ kept = box.names.get(ip)
855
+ now = self.clock()
856
+ if kept and now - kept[1] < 30:
857
+ return kept[0]
858
+ try:
859
+ row = self.db.execute('SELECT name FROM dns_cache WHERE client = ? AND ip = ?', (box.ip or '', ip)).fetchone()
860
+ except sqlite3.Error:
861
+ row = None
862
+ box.names[ip] = (row[0] if row else None, now)
863
+ if len(box.names) > 5000:
864
+ box.names.clear()
865
+ return row[0] if row else None
866
+
867
+ def connected(self, box, event):
868
+ if event.get('family') != 'inet':
869
+ return # a socket file inside the sandbox, or a family that leaves no machine
870
+ ip, port, errno = event['ip'], event['port'], event.get('errno', 0)
871
+ there = ipaddress.ip_address(ip)
872
+ at_ns = int(event.get('timestamp_ns') or 0) or time.time_ns()
873
+ listening = event['kind'] == 'listen'
874
+ if not listening and (there.is_loopback or there.is_unspecified):
875
+ return # the sandbox talking to itself
876
+ if not listening and port == 0:
877
+ # No traffic: the resolver library "connects" a datagram socket to each
878
+ # address a name answered with, port 0, to choose which to use.
879
+ return
880
+ now = self.clock()
881
+ if now - self.bridge[1] > 30:
882
+ try:
883
+ self.bridge = (BRIDGE.read_text().strip().removeprefix('http://'), now)
884
+ except OSError:
885
+ self.bridge = ('', now)
886
+ if f'{ip}:{port}' in (self.bridge[0], f'{network.GATEWAY}:53'):
887
+ return # RayTrace's own: the model and transcript bridge, and the resolver (dns_forwarder.py records lookups)
888
+ who = self.who(box, event)
889
+ if listening:
890
+ outcome = 'failed' if errno else 'listening'
891
+ elif there.version == 4 and any(there in block for block in REJECTED):
892
+ outcome = 'refused_by_policy' # the firewall rejects it, whatever a non-blocking call returned
893
+ else:
894
+ outcome = 'connected' if not errno else 'attempted' if errno == EINPROGRESS else 'failed'
895
+ name = None if listening else self.host_name(box, ip)
896
+ # One fact for a site, not one for each address its name answers with:
897
+ # a registry has a dozen, and a program tries IPv6 before IPv4.
898
+ fact = self.fact(box, 'network', (name or ip, port, event['kind']), who,
899
+ {'ip': ip, 'port': port, 'host': name, 'addresses': [], 'effect': event['kind'], 'outcomes': {},
900
+ 'basis': {'syscall': event['kind'], 'note': 'the address is what the program asked for; the name is the lookup that answered with it'}}, at_ns)
901
+ fact['outcomes'][outcome] = fact['outcomes'].get(outcome, 0) + 1
902
+ if ip not in fact['addresses'] and len(fact['addresses']) < 32:
903
+ fact['addresses'].append(ip)
904
+ if outcome == 'connected':
905
+ fact['ip'] = ip # shown first: an address that answered
906
+ if errno and outcome == 'failed':
907
+ fact['errno'] = ERRNO.get(errno, str(errno))
908
+ if name in REGISTRIES:
909
+ box.registry_at = self.clock()
910
+
911
+ def identity(self, box, event):
912
+ errno = event.get('errno', 0)
913
+ call = self.calls.get(event.get('sysno'), ('setid', None))[0]
914
+ at_ns = int(event.get('timestamp_ns') or 0) or time.time_ns()
915
+ self.fact(box, 'identity', (call, *event.get('ids', []), bool(errno)), self.who(box, event),
916
+ {'effect': 'attempted' if errno else 'changed', 'certainty': 'attempted' if errno else 'confirmed',
917
+ 'ids': event.get('ids', []), 'basis': {'syscall': call, **({'errno': ERRNO.get(errno, str(errno))} if errno else {})}}, at_ns)
918
+
919
+ def packages(self, box):
920
+ """What a package manager left that was not there before."""
921
+ if not box.workspace:
922
+ return
923
+ now = installed(box.workspace, box.home)
924
+ before, box.packages = box.packages or {}, now
925
+ fresh = sorted(key for key in now if key not in before)
926
+ if not fresh:
927
+ return
928
+ who = box.package_who or {}
929
+ # Seen reaching a registry about then: the list is the manager's own word, with that beside it.
930
+ certainty = 'corroborated' if self.clock() - box.registry_at < 300 else 'reported'
931
+ for manager in sorted({key[0] for key in fresh}):
932
+ mine = [key for key in fresh if key[0] == manager]
933
+ fact = self.fact(box, 'package', (manager,), who, {'manager': manager, 'packages': [], 'total': 0, 'effect': 'installed',
934
+ 'basis': {'note': "read from the package manager's own records in the sandbox after it ran; the agent can write those files"}},
935
+ time.time_ns())
936
+ fact['certainty'] = certainty
937
+ fact['total'] += len(mine)
938
+ room = MAX_PACKAGES - len(fact['packages'])
939
+ fact['packages'] += [{'name': name, 'version': version, **({'source': now[(manager, name, version)]} if now[(manager, name, version)] else {})}
940
+ for _, name, version in mine[:max(0, room)]]