raytrace-cli 0.11.1__py3-none-win_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. raytrace/__init__.py +1 -0
  2. raytrace/__main__.py +33 -0
  3. raytrace/app/LICENSE +21 -0
  4. raytrace/app/README.md +599 -0
  5. raytrace/app/bin/raytrace.mjs +297 -0
  6. raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-BwCSEQnW.woff +0 -0
  7. raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-CW0RaeGs.woff2 +0 -0
  8. raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-B9HHJjqV.woff2 +0 -0
  9. raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-Dr3UlScf.woff +0 -0
  10. raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-Aqo67rzb.woff2 +0 -0
  11. raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-BmdmIIQ2.woff +0 -0
  12. raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-CUSSCpQX.woff +0 -0
  13. raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-DvUfVpUG.woff2 +0 -0
  14. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BjWJ59Pq.woff +0 -0
  15. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BtiwyxMk.woff2 +0 -0
  16. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-BJfUCQsA.woff2 +0 -0
  17. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-DR84L5F-.woff +0 -0
  18. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-4vooXBpG.woff2 +0 -0
  19. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-DRtaDpgU.woff +0 -0
  20. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-BLI3TTWz.woff +0 -0
  21. raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-CJIcxD6K.woff2 +0 -0
  22. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
  23. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
  24. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
  25. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
  26. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-CTOM6hUh.woff2 +0 -0
  27. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-fLZuRloM.woff +0 -0
  28. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
  29. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
  30. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
  31. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
  32. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-9HEixskS.woff +0 -0
  33. raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-V-xxqcpd.woff2 +0 -0
  34. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
  35. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
  36. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
  37. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
  38. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-BgSNZQsw.woff2 +0 -0
  39. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-DWFSQ4vo.woff +0 -0
  40. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
  41. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
  42. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
  43. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
  44. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-D38SheWl.woff2 +0 -0
  45. raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-DmB0ttJJ.woff +0 -0
  46. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
  47. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
  48. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
  49. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
  50. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-D2EvbN8M.woff2 +0 -0
  51. raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-iLQfcSjf.woff +0 -0
  52. raytrace/app/dist/dashboard/assets/index-DepcI8Fi.js +38 -0
  53. raytrace/app/dist/dashboard/assets/index-H97VLD5l.css +1 -0
  54. raytrace/app/dist/dashboard/favicon.svg +6 -0
  55. raytrace/app/dist/dashboard/index.html +15 -0
  56. raytrace/app/package.json +48 -0
  57. raytrace/app/proxy/agent-intent-schema.test.mjs +142 -0
  58. raytrace/app/proxy/call-assessment.mjs +385 -0
  59. raytrace/app/proxy/call-assessment.test.mjs +167 -0
  60. raytrace/app/proxy/call-category.mjs +461 -0
  61. raytrace/app/proxy/capture-health.mjs +30 -0
  62. raytrace/app/proxy/capture-health.test.mjs +44 -0
  63. raytrace/app/proxy/capture.mjs +38 -0
  64. raytrace/app/proxy/capture.test.mjs +49 -0
  65. raytrace/app/proxy/claude-code-adapter.mjs +301 -0
  66. raytrace/app/proxy/claude-code-hook.mjs +22 -0
  67. raytrace/app/proxy/codex-config.mjs +36 -0
  68. raytrace/app/proxy/codex.mjs +14 -0
  69. raytrace/app/proxy/evidence-schema.test.mjs +240 -0
  70. raytrace/app/proxy/evidence-store.mjs +993 -0
  71. raytrace/app/proxy/evidence-store.test.mjs +301 -0
  72. raytrace/app/proxy/exec-narrative.mjs +625 -0
  73. raytrace/app/proxy/exec-narrative.test.mjs +262 -0
  74. raytrace/app/proxy/execution-correlation.mjs +185 -0
  75. raytrace/app/proxy/execution-correlation.test.mjs +146 -0
  76. raytrace/app/proxy/execution-runner.mjs +89 -0
  77. raytrace/app/proxy/execution-runner.test.mjs +38 -0
  78. raytrace/app/proxy/experiment-engine.mjs +195 -0
  79. raytrace/app/proxy/experiment-engine.test.mjs +93 -0
  80. raytrace/app/proxy/explanations.mjs +36 -0
  81. raytrace/app/proxy/explanations.test.mjs +20 -0
  82. raytrace/app/proxy/failure-signature.mjs +56 -0
  83. raytrace/app/proxy/gvisor-forwarder.mjs +493 -0
  84. raytrace/app/proxy/gvisor-forwarder.test.mjs +433 -0
  85. raytrace/app/proxy/init-evidence-db.mjs +227 -0
  86. raytrace/app/proxy/local-sandboxes.mjs +94 -0
  87. raytrace/app/proxy/local-token.mjs +30 -0
  88. raytrace/app/proxy/playground-recovery.test.mjs +130 -0
  89. raytrace/app/proxy/playground.mjs +439 -0
  90. raytrace/app/proxy/prompt-context.mjs +278 -0
  91. raytrace/app/proxy/prompt-traces.mjs +40 -0
  92. raytrace/app/proxy/prompt-traces.test.mjs +32 -0
  93. raytrace/app/proxy/providers.mjs +65 -0
  94. raytrace/app/proxy/providers.test.mjs +76 -0
  95. raytrace/app/proxy/proxy.integration.test.mjs +135 -0
  96. raytrace/app/proxy/raytace-proxy.mjs +941 -0
  97. raytrace/app/proxy/request-metrics.mjs +13 -0
  98. raytrace/app/proxy/request-metrics.test.mjs +19 -0
  99. raytrace/app/proxy/request-policy.mjs +45 -0
  100. raytrace/app/proxy/request-policy.test.mjs +47 -0
  101. raytrace/app/proxy/run-recap.mjs +139 -0
  102. raytrace/app/proxy/runtime-attribution.mjs +110 -0
  103. raytrace/app/proxy/sandbox-bridge.mjs +49 -0
  104. raytrace/app/proxy/schema/EVIDENCE_DATABASE.md +276 -0
  105. raytrace/app/proxy/schema/agent_intent.sql +101 -0
  106. raytrace/app/proxy/schema/auxiliary.sql +90 -0
  107. raytrace/app/proxy/schema/capture_health.sql +14 -0
  108. raytrace/app/proxy/schema/classification.sql +89 -0
  109. raytrace/app/proxy/schema/file_context.sql +53 -0
  110. raytrace/app/proxy/schema/runtime_evidence.sql +201 -0
  111. raytrace/app/proxy/schema/search.sql +66 -0
  112. raytrace/app/proxy/schema/shared_storage.sql +21 -0
  113. raytrace/app/proxy/search.mjs +418 -0
  114. raytrace/app/proxy/search.test.mjs +161 -0
  115. raytrace/app/proxy/smoke.test.mjs +93 -0
  116. raytrace/app/proxy/step-attribution.mjs +380 -0
  117. raytrace/app/proxy/step-lab.mjs +78 -0
  118. raytrace/app/proxy/step-lab.test.mjs +38 -0
  119. raytrace/app/proxy/summaries.mjs +151 -0
  120. raytrace/app/proxy/summaries.test.mjs +69 -0
  121. raytrace/app/proxy/tool-metadata.mjs +111 -0
  122. raytrace/app/proxy/tool-metadata.test.mjs +38 -0
  123. raytrace/app/proxy/trace-view.mjs +324 -0
  124. raytrace/app/proxy/trace-view.test.mjs +92 -0
  125. raytrace/app/proxy/typesafe.mjs +63 -0
  126. raytrace/app/proxy/typesafe.test.mjs +39 -0
  127. raytrace/app/src/agents.mjs +7 -0
  128. raytrace/app/src/api.mjs +57 -0
  129. raytrace/app/src/auth.mjs +205 -0
  130. raytrace/app/src/beta.mjs +96 -0
  131. raytrace/app/src/claude-code.mjs +93 -0
  132. raytrace/app/src/commands.mjs +285 -0
  133. raytrace/app/src/config.mjs +49 -0
  134. raytrace/app/src/export.mjs +200 -0
  135. raytrace/app/src/lima.mjs +176 -0
  136. raytrace/app/src/local-sandbox.mjs +358 -0
  137. raytrace/app/src/mcp.mjs +688 -0
  138. raytrace/app/src/paths.mjs +33 -0
  139. raytrace/app/src/prompt.mjs +42 -0
  140. raytrace/app/src/proxy-process.mjs +143 -0
  141. raytrace/app/src/sandbox.mjs +306 -0
  142. raytrace/app/src/share.mjs +86 -0
  143. raytrace/app/src/terminal.mjs +68 -0
  144. raytrace/app/src/upload.mjs +198 -0
  145. raytrace/app/worker/sandbox/Dockerfile +8 -0
  146. raytrace/app/worker/sandbox/SYSCALL_CATALOG.md +197 -0
  147. raytrace/app/worker/sandbox/claude-hook.mjs +364 -0
  148. raytrace/app/worker/sandbox/collector.py +405 -0
  149. raytrace/app/worker/sandbox/dns_forwarder.py +255 -0
  150. raytrace/app/worker/sandbox/engine.py +940 -0
  151. raytrace/app/worker/sandbox/file_scan.py +491 -0
  152. raytrace/app/worker/sandbox/lima.yaml +29 -0
  153. raytrace/app/worker/sandbox/network.py +90 -0
  154. raytrace/app/worker/sandbox/projects.py +702 -0
  155. raytrace/app/worker/sandbox/setup.py +159 -0
  156. raytrace/app/worker/sandbox/syscall_catalog.sql +985 -0
  157. raytrace/app/worker/sandbox/transcript-delivery.test.mjs +153 -0
  158. raytrace/app/worker/sandbox/viewer.py +195 -0
  159. raytrace/node/LICENSE +2946 -0
  160. raytrace/node/node.exe +4 -0
  161. raytrace_cli-0.11.1.dist-info/LICENSE +21 -0
  162. raytrace_cli-0.11.1.dist-info/METADATA +42 -0
  163. raytrace_cli-0.11.1.dist-info/RECORD +165 -0
  164. raytrace_cli-0.11.1.dist-info/WHEEL +4 -0
  165. raytrace_cli-0.11.1.dist-info/entry_points.txt +2 -0
@@ -0,0 +1,405 @@
1
+ """External gVisor SecCheck v1 collector. Python stdlib only; never mount into guests.
2
+
3
+ Wire/schema references: google/gvisor pkg/sentry/seccheck/{points,sinks/remote}.
4
+ Only selected fields are decoded; unknown event payloads are retained as base64.
5
+ The bounded protobuf reader handles wire types, not arbitrary recursive schemas.
6
+
7
+ Two kinds of event leave here. Process events (exec, clone, exit) are written
8
+ to runtime_events as they always were, now a batch at a time. File and network
9
+ calls are far too many to keep: each goes to the engine (engine.py), which
10
+ keeps what matters of it as a fact and counts the rest, and is never written.
11
+ """
12
+ import argparse
13
+ import base64
14
+ import ipaddress
15
+ import json
16
+ import os
17
+ import queue
18
+ import socket
19
+ import sqlite3
20
+ import struct
21
+ import threading
22
+ import time
23
+ import uuid
24
+
25
+ try:
26
+ import engine as facts_engine
27
+ except Exception as problem: # a host without the engine still records processes
28
+ facts_engine = None
29
+ print(f'RayTrace collector: no fact engine ({problem}); recording processes only', flush=True)
30
+
31
+ MAX_PACKET = 1024 * 1024
32
+
33
+ # The ONLY environment variables ever read off a traced process. `envv` is
34
+ # requested on the execve points (see setup.py) purely to carry these two
35
+ # markers, which turn tool-call correlation from a text-similarity guess into
36
+ # an exact join. Everything else in the environment block -- API keys, tokens,
37
+ # database passwords -- is dropped here, before any write, and the raw packet
38
+ # is withheld for events that carried one (see decode). Never widen this list.
39
+ MARKER_KEYS = ('RAYTRACE_CALL_ID', 'RAYTRACE_SESSION')
40
+
41
+
42
+ def markers(entries):
43
+ """Pick the allowlisted markers out of a raw `envv` list. Returns {} when
44
+ none are present, and never retains a variable outside MARKER_KEYS."""
45
+ found = {}
46
+ for entry in entries:
47
+ name, separator, value = string(entry).partition('=')
48
+ if separator and name in MARKER_KEYS and len(value) <= 256:
49
+ found[name] = value
50
+ return found
51
+
52
+
53
+ def fields(data):
54
+ result = {}
55
+ pos = 0
56
+
57
+ def varint():
58
+ nonlocal pos
59
+ value = 0
60
+ for shift in range(0, 70, 7):
61
+ if pos >= len(data):
62
+ raise ValueError('truncated varint')
63
+ byte = data[pos]
64
+ pos += 1
65
+ if shift == 63 and byte > 1:
66
+ raise ValueError('varint overflow')
67
+ value |= (byte & 127) << shift
68
+ if byte < 128:
69
+ return value
70
+ raise ValueError('invalid varint')
71
+
72
+ while pos < len(data):
73
+ tag = varint()
74
+ number, wire = tag >> 3, tag & 7
75
+ if not number:
76
+ raise ValueError('invalid field zero')
77
+ if wire == 0:
78
+ value = varint()
79
+ elif wire in (1, 2, 5):
80
+ size = varint() if wire == 2 else (8 if wire == 1 else 4)
81
+ if pos + size > len(data):
82
+ raise ValueError('truncated field')
83
+ value = data[pos:pos + size]
84
+ pos += size
85
+ else:
86
+ raise ValueError('unsupported wire type')
87
+ result.setdefault(number, []).append(value)
88
+ return result
89
+
90
+
91
+ def one(data, key, default=0):
92
+ return data.get(key, [default])[-1]
93
+
94
+
95
+ def string(value):
96
+ if not isinstance(value, bytes):
97
+ raise ValueError('expected string field')
98
+ return value.decode('utf-8', errors='replace')
99
+
100
+
101
+ def signed(value):
102
+ return value - (1 << 64) if value >= (1 << 63) else value
103
+
104
+
105
+ def result_of(data):
106
+ """A syscall's result on its exit point; nothing on an enter point. gVisor
107
+ reports a failure in `errno` and leaves the result zero, so errno alone
108
+ says whether a call succeeded."""
109
+ if 2 not in data:
110
+ return {}
111
+ status = fields(one(data, 2, b''))
112
+ return {'result': signed(one(status, 1)), 'errno': abs(signed(one(status, 2)))}
113
+
114
+
115
+ def address(raw):
116
+ """A socket address as the program passed it (Linux's struct sockaddr): an
117
+ IP address and port, never a host name."""
118
+ family = struct.unpack_from('<H', raw)[0] if len(raw) >= 2 else None
119
+ if family == 2 and len(raw) >= 8:
120
+ return {'family': 'inet', 'ip': str(ipaddress.IPv4Address(raw[4:8])), 'port': struct.unpack_from('>H', raw, 2)[0]}
121
+ if family == 10 and len(raw) >= 24:
122
+ found = ipaddress.IPv6Address(raw[8:24])
123
+ return {'family': 'inet', 'ip': str(found.ipv4_mapped or found), 'port': struct.unpack_from('>H', raw, 2)[0]}
124
+ if family == 1:
125
+ return {'family': 'unix', 'path': raw[2:].split(b'\0')[0].decode('utf-8', errors='replace')[:200]}
126
+ return {'family': 'other' if family is not None else 'unknown'}
127
+
128
+
129
+ def decode(packet):
130
+ if len(packet) < 8 or len(packet) > MAX_PACKET:
131
+ raise ValueError('invalid packet size')
132
+ size, kind, dropped = struct.unpack_from('<HHI', packet)
133
+ if size < 8 or size > len(packet):
134
+ raise ValueError('invalid header size')
135
+ data = fields(packet[size:])
136
+ context = fields(one(data, 1, b''))
137
+ carried_env = False
138
+ event = {
139
+ 'source': 'gvisor_seccheck', 'message_type': kind,
140
+ 'dropped_count': dropped, 'timestamp_ns': str(one(context, 1)),
141
+ 'container_id': string(one(context, 6, b'')),
142
+ 'pid': one(context, 4), 'tid': one(context, 2),
143
+ 'process_start_ns': str(one(context, 5)),
144
+ 'ppid': one(context, 10), 'cwd': string(one(context, 8, b'')),
145
+ 'process_name': string(one(context, 9, b'')),
146
+ 'kind': 'runtime_event',
147
+ }
148
+ if kind == 3:
149
+ event.update(kind='exec_succeeded', executable=string(one(data, 2, b'')),
150
+ argv=[string(v) for v in data.get(3, [])])
151
+ # Field 4 is `envv` on sentry/execve. Unlike the syscall points this is
152
+ # NOT opt-in -- the Sentry has always sent it, so every raw packet kept
153
+ # here carried the process's entire environment (keys, tokens,
154
+ # passwords) into the database. Same treatment as the syscall points:
155
+ # keep the allowlisted markers, withhold the raw bytes.
156
+ if 4 in data:
157
+ carried_env = True
158
+ found = markers(data[4])
159
+ if found:
160
+ event['markers'] = found
161
+ elif kind == 11:
162
+ event.update(kind='exec_attempt', executable=string(one(data, 6, b'')),
163
+ argv=[string(v) for v in data.get(7, [])])
164
+ # Field 8 is `envv`, requested only for the markers (see MARKER_KEYS).
165
+ # `carried_env` records that an environment block was present so the
166
+ # raw packet can be withheld below -- the markers themselves are safe
167
+ # to keep, the block they arrived in is not.
168
+ if 8 in data:
169
+ carried_env = True
170
+ found = markers(data[8])
171
+ if found:
172
+ event['markers'] = found
173
+ if 2 in data:
174
+ status = fields(one(data, 2))
175
+ errno = one(status, 2)
176
+ event.update(kind='exec_failed' if errno else 'exec_syscall_return',
177
+ errno=errno, result=signed(one(status, 1)))
178
+ elif kind == 2:
179
+ event.update(kind='clone', child_pid=one(data, 4), child_tid=one(data, 3),
180
+ child_start_ns=str(one(data, 5)))
181
+ elif kind in (4, 5):
182
+ status = one(data, 2)
183
+ event.update(kind='process_exit' if kind == 4 else 'thread_exit',
184
+ wait_status=status,
185
+ exit_code=(status >> 8) & 255 if status & 127 == 0 else None,
186
+ signal=status & 127 if status & 127 else None)
187
+ elif kind == 6:
188
+ status = fields(one(data, 2, b''))
189
+ # A call gVisor has no typed point for: its number and six integers.
190
+ # A path argument is a pointer into the program's memory, so the
191
+ # integers name no file.
192
+ event.update(kind='syscall_return' if 2 in data else 'syscall_enter',
193
+ sysno=one(data, 4), result=signed(one(status, 1)), errno=one(status, 2),
194
+ args=[one(data, number) for number in range(5, 11)])
195
+ elif kind == 7:
196
+ # open, openat and creat. `dirfd` is what a relative path is relative
197
+ # to (-100: the working directory), `fd_path` that folder's path.
198
+ event.update(kind='open', sysno=one(data, 3), dirfd=signed(one(data, 4)), fd_path=string(one(data, 5, b'')),
199
+ pathname=string(one(data, 6, b'')), flags=one(data, 7), mode=one(data, 8), **result_of(data))
200
+ elif kind == 10:
201
+ event.update(kind='connect', sysno=one(data, 3), **address(one(data, 6, b'')), **result_of(data))
202
+ elif kind == 37:
203
+ event.update(kind='listen', sysno=one(data, 3), backlog=one(data, 6), **address(one(data, 7, b'')), **result_of(data))
204
+ elif kind in (14, 15):
205
+ # setuid/setgid carry one id; setresuid/setresgid the real, effective and saved ones.
206
+ event.update(kind='setid', sysno=one(data, 3), ids=[one(data, number) for number in ((4,) if kind == 14 else (4, 5, 6))],
207
+ **result_of(data))
208
+ else:
209
+ event['payload_base64'] = base64.b64encode(packet[size:]).decode('ascii')
210
+ if carried_env:
211
+ event['raw_withheld'] = 'envv'
212
+ return event
213
+
214
+
215
+ def with_raw(event, packet):
216
+ """The event with its original packet, for the ones that are stored.
217
+ Preserves original evidence for decoder corrections and schema upgrades.
218
+ This may contain sensitive argv; the DB must stay outside the workload.
219
+ Withheld entirely when the packet carried an environment block: the
220
+ decoded event keeps only the allowlisted markers, and storing the raw
221
+ bytes would put every other variable -- keys, tokens, passwords -- back
222
+ into the database through the side door."""
223
+ if 'raw_withheld' not in event:
224
+ event['raw_packet_base64'] = base64.b64encode(packet).decode('ascii')
225
+ return event
226
+
227
+
228
+ def init_db(path):
229
+ db = sqlite3.connect(path, timeout=10)
230
+ db.execute('PRAGMA journal_mode=WAL')
231
+ db.execute('''CREATE TABLE IF NOT EXISTS runtime_events (
232
+ id INTEGER PRIMARY KEY, connection TEXT NOT NULL, received_ms INTEGER NOT NULL,
233
+ container_id TEXT NOT NULL, kind TEXT NOT NULL, body TEXT NOT NULL)''')
234
+ db.execute('CREATE INDEX IF NOT EXISTS runtime_container ON runtime_events(container_id,id)')
235
+ db.execute("CREATE TABLE IF NOT EXISTS collector_state (id INTEGER PRIMARY KEY CHECK(id=1), instance_id TEXT NOT NULL, heartbeat_ms INTEGER NOT NULL)")
236
+ db.execute("""CREATE TABLE IF NOT EXISTS capture_streams (
237
+ id TEXT PRIMARY KEY, collector_id TEXT NOT NULL, container_id TEXT NOT NULL,
238
+ started_ms INTEGER NOT NULL, ended_ms INTEGER, heartbeat_ms INTEGER NOT NULL,
239
+ dropped INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0,
240
+ capabilities_json TEXT NOT NULL)""")
241
+ db.execute('CREATE INDEX IF NOT EXISTS capture_stream_container ON capture_streams(container_id,started_ms)')
242
+ db.commit()
243
+ return db
244
+
245
+
246
+ class Writer(threading.Thread):
247
+ """Writes runtime_events a batch at a time: one commit for a quarter second
248
+ of events, not one for each. A sandbox's thread only queues its row, so a
249
+ slow disk never holds up reading from gVisor."""
250
+ def __init__(self, database):
251
+ super().__init__(daemon=True, name='raytace-writer')
252
+ self.database = database
253
+ self.rows = queue.SimpleQueue()
254
+ self.instance_id = str(uuid.uuid4())
255
+ self.last_heartbeat = None
256
+ self.capabilities = {'processes': True, 'files': False, 'network': False}
257
+
258
+ def put(self, connection, event):
259
+ now = int(time.time() * 1000)
260
+ event = {**event, 'source_instance_id': self.instance_id, 'source_connection_id': connection, 'received_ms': now}
261
+ self.rows.put((connection, now, event.get('container_id', ''), event['kind'], json.dumps(event)))
262
+
263
+ def write_batch(self, db, batch, now=None):
264
+ """Events and health progress commit together; idle heartbeats add no event rows."""
265
+ now = int(time.time() * 1000) if now is None else now
266
+ wrote_heartbeat = False
267
+ with db:
268
+ for connection, received, container, kind, body in batch:
269
+ db.execute('INSERT INTO runtime_events(connection,received_ms,container_id,kind,body) VALUES(?,?,?,?,?)',
270
+ (connection, received, container, kind, body))
271
+ event = json.loads(body)
272
+ db.execute("""INSERT INTO capture_streams VALUES(?,?,?,?,?,?,?,?,?)
273
+ ON CONFLICT(id) DO UPDATE SET
274
+ container_id=CASE WHEN excluded.container_id <> '' THEN excluded.container_id ELSE capture_streams.container_id END,
275
+ ended_ms=COALESCE(excluded.ended_ms,capture_streams.ended_ms),
276
+ heartbeat_ms=CASE WHEN excluded.ended_ms IS NOT NULL THEN excluded.ended_ms ELSE capture_streams.heartbeat_ms END,
277
+ dropped=capture_streams.dropped+excluded.dropped,
278
+ errors=capture_streams.errors+excluded.errors""",
279
+ (connection, self.instance_id, container, received,
280
+ received if kind == 'collector_disconnected' else None, received,
281
+ event.get('dropped_delta', 0) if kind == 'collection_gap' else 0,
282
+ 1 if kind == 'collection_error' else 0, json.dumps(self.capabilities)))
283
+ if container:
284
+ # The greeting precedes container identification. Bind only
285
+ # this connection's greeting once a real event names it.
286
+ db.execute("UPDATE runtime_events SET container_id=? WHERE connection=? AND container_id=''", (container, connection))
287
+ if self.rows.empty() and (self.last_heartbeat is None or now-self.last_heartbeat>=5000):
288
+ wrote_heartbeat = True
289
+ db.execute('INSERT INTO collector_state VALUES(1,?,?) ON CONFLICT(id) DO UPDATE SET instance_id=excluded.instance_id,heartbeat_ms=excluded.heartbeat_ms', (self.instance_id, now))
290
+ db.execute('UPDATE capture_streams SET heartbeat_ms=? WHERE collector_id=? AND ended_ms IS NULL', (now, self.instance_id))
291
+ if wrote_heartbeat:
292
+ self.last_heartbeat = now
293
+
294
+ def run(self):
295
+ db = init_db(self.database)
296
+ while True:
297
+ try:
298
+ batch = [self.rows.get(timeout=5)]
299
+ except queue.Empty:
300
+ batch = []
301
+ deadline = time.monotonic() + 0.25
302
+ while batch and len(batch) < 200:
303
+ try:
304
+ batch.append(self.rows.get(timeout=max(0.0, deadline - time.monotonic())))
305
+ except queue.Empty:
306
+ break
307
+ while True:
308
+ try:
309
+ self.write_batch(db, batch)
310
+ break
311
+ except sqlite3.Error as error:
312
+ print(f'RayTrace collector: runtime_events not written ({error}); retrying', flush=True)
313
+ time.sleep(1)
314
+
315
+
316
+ def serve(endpoint, database):
317
+ os.umask(0o077)
318
+ init_db(database).close()
319
+ server = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
320
+ # Refuse to replace an existing socket: a second collector must not steal it.
321
+ server.bind(endpoint)
322
+ server.listen(32)
323
+ slots = threading.BoundedSemaphore(64)
324
+ writer = Writer(database)
325
+ writer.start()
326
+ engine = None
327
+ if facts_engine:
328
+ try:
329
+ engine = facts_engine.Engine(database)
330
+ engine.start()
331
+ writer.capabilities.update(files=True, network=True)
332
+ except Exception as error:
333
+ print(f'RayTrace collector: fact engine not started ({error}); recording processes only', flush=True)
334
+
335
+ def taken(event):
336
+ """Whether the engine took the event as its own: a file or network
337
+ call, which is then not written. It sees process events too, to know
338
+ which program and tool call a later call belongs to."""
339
+ return bool(engine and engine.accept(event))
340
+
341
+ def client(conn):
342
+ connection = str(uuid.uuid4())
343
+ container = ''
344
+ def record(event):
345
+ writer.put(connection, event)
346
+ try:
347
+ conn.settimeout(5)
348
+ greeting, _, flags, _ = conn.recvmsg(4096)
349
+ if flags & socket.MSG_TRUNC or one(fields(greeting), 1) != 1:
350
+ raise ValueError('unsupported handshake')
351
+ conn.sendall(b'\x08\x01')
352
+ conn.settimeout(None)
353
+ record({'kind': 'collector_connected', 'source': 'collector'})
354
+ previous_dropped = 0
355
+ previous_received = int(time.time()*1000)
356
+ while True:
357
+ packet, _, flags, _ = conn.recvmsg(MAX_PACKET)
358
+ if not packet:
359
+ break
360
+ if flags & socket.MSG_TRUNC:
361
+ raise ValueError('oversized event')
362
+ event = decode(packet)
363
+ if event['container_id']:
364
+ container = event['container_id']
365
+ if event['dropped_count'] != previous_dropped:
366
+ gap = {'kind': 'collection_gap', 'container_id': event['container_id'],
367
+ 'gap_from_ms': previous_received, 'dropped_delta': (event['dropped_count'] - previous_dropped) % (1 << 32)}
368
+ record(gap)
369
+ taken(gap)
370
+ previous_dropped = event['dropped_count']
371
+ previous_received = int(time.time()*1000)
372
+ if not taken(event):
373
+ record(with_raw(event, packet))
374
+ except Exception as error:
375
+ record({'kind': 'collection_error', 'container_id': container, 'error': str(error)[:500]})
376
+ finally:
377
+ # EOF alone does not prove a complete trace; final dropped events
378
+ # may never have been reported in a subsequent packet. Tagged with
379
+ # the sandbox this connection carried (one per sandbox), so a
380
+ # consumer can tell whose open processes will now never report an
381
+ # exit; empty if no event ever arrived.
382
+ gone = {'kind': 'collector_disconnected', 'container_id': container, 'coverage': 'not_certified'}
383
+ record(gone)
384
+ taken(gone)
385
+ conn.close()
386
+ slots.release()
387
+ print('RayTrace collector ready: ' + endpoint, flush=True)
388
+ try:
389
+ while True:
390
+ conn, _ = server.accept()
391
+ if not slots.acquire(blocking=False):
392
+ conn.close()
393
+ continue
394
+ threading.Thread(target=client, args=(conn,), daemon=True).start()
395
+ finally:
396
+ server.close()
397
+ os.unlink(endpoint)
398
+
399
+
400
+ if __name__ == '__main__':
401
+ parser = argparse.ArgumentParser()
402
+ parser.add_argument('--socket', required=True)
403
+ parser.add_argument('--database', required=True)
404
+ opts = parser.parse_args()
405
+ serve(opts.socket, opts.database)
@@ -0,0 +1,255 @@
1
+ """The sandbox network's resolver. Python stdlib only; never mount into guests.
2
+
3
+ Sandboxes resolve names here, on their network's gateway (network.py writes
4
+ their resolv.conf), and each lookup is sent on to the real resolver unchanged
5
+ and answered with whatever it says. Along the way the name and the addresses it
6
+ answered with are written down, which is the only place a host name is seen:
7
+ a program connects to an address, never to a name, and HTTPS is not opened.
8
+
9
+ dns_cache for each sandbox, the name that last answered with an address.
10
+ engine.py reads it to put a name beside a connection. Written
11
+ before the answer goes back, so it is there when the program connects.
12
+ facts one `dns` fact for each name a sandbox looked up, with a count
13
+
14
+ It decides nothing: no name is refused or changed. A lookup it cannot read is
15
+ still forwarded. A sandbox that asks a public resolver directly, or uses DNS
16
+ over HTTPS, is not seen here, and its connections then show bare addresses.
17
+ If this stops, sandboxes cannot resolve names until systemd starts it again.
18
+ """
19
+ import hashlib
20
+ import ipaddress
21
+ import json
22
+ from pathlib import Path
23
+ import socket
24
+ import socketserver
25
+ import sqlite3
26
+ import struct
27
+ import subprocess
28
+ import threading
29
+ import time
30
+
31
+ import engine
32
+ import network
33
+
34
+ DATABASE = '/var/lib/raytace/runtime.db'
35
+ MAX_NAME = 253
36
+ slots = threading.BoundedSemaphore(64) # lookups in flight; past that a sandbox's packet is dropped and it asks again
37
+ lock = threading.Lock()
38
+ containers = {'read': 0.0, 'by_ip': {}, 'moved': set()}
39
+ facts, dirty = {}, set()
40
+
41
+
42
+ def name_at(message, at):
43
+ """(the name at `at`, where the next field starts). Follows the pointers
44
+ DNS uses to repeat a name, a bounded number of times."""
45
+ labels, jumps, after = [], 0, None
46
+ while True:
47
+ size = message[at]
48
+ if size & 0xC0 == 0xC0:
49
+ if after is None:
50
+ after = at + 2
51
+ at = ((size & 0x3F) << 8) | message[at + 1]
52
+ jumps += 1
53
+ if jumps > 20:
54
+ raise ValueError('name loops')
55
+ continue
56
+ at += 1
57
+ if size == 0:
58
+ break
59
+ labels.append(message[at:at + size].decode('ascii', errors='replace'))
60
+ at += size
61
+ name = '.'.join(labels).lower()
62
+ if len(name) > MAX_NAME:
63
+ raise ValueError('name too long')
64
+ return name, at if after is None else after
65
+
66
+
67
+ def lookup(query, answer):
68
+ """(name asked, record type, result code, addresses answered), or None when either message is not a lookup this reads."""
69
+ try:
70
+ if len(query) < 12 or struct.unpack_from('>H', query, 4)[0] != 1:
71
+ return None
72
+ name, at = name_at(query, 12)
73
+ kind = struct.unpack_from('>H', query, at)[0]
74
+ if len(answer) < 12:
75
+ return name, kind, None, []
76
+ flags, _, count = struct.unpack_from('>HHH', answer, 2)
77
+ _, at = name_at(answer, 12)
78
+ at += 4
79
+ addresses = []
80
+ for _ in range(min(count, 64)):
81
+ _, at = name_at(answer, at)
82
+ record, _, _, size = struct.unpack_from('>HHIH', answer, at)
83
+ at += 10
84
+ if record == 1 and size == 4:
85
+ addresses.append(str(ipaddress.IPv4Address(answer[at:at + 4])))
86
+ elif record == 28 and size == 16:
87
+ addresses.append(str(ipaddress.IPv6Address(answer[at:at + 16])))
88
+ at += size
89
+ return name, kind, flags & 0xF, addresses
90
+ except (IndexError, struct.error, ValueError):
91
+ return None
92
+
93
+
94
+ def container_of(ip):
95
+ """The sandbox container that holds an address of the sandbox network now,
96
+ as (container id, whether it is a fork), or None. Docker is asked, not a
97
+ record: an address goes to the next container as soon as one stops, and a
98
+ lookup filed by a remembered address would be filed under a sandbox that
99
+ did not make it, which may be another organization's. What Docker said is
100
+ kept for two seconds; a container without a record of projects.py's (a
101
+ throwaway job) is no sandbox."""
102
+ now = time.monotonic()
103
+ if now - containers['read'] > 2:
104
+ try:
105
+ listed = json.loads(subprocess.check_output(
106
+ ['docker', 'network', 'inspect', 'raytace-egress', '--format', '{{json .Containers}}'], timeout=5)) or {}
107
+ found = {str(joined.get('IPv4Address', '')).partition('/')[0]: container for container, joined in listed.items()}
108
+ except (subprocess.SubprocessError, OSError, ValueError):
109
+ found = {} # nobody is named when Docker cannot say: a lookup unfiled, never misfiled
110
+ # An address that changed hands: what its last holder looked up is not the new one's.
111
+ before = containers['by_ip']
112
+ containers['moved'].update(address for address in before.keys() | found.keys() if before.get(address) != found.get(address))
113
+ containers.update(read=now, by_ip=found)
114
+ container = containers['by_ip'].get(ip)
115
+ if not container:
116
+ return None
117
+ try:
118
+ record = json.loads((engine.CONTAINERS / (container + '.json')).read_text())
119
+ except (OSError, ValueError):
120
+ return None
121
+ return container, bool(record.get('fork_of'))
122
+
123
+
124
+ def record(db, client, query, answer):
125
+ found = lookup(query, answer)
126
+ if not found or not found[0]:
127
+ return
128
+ name, kind, code, addresses = found
129
+ now = time.time_ns()
130
+ with lock:
131
+ container = container_of(client)
132
+ if containers['moved']:
133
+ db.executemany('DELETE FROM dns_cache WHERE client = ?', [(address,) for address in containers['moved']])
134
+ containers['moved'].clear()
135
+ db.commit()
136
+ if addresses:
137
+ db.executemany('INSERT OR REPLACE INTO dns_cache (client, ip, name, seen_ns) VALUES (?,?,?,?)',
138
+ [(client, address, name, now) for address in addresses])
139
+ db.commit()
140
+ if not container:
141
+ return
142
+ fact_id = hashlib.sha256('\0'.join((container[0], 'dns', name)).encode()).hexdigest()[:32]
143
+ fact = facts.get(fact_id)
144
+ if fact is None:
145
+ fact = facts[fact_id] = {'fact_id': fact_id, 'container_id': container[0], 'kind': 'dns', 'effect': 'looked_up', 'name': name,
146
+ 'count': 0, 'first_ns': str(now), 'addresses': [], 'pid': None, 'call_id': None,
147
+ 'basis': {'note': 'asked of the sandbox network\'s resolver; which process asked is not known'}}
148
+ if container[1]:
149
+ fact['fork'] = True
150
+ fact['count'] += 1
151
+ fact['last_ns'] = str(now)
152
+ if addresses:
153
+ fact['addresses'] = sorted(set(fact['addresses']) | set(addresses))[:16]
154
+ if code:
155
+ fact['failed'] = {2: 'SERVFAIL', 3: 'NXDOMAIN', 5: 'REFUSED'}.get(code, str(code))
156
+ elif kind in (1, 28):
157
+ fact.pop('failed', None)
158
+ dirty.add(fact_id)
159
+
160
+
161
+ def flush(database):
162
+ """Writes the lookups that changed, every two seconds: a name asked a
163
+ thousand times is one row written a few times, not a thousand."""
164
+ db = sqlite3.connect(database, timeout=10)
165
+ while True:
166
+ time.sleep(2)
167
+ with lock:
168
+ rows = [(fact_id, facts[fact_id]['container_id'], 'dns', facts[fact_id]['last_ns'], int(time.time() * 1000), json.dumps(facts[fact_id]))
169
+ for fact_id in dirty]
170
+ dirty.clear()
171
+ if len(facts) > 20_000:
172
+ facts.clear()
173
+ if rows:
174
+ try:
175
+ db.executemany('INSERT OR REPLACE INTO facts (fact_id, container_id, kind, at_ns, received_ms, body) VALUES (?,?,?,?,?,?)', rows)
176
+ db.commit()
177
+ except sqlite3.Error as error:
178
+ print(f'RayTrace resolver: lookups not written ({error})', flush=True)
179
+
180
+
181
+ def serve(listen, upstream, database):
182
+ setup = sqlite3.connect(database, timeout=10)
183
+ setup.execute('PRAGMA journal_mode=WAL')
184
+ setup.executescript(engine.SCHEMA)
185
+ setup.close()
186
+ local = threading.local()
187
+
188
+ def db():
189
+ if not hasattr(local, 'db'):
190
+ local.db = sqlite3.connect(database, timeout=10)
191
+ local.db.execute('PRAGMA synchronous=NORMAL')
192
+ return local.db
193
+
194
+ def note(client, query, answer):
195
+ try:
196
+ record(db(), client, query, answer)
197
+ except Exception as error: # writing a lookup down never costs the sandbox its answer
198
+ print(f'RayTrace resolver: {type(error).__name__}: {error}', flush=True)
199
+
200
+ class Datagram(socketserver.BaseRequestHandler):
201
+ def handle(self):
202
+ if not slots.acquire(blocking=False):
203
+ return
204
+ try:
205
+ query, reply_on = self.request
206
+ with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as out:
207
+ out.settimeout(5)
208
+ out.sendto(query, (upstream, 53))
209
+ answer = out.recv(65535)
210
+ note(self.client_address[0], query, answer)
211
+ reply_on.sendto(answer, self.client_address)
212
+ except OSError:
213
+ pass # no answer: the sandbox asks again
214
+ finally:
215
+ slots.release()
216
+
217
+ class Stream(socketserver.BaseRequestHandler):
218
+ def read(self, connection):
219
+ head = connection.recv(2, socket.MSG_WAITALL)
220
+ if len(head) < 2:
221
+ return None
222
+ body = connection.recv(struct.unpack('>H', head)[0], socket.MSG_WAITALL)
223
+ return body if len(body) == struct.unpack('>H', head)[0] else None
224
+
225
+ def handle(self):
226
+ if not slots.acquire(blocking=False):
227
+ return
228
+ try:
229
+ self.request.settimeout(5)
230
+ with socket.create_connection((upstream, 53), timeout=5) as out:
231
+ while (query := self.read(self.request)) is not None:
232
+ out.sendall(struct.pack('>H', len(query)) + query)
233
+ answer = self.read(out)
234
+ if answer is None:
235
+ break
236
+ note(self.client_address[0], query, answer)
237
+ self.request.sendall(struct.pack('>H', len(answer)) + answer)
238
+ except OSError:
239
+ pass
240
+ finally:
241
+ slots.release()
242
+
243
+ socketserver.ThreadingUDPServer.allow_reuse_address = socketserver.ThreadingTCPServer.allow_reuse_address = True
244
+ socketserver.ThreadingUDPServer.daemon_threads = socketserver.ThreadingTCPServer.daemon_threads = True
245
+ datagrams = socketserver.ThreadingUDPServer(listen, Datagram)
246
+ streams = socketserver.ThreadingTCPServer(listen, Stream)
247
+ threading.Thread(target=flush, args=(database,), daemon=True).start()
248
+ threading.Thread(target=streams.serve_forever, daemon=True).start()
249
+ print(f'RayTrace resolver ready: {listen[0]}:{listen[1]} -> {upstream}', flush=True)
250
+ datagrams.serve_forever()
251
+
252
+
253
+ if __name__ == '__main__':
254
+ Path(DATABASE).parent.mkdir(parents=True, exist_ok=True)
255
+ serve((network.GATEWAY, 53), network.resolver(), DATABASE)