raytrace-cli 0.11.1__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raytrace/__init__.py +1 -0
- raytrace/__main__.py +33 -0
- raytrace/app/LICENSE +21 -0
- raytrace/app/README.md +599 -0
- raytrace/app/bin/raytrace.mjs +297 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-BwCSEQnW.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-CW0RaeGs.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-B9HHJjqV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-Dr3UlScf.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-Aqo67rzb.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-BmdmIIQ2.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-CUSSCpQX.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-DvUfVpUG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BjWJ59Pq.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BtiwyxMk.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-BJfUCQsA.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-DR84L5F-.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-4vooXBpG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-DRtaDpgU.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-BLI3TTWz.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-CJIcxD6K.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-CTOM6hUh.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-fLZuRloM.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-9HEixskS.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-V-xxqcpd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-BgSNZQsw.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-DWFSQ4vo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-D38SheWl.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-DmB0ttJJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-D2EvbN8M.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-iLQfcSjf.woff +0 -0
- raytrace/app/dist/dashboard/assets/index-DepcI8Fi.js +38 -0
- raytrace/app/dist/dashboard/assets/index-H97VLD5l.css +1 -0
- raytrace/app/dist/dashboard/favicon.svg +6 -0
- raytrace/app/dist/dashboard/index.html +15 -0
- raytrace/app/package.json +48 -0
- raytrace/app/proxy/agent-intent-schema.test.mjs +142 -0
- raytrace/app/proxy/call-assessment.mjs +385 -0
- raytrace/app/proxy/call-assessment.test.mjs +167 -0
- raytrace/app/proxy/call-category.mjs +461 -0
- raytrace/app/proxy/capture-health.mjs +30 -0
- raytrace/app/proxy/capture-health.test.mjs +44 -0
- raytrace/app/proxy/capture.mjs +38 -0
- raytrace/app/proxy/capture.test.mjs +49 -0
- raytrace/app/proxy/claude-code-adapter.mjs +301 -0
- raytrace/app/proxy/claude-code-hook.mjs +22 -0
- raytrace/app/proxy/codex-config.mjs +36 -0
- raytrace/app/proxy/codex.mjs +14 -0
- raytrace/app/proxy/evidence-schema.test.mjs +240 -0
- raytrace/app/proxy/evidence-store.mjs +993 -0
- raytrace/app/proxy/evidence-store.test.mjs +301 -0
- raytrace/app/proxy/exec-narrative.mjs +625 -0
- raytrace/app/proxy/exec-narrative.test.mjs +262 -0
- raytrace/app/proxy/execution-correlation.mjs +185 -0
- raytrace/app/proxy/execution-correlation.test.mjs +146 -0
- raytrace/app/proxy/execution-runner.mjs +89 -0
- raytrace/app/proxy/execution-runner.test.mjs +38 -0
- raytrace/app/proxy/experiment-engine.mjs +195 -0
- raytrace/app/proxy/experiment-engine.test.mjs +93 -0
- raytrace/app/proxy/explanations.mjs +36 -0
- raytrace/app/proxy/explanations.test.mjs +20 -0
- raytrace/app/proxy/failure-signature.mjs +56 -0
- raytrace/app/proxy/gvisor-forwarder.mjs +493 -0
- raytrace/app/proxy/gvisor-forwarder.test.mjs +433 -0
- raytrace/app/proxy/init-evidence-db.mjs +227 -0
- raytrace/app/proxy/local-sandboxes.mjs +94 -0
- raytrace/app/proxy/local-token.mjs +30 -0
- raytrace/app/proxy/playground-recovery.test.mjs +130 -0
- raytrace/app/proxy/playground.mjs +439 -0
- raytrace/app/proxy/prompt-context.mjs +278 -0
- raytrace/app/proxy/prompt-traces.mjs +40 -0
- raytrace/app/proxy/prompt-traces.test.mjs +32 -0
- raytrace/app/proxy/providers.mjs +65 -0
- raytrace/app/proxy/providers.test.mjs +76 -0
- raytrace/app/proxy/proxy.integration.test.mjs +135 -0
- raytrace/app/proxy/raytace-proxy.mjs +941 -0
- raytrace/app/proxy/request-metrics.mjs +13 -0
- raytrace/app/proxy/request-metrics.test.mjs +19 -0
- raytrace/app/proxy/request-policy.mjs +45 -0
- raytrace/app/proxy/request-policy.test.mjs +47 -0
- raytrace/app/proxy/run-recap.mjs +139 -0
- raytrace/app/proxy/runtime-attribution.mjs +110 -0
- raytrace/app/proxy/sandbox-bridge.mjs +49 -0
- raytrace/app/proxy/schema/EVIDENCE_DATABASE.md +276 -0
- raytrace/app/proxy/schema/agent_intent.sql +101 -0
- raytrace/app/proxy/schema/auxiliary.sql +90 -0
- raytrace/app/proxy/schema/capture_health.sql +14 -0
- raytrace/app/proxy/schema/classification.sql +89 -0
- raytrace/app/proxy/schema/file_context.sql +53 -0
- raytrace/app/proxy/schema/runtime_evidence.sql +201 -0
- raytrace/app/proxy/schema/search.sql +66 -0
- raytrace/app/proxy/schema/shared_storage.sql +21 -0
- raytrace/app/proxy/search.mjs +418 -0
- raytrace/app/proxy/search.test.mjs +161 -0
- raytrace/app/proxy/smoke.test.mjs +93 -0
- raytrace/app/proxy/step-attribution.mjs +380 -0
- raytrace/app/proxy/step-lab.mjs +78 -0
- raytrace/app/proxy/step-lab.test.mjs +38 -0
- raytrace/app/proxy/summaries.mjs +151 -0
- raytrace/app/proxy/summaries.test.mjs +69 -0
- raytrace/app/proxy/tool-metadata.mjs +111 -0
- raytrace/app/proxy/tool-metadata.test.mjs +38 -0
- raytrace/app/proxy/trace-view.mjs +324 -0
- raytrace/app/proxy/trace-view.test.mjs +92 -0
- raytrace/app/proxy/typesafe.mjs +63 -0
- raytrace/app/proxy/typesafe.test.mjs +39 -0
- raytrace/app/src/agents.mjs +7 -0
- raytrace/app/src/api.mjs +57 -0
- raytrace/app/src/auth.mjs +205 -0
- raytrace/app/src/beta.mjs +96 -0
- raytrace/app/src/claude-code.mjs +93 -0
- raytrace/app/src/commands.mjs +285 -0
- raytrace/app/src/config.mjs +49 -0
- raytrace/app/src/export.mjs +200 -0
- raytrace/app/src/lima.mjs +176 -0
- raytrace/app/src/local-sandbox.mjs +358 -0
- raytrace/app/src/mcp.mjs +688 -0
- raytrace/app/src/paths.mjs +33 -0
- raytrace/app/src/prompt.mjs +42 -0
- raytrace/app/src/proxy-process.mjs +143 -0
- raytrace/app/src/sandbox.mjs +306 -0
- raytrace/app/src/share.mjs +86 -0
- raytrace/app/src/terminal.mjs +68 -0
- raytrace/app/src/upload.mjs +198 -0
- raytrace/app/worker/sandbox/Dockerfile +8 -0
- raytrace/app/worker/sandbox/SYSCALL_CATALOG.md +197 -0
- raytrace/app/worker/sandbox/claude-hook.mjs +364 -0
- raytrace/app/worker/sandbox/collector.py +405 -0
- raytrace/app/worker/sandbox/dns_forwarder.py +255 -0
- raytrace/app/worker/sandbox/engine.py +940 -0
- raytrace/app/worker/sandbox/file_scan.py +491 -0
- raytrace/app/worker/sandbox/lima.yaml +29 -0
- raytrace/app/worker/sandbox/network.py +90 -0
- raytrace/app/worker/sandbox/projects.py +702 -0
- raytrace/app/worker/sandbox/setup.py +159 -0
- raytrace/app/worker/sandbox/syscall_catalog.sql +985 -0
- raytrace/app/worker/sandbox/transcript-delivery.test.mjs +153 -0
- raytrace/app/worker/sandbox/viewer.py +195 -0
- raytrace/node/LICENSE +2946 -0
- raytrace/node/node.exe +4 -0
- raytrace_cli-0.11.1.dist-info/LICENSE +21 -0
- raytrace_cli-0.11.1.dist-info/METADATA +42 -0
- raytrace_cli-0.11.1.dist-info/RECORD +165 -0
- raytrace_cli-0.11.1.dist-info/WHEEL +4 -0
- raytrace_cli-0.11.1.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
"""External gVisor SecCheck v1 collector. Python stdlib only; never mount into guests.
|
|
2
|
+
|
|
3
|
+
Wire/schema references: google/gvisor pkg/sentry/seccheck/{points,sinks/remote}.
|
|
4
|
+
Only selected fields are decoded; unknown event payloads are retained as base64.
|
|
5
|
+
The bounded protobuf reader handles wire types, not arbitrary recursive schemas.
|
|
6
|
+
|
|
7
|
+
Two kinds of event leave here. Process events (exec, clone, exit) are written
|
|
8
|
+
to runtime_events as they always were, now a batch at a time. File and network
|
|
9
|
+
calls are far too many to keep: each goes to the engine (engine.py), which
|
|
10
|
+
keeps what matters of it as a fact and counts the rest, and is never written.
|
|
11
|
+
"""
|
|
12
|
+
import argparse
|
|
13
|
+
import base64
|
|
14
|
+
import ipaddress
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import queue
|
|
18
|
+
import socket
|
|
19
|
+
import sqlite3
|
|
20
|
+
import struct
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
import uuid
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
import engine as facts_engine
|
|
27
|
+
except Exception as problem: # a host without the engine still records processes
|
|
28
|
+
facts_engine = None
|
|
29
|
+
print(f'RayTrace collector: no fact engine ({problem}); recording processes only', flush=True)
|
|
30
|
+
|
|
31
|
+
MAX_PACKET = 1024 * 1024
|
|
32
|
+
|
|
33
|
+
# The ONLY environment variables ever read off a traced process. `envv` is
|
|
34
|
+
# requested on the execve points (see setup.py) purely to carry these two
|
|
35
|
+
# markers, which turn tool-call correlation from a text-similarity guess into
|
|
36
|
+
# an exact join. Everything else in the environment block -- API keys, tokens,
|
|
37
|
+
# database passwords -- is dropped here, before any write, and the raw packet
|
|
38
|
+
# is withheld for events that carried one (see decode). Never widen this list.
|
|
39
|
+
MARKER_KEYS = ('RAYTRACE_CALL_ID', 'RAYTRACE_SESSION')
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def markers(entries):
|
|
43
|
+
"""Pick the allowlisted markers out of a raw `envv` list. Returns {} when
|
|
44
|
+
none are present, and never retains a variable outside MARKER_KEYS."""
|
|
45
|
+
found = {}
|
|
46
|
+
for entry in entries:
|
|
47
|
+
name, separator, value = string(entry).partition('=')
|
|
48
|
+
if separator and name in MARKER_KEYS and len(value) <= 256:
|
|
49
|
+
found[name] = value
|
|
50
|
+
return found
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def fields(data):
|
|
54
|
+
result = {}
|
|
55
|
+
pos = 0
|
|
56
|
+
|
|
57
|
+
def varint():
|
|
58
|
+
nonlocal pos
|
|
59
|
+
value = 0
|
|
60
|
+
for shift in range(0, 70, 7):
|
|
61
|
+
if pos >= len(data):
|
|
62
|
+
raise ValueError('truncated varint')
|
|
63
|
+
byte = data[pos]
|
|
64
|
+
pos += 1
|
|
65
|
+
if shift == 63 and byte > 1:
|
|
66
|
+
raise ValueError('varint overflow')
|
|
67
|
+
value |= (byte & 127) << shift
|
|
68
|
+
if byte < 128:
|
|
69
|
+
return value
|
|
70
|
+
raise ValueError('invalid varint')
|
|
71
|
+
|
|
72
|
+
while pos < len(data):
|
|
73
|
+
tag = varint()
|
|
74
|
+
number, wire = tag >> 3, tag & 7
|
|
75
|
+
if not number:
|
|
76
|
+
raise ValueError('invalid field zero')
|
|
77
|
+
if wire == 0:
|
|
78
|
+
value = varint()
|
|
79
|
+
elif wire in (1, 2, 5):
|
|
80
|
+
size = varint() if wire == 2 else (8 if wire == 1 else 4)
|
|
81
|
+
if pos + size > len(data):
|
|
82
|
+
raise ValueError('truncated field')
|
|
83
|
+
value = data[pos:pos + size]
|
|
84
|
+
pos += size
|
|
85
|
+
else:
|
|
86
|
+
raise ValueError('unsupported wire type')
|
|
87
|
+
result.setdefault(number, []).append(value)
|
|
88
|
+
return result
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def one(data, key, default=0):
|
|
92
|
+
return data.get(key, [default])[-1]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def string(value):
|
|
96
|
+
if not isinstance(value, bytes):
|
|
97
|
+
raise ValueError('expected string field')
|
|
98
|
+
return value.decode('utf-8', errors='replace')
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def signed(value):
|
|
102
|
+
return value - (1 << 64) if value >= (1 << 63) else value
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def result_of(data):
|
|
106
|
+
"""A syscall's result on its exit point; nothing on an enter point. gVisor
|
|
107
|
+
reports a failure in `errno` and leaves the result zero, so errno alone
|
|
108
|
+
says whether a call succeeded."""
|
|
109
|
+
if 2 not in data:
|
|
110
|
+
return {}
|
|
111
|
+
status = fields(one(data, 2, b''))
|
|
112
|
+
return {'result': signed(one(status, 1)), 'errno': abs(signed(one(status, 2)))}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def address(raw):
|
|
116
|
+
"""A socket address as the program passed it (Linux's struct sockaddr): an
|
|
117
|
+
IP address and port, never a host name."""
|
|
118
|
+
family = struct.unpack_from('<H', raw)[0] if len(raw) >= 2 else None
|
|
119
|
+
if family == 2 and len(raw) >= 8:
|
|
120
|
+
return {'family': 'inet', 'ip': str(ipaddress.IPv4Address(raw[4:8])), 'port': struct.unpack_from('>H', raw, 2)[0]}
|
|
121
|
+
if family == 10 and len(raw) >= 24:
|
|
122
|
+
found = ipaddress.IPv6Address(raw[8:24])
|
|
123
|
+
return {'family': 'inet', 'ip': str(found.ipv4_mapped or found), 'port': struct.unpack_from('>H', raw, 2)[0]}
|
|
124
|
+
if family == 1:
|
|
125
|
+
return {'family': 'unix', 'path': raw[2:].split(b'\0')[0].decode('utf-8', errors='replace')[:200]}
|
|
126
|
+
return {'family': 'other' if family is not None else 'unknown'}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def decode(packet):
|
|
130
|
+
if len(packet) < 8 or len(packet) > MAX_PACKET:
|
|
131
|
+
raise ValueError('invalid packet size')
|
|
132
|
+
size, kind, dropped = struct.unpack_from('<HHI', packet)
|
|
133
|
+
if size < 8 or size > len(packet):
|
|
134
|
+
raise ValueError('invalid header size')
|
|
135
|
+
data = fields(packet[size:])
|
|
136
|
+
context = fields(one(data, 1, b''))
|
|
137
|
+
carried_env = False
|
|
138
|
+
event = {
|
|
139
|
+
'source': 'gvisor_seccheck', 'message_type': kind,
|
|
140
|
+
'dropped_count': dropped, 'timestamp_ns': str(one(context, 1)),
|
|
141
|
+
'container_id': string(one(context, 6, b'')),
|
|
142
|
+
'pid': one(context, 4), 'tid': one(context, 2),
|
|
143
|
+
'process_start_ns': str(one(context, 5)),
|
|
144
|
+
'ppid': one(context, 10), 'cwd': string(one(context, 8, b'')),
|
|
145
|
+
'process_name': string(one(context, 9, b'')),
|
|
146
|
+
'kind': 'runtime_event',
|
|
147
|
+
}
|
|
148
|
+
if kind == 3:
|
|
149
|
+
event.update(kind='exec_succeeded', executable=string(one(data, 2, b'')),
|
|
150
|
+
argv=[string(v) for v in data.get(3, [])])
|
|
151
|
+
# Field 4 is `envv` on sentry/execve. Unlike the syscall points this is
|
|
152
|
+
# NOT opt-in -- the Sentry has always sent it, so every raw packet kept
|
|
153
|
+
# here carried the process's entire environment (keys, tokens,
|
|
154
|
+
# passwords) into the database. Same treatment as the syscall points:
|
|
155
|
+
# keep the allowlisted markers, withhold the raw bytes.
|
|
156
|
+
if 4 in data:
|
|
157
|
+
carried_env = True
|
|
158
|
+
found = markers(data[4])
|
|
159
|
+
if found:
|
|
160
|
+
event['markers'] = found
|
|
161
|
+
elif kind == 11:
|
|
162
|
+
event.update(kind='exec_attempt', executable=string(one(data, 6, b'')),
|
|
163
|
+
argv=[string(v) for v in data.get(7, [])])
|
|
164
|
+
# Field 8 is `envv`, requested only for the markers (see MARKER_KEYS).
|
|
165
|
+
# `carried_env` records that an environment block was present so the
|
|
166
|
+
# raw packet can be withheld below -- the markers themselves are safe
|
|
167
|
+
# to keep, the block they arrived in is not.
|
|
168
|
+
if 8 in data:
|
|
169
|
+
carried_env = True
|
|
170
|
+
found = markers(data[8])
|
|
171
|
+
if found:
|
|
172
|
+
event['markers'] = found
|
|
173
|
+
if 2 in data:
|
|
174
|
+
status = fields(one(data, 2))
|
|
175
|
+
errno = one(status, 2)
|
|
176
|
+
event.update(kind='exec_failed' if errno else 'exec_syscall_return',
|
|
177
|
+
errno=errno, result=signed(one(status, 1)))
|
|
178
|
+
elif kind == 2:
|
|
179
|
+
event.update(kind='clone', child_pid=one(data, 4), child_tid=one(data, 3),
|
|
180
|
+
child_start_ns=str(one(data, 5)))
|
|
181
|
+
elif kind in (4, 5):
|
|
182
|
+
status = one(data, 2)
|
|
183
|
+
event.update(kind='process_exit' if kind == 4 else 'thread_exit',
|
|
184
|
+
wait_status=status,
|
|
185
|
+
exit_code=(status >> 8) & 255 if status & 127 == 0 else None,
|
|
186
|
+
signal=status & 127 if status & 127 else None)
|
|
187
|
+
elif kind == 6:
|
|
188
|
+
status = fields(one(data, 2, b''))
|
|
189
|
+
# A call gVisor has no typed point for: its number and six integers.
|
|
190
|
+
# A path argument is a pointer into the program's memory, so the
|
|
191
|
+
# integers name no file.
|
|
192
|
+
event.update(kind='syscall_return' if 2 in data else 'syscall_enter',
|
|
193
|
+
sysno=one(data, 4), result=signed(one(status, 1)), errno=one(status, 2),
|
|
194
|
+
args=[one(data, number) for number in range(5, 11)])
|
|
195
|
+
elif kind == 7:
|
|
196
|
+
# open, openat and creat. `dirfd` is what a relative path is relative
|
|
197
|
+
# to (-100: the working directory), `fd_path` that folder's path.
|
|
198
|
+
event.update(kind='open', sysno=one(data, 3), dirfd=signed(one(data, 4)), fd_path=string(one(data, 5, b'')),
|
|
199
|
+
pathname=string(one(data, 6, b'')), flags=one(data, 7), mode=one(data, 8), **result_of(data))
|
|
200
|
+
elif kind == 10:
|
|
201
|
+
event.update(kind='connect', sysno=one(data, 3), **address(one(data, 6, b'')), **result_of(data))
|
|
202
|
+
elif kind == 37:
|
|
203
|
+
event.update(kind='listen', sysno=one(data, 3), backlog=one(data, 6), **address(one(data, 7, b'')), **result_of(data))
|
|
204
|
+
elif kind in (14, 15):
|
|
205
|
+
# setuid/setgid carry one id; setresuid/setresgid the real, effective and saved ones.
|
|
206
|
+
event.update(kind='setid', sysno=one(data, 3), ids=[one(data, number) for number in ((4,) if kind == 14 else (4, 5, 6))],
|
|
207
|
+
**result_of(data))
|
|
208
|
+
else:
|
|
209
|
+
event['payload_base64'] = base64.b64encode(packet[size:]).decode('ascii')
|
|
210
|
+
if carried_env:
|
|
211
|
+
event['raw_withheld'] = 'envv'
|
|
212
|
+
return event
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def with_raw(event, packet):
|
|
216
|
+
"""The event with its original packet, for the ones that are stored.
|
|
217
|
+
Preserves original evidence for decoder corrections and schema upgrades.
|
|
218
|
+
This may contain sensitive argv; the DB must stay outside the workload.
|
|
219
|
+
Withheld entirely when the packet carried an environment block: the
|
|
220
|
+
decoded event keeps only the allowlisted markers, and storing the raw
|
|
221
|
+
bytes would put every other variable -- keys, tokens, passwords -- back
|
|
222
|
+
into the database through the side door."""
|
|
223
|
+
if 'raw_withheld' not in event:
|
|
224
|
+
event['raw_packet_base64'] = base64.b64encode(packet).decode('ascii')
|
|
225
|
+
return event
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def init_db(path):
|
|
229
|
+
db = sqlite3.connect(path, timeout=10)
|
|
230
|
+
db.execute('PRAGMA journal_mode=WAL')
|
|
231
|
+
db.execute('''CREATE TABLE IF NOT EXISTS runtime_events (
|
|
232
|
+
id INTEGER PRIMARY KEY, connection TEXT NOT NULL, received_ms INTEGER NOT NULL,
|
|
233
|
+
container_id TEXT NOT NULL, kind TEXT NOT NULL, body TEXT NOT NULL)''')
|
|
234
|
+
db.execute('CREATE INDEX IF NOT EXISTS runtime_container ON runtime_events(container_id,id)')
|
|
235
|
+
db.execute("CREATE TABLE IF NOT EXISTS collector_state (id INTEGER PRIMARY KEY CHECK(id=1), instance_id TEXT NOT NULL, heartbeat_ms INTEGER NOT NULL)")
|
|
236
|
+
db.execute("""CREATE TABLE IF NOT EXISTS capture_streams (
|
|
237
|
+
id TEXT PRIMARY KEY, collector_id TEXT NOT NULL, container_id TEXT NOT NULL,
|
|
238
|
+
started_ms INTEGER NOT NULL, ended_ms INTEGER, heartbeat_ms INTEGER NOT NULL,
|
|
239
|
+
dropped INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0,
|
|
240
|
+
capabilities_json TEXT NOT NULL)""")
|
|
241
|
+
db.execute('CREATE INDEX IF NOT EXISTS capture_stream_container ON capture_streams(container_id,started_ms)')
|
|
242
|
+
db.commit()
|
|
243
|
+
return db
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class Writer(threading.Thread):
|
|
247
|
+
"""Writes runtime_events a batch at a time: one commit for a quarter second
|
|
248
|
+
of events, not one for each. A sandbox's thread only queues its row, so a
|
|
249
|
+
slow disk never holds up reading from gVisor."""
|
|
250
|
+
def __init__(self, database):
|
|
251
|
+
super().__init__(daemon=True, name='raytace-writer')
|
|
252
|
+
self.database = database
|
|
253
|
+
self.rows = queue.SimpleQueue()
|
|
254
|
+
self.instance_id = str(uuid.uuid4())
|
|
255
|
+
self.last_heartbeat = None
|
|
256
|
+
self.capabilities = {'processes': True, 'files': False, 'network': False}
|
|
257
|
+
|
|
258
|
+
def put(self, connection, event):
|
|
259
|
+
now = int(time.time() * 1000)
|
|
260
|
+
event = {**event, 'source_instance_id': self.instance_id, 'source_connection_id': connection, 'received_ms': now}
|
|
261
|
+
self.rows.put((connection, now, event.get('container_id', ''), event['kind'], json.dumps(event)))
|
|
262
|
+
|
|
263
|
+
def write_batch(self, db, batch, now=None):
|
|
264
|
+
"""Events and health progress commit together; idle heartbeats add no event rows."""
|
|
265
|
+
now = int(time.time() * 1000) if now is None else now
|
|
266
|
+
wrote_heartbeat = False
|
|
267
|
+
with db:
|
|
268
|
+
for connection, received, container, kind, body in batch:
|
|
269
|
+
db.execute('INSERT INTO runtime_events(connection,received_ms,container_id,kind,body) VALUES(?,?,?,?,?)',
|
|
270
|
+
(connection, received, container, kind, body))
|
|
271
|
+
event = json.loads(body)
|
|
272
|
+
db.execute("""INSERT INTO capture_streams VALUES(?,?,?,?,?,?,?,?,?)
|
|
273
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
274
|
+
container_id=CASE WHEN excluded.container_id <> '' THEN excluded.container_id ELSE capture_streams.container_id END,
|
|
275
|
+
ended_ms=COALESCE(excluded.ended_ms,capture_streams.ended_ms),
|
|
276
|
+
heartbeat_ms=CASE WHEN excluded.ended_ms IS NOT NULL THEN excluded.ended_ms ELSE capture_streams.heartbeat_ms END,
|
|
277
|
+
dropped=capture_streams.dropped+excluded.dropped,
|
|
278
|
+
errors=capture_streams.errors+excluded.errors""",
|
|
279
|
+
(connection, self.instance_id, container, received,
|
|
280
|
+
received if kind == 'collector_disconnected' else None, received,
|
|
281
|
+
event.get('dropped_delta', 0) if kind == 'collection_gap' else 0,
|
|
282
|
+
1 if kind == 'collection_error' else 0, json.dumps(self.capabilities)))
|
|
283
|
+
if container:
|
|
284
|
+
# The greeting precedes container identification. Bind only
|
|
285
|
+
# this connection's greeting once a real event names it.
|
|
286
|
+
db.execute("UPDATE runtime_events SET container_id=? WHERE connection=? AND container_id=''", (container, connection))
|
|
287
|
+
if self.rows.empty() and (self.last_heartbeat is None or now-self.last_heartbeat>=5000):
|
|
288
|
+
wrote_heartbeat = True
|
|
289
|
+
db.execute('INSERT INTO collector_state VALUES(1,?,?) ON CONFLICT(id) DO UPDATE SET instance_id=excluded.instance_id,heartbeat_ms=excluded.heartbeat_ms', (self.instance_id, now))
|
|
290
|
+
db.execute('UPDATE capture_streams SET heartbeat_ms=? WHERE collector_id=? AND ended_ms IS NULL', (now, self.instance_id))
|
|
291
|
+
if wrote_heartbeat:
|
|
292
|
+
self.last_heartbeat = now
|
|
293
|
+
|
|
294
|
+
def run(self):
|
|
295
|
+
db = init_db(self.database)
|
|
296
|
+
while True:
|
|
297
|
+
try:
|
|
298
|
+
batch = [self.rows.get(timeout=5)]
|
|
299
|
+
except queue.Empty:
|
|
300
|
+
batch = []
|
|
301
|
+
deadline = time.monotonic() + 0.25
|
|
302
|
+
while batch and len(batch) < 200:
|
|
303
|
+
try:
|
|
304
|
+
batch.append(self.rows.get(timeout=max(0.0, deadline - time.monotonic())))
|
|
305
|
+
except queue.Empty:
|
|
306
|
+
break
|
|
307
|
+
while True:
|
|
308
|
+
try:
|
|
309
|
+
self.write_batch(db, batch)
|
|
310
|
+
break
|
|
311
|
+
except sqlite3.Error as error:
|
|
312
|
+
print(f'RayTrace collector: runtime_events not written ({error}); retrying', flush=True)
|
|
313
|
+
time.sleep(1)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def serve(endpoint, database):
|
|
317
|
+
os.umask(0o077)
|
|
318
|
+
init_db(database).close()
|
|
319
|
+
server = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
|
|
320
|
+
# Refuse to replace an existing socket: a second collector must not steal it.
|
|
321
|
+
server.bind(endpoint)
|
|
322
|
+
server.listen(32)
|
|
323
|
+
slots = threading.BoundedSemaphore(64)
|
|
324
|
+
writer = Writer(database)
|
|
325
|
+
writer.start()
|
|
326
|
+
engine = None
|
|
327
|
+
if facts_engine:
|
|
328
|
+
try:
|
|
329
|
+
engine = facts_engine.Engine(database)
|
|
330
|
+
engine.start()
|
|
331
|
+
writer.capabilities.update(files=True, network=True)
|
|
332
|
+
except Exception as error:
|
|
333
|
+
print(f'RayTrace collector: fact engine not started ({error}); recording processes only', flush=True)
|
|
334
|
+
|
|
335
|
+
def taken(event):
|
|
336
|
+
"""Whether the engine took the event as its own: a file or network
|
|
337
|
+
call, which is then not written. It sees process events too, to know
|
|
338
|
+
which program and tool call a later call belongs to."""
|
|
339
|
+
return bool(engine and engine.accept(event))
|
|
340
|
+
|
|
341
|
+
def client(conn):
|
|
342
|
+
connection = str(uuid.uuid4())
|
|
343
|
+
container = ''
|
|
344
|
+
def record(event):
|
|
345
|
+
writer.put(connection, event)
|
|
346
|
+
try:
|
|
347
|
+
conn.settimeout(5)
|
|
348
|
+
greeting, _, flags, _ = conn.recvmsg(4096)
|
|
349
|
+
if flags & socket.MSG_TRUNC or one(fields(greeting), 1) != 1:
|
|
350
|
+
raise ValueError('unsupported handshake')
|
|
351
|
+
conn.sendall(b'\x08\x01')
|
|
352
|
+
conn.settimeout(None)
|
|
353
|
+
record({'kind': 'collector_connected', 'source': 'collector'})
|
|
354
|
+
previous_dropped = 0
|
|
355
|
+
previous_received = int(time.time()*1000)
|
|
356
|
+
while True:
|
|
357
|
+
packet, _, flags, _ = conn.recvmsg(MAX_PACKET)
|
|
358
|
+
if not packet:
|
|
359
|
+
break
|
|
360
|
+
if flags & socket.MSG_TRUNC:
|
|
361
|
+
raise ValueError('oversized event')
|
|
362
|
+
event = decode(packet)
|
|
363
|
+
if event['container_id']:
|
|
364
|
+
container = event['container_id']
|
|
365
|
+
if event['dropped_count'] != previous_dropped:
|
|
366
|
+
gap = {'kind': 'collection_gap', 'container_id': event['container_id'],
|
|
367
|
+
'gap_from_ms': previous_received, 'dropped_delta': (event['dropped_count'] - previous_dropped) % (1 << 32)}
|
|
368
|
+
record(gap)
|
|
369
|
+
taken(gap)
|
|
370
|
+
previous_dropped = event['dropped_count']
|
|
371
|
+
previous_received = int(time.time()*1000)
|
|
372
|
+
if not taken(event):
|
|
373
|
+
record(with_raw(event, packet))
|
|
374
|
+
except Exception as error:
|
|
375
|
+
record({'kind': 'collection_error', 'container_id': container, 'error': str(error)[:500]})
|
|
376
|
+
finally:
|
|
377
|
+
# EOF alone does not prove a complete trace; final dropped events
|
|
378
|
+
# may never have been reported in a subsequent packet. Tagged with
|
|
379
|
+
# the sandbox this connection carried (one per sandbox), so a
|
|
380
|
+
# consumer can tell whose open processes will now never report an
|
|
381
|
+
# exit; empty if no event ever arrived.
|
|
382
|
+
gone = {'kind': 'collector_disconnected', 'container_id': container, 'coverage': 'not_certified'}
|
|
383
|
+
record(gone)
|
|
384
|
+
taken(gone)
|
|
385
|
+
conn.close()
|
|
386
|
+
slots.release()
|
|
387
|
+
print('RayTrace collector ready: ' + endpoint, flush=True)
|
|
388
|
+
try:
|
|
389
|
+
while True:
|
|
390
|
+
conn, _ = server.accept()
|
|
391
|
+
if not slots.acquire(blocking=False):
|
|
392
|
+
conn.close()
|
|
393
|
+
continue
|
|
394
|
+
threading.Thread(target=client, args=(conn,), daemon=True).start()
|
|
395
|
+
finally:
|
|
396
|
+
server.close()
|
|
397
|
+
os.unlink(endpoint)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
if __name__ == '__main__':
|
|
401
|
+
parser = argparse.ArgumentParser()
|
|
402
|
+
parser.add_argument('--socket', required=True)
|
|
403
|
+
parser.add_argument('--database', required=True)
|
|
404
|
+
opts = parser.parse_args()
|
|
405
|
+
serve(opts.socket, opts.database)
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""The sandbox network's resolver. Python stdlib only; never mount into guests.
|
|
2
|
+
|
|
3
|
+
Sandboxes resolve names here, on their network's gateway (network.py writes
|
|
4
|
+
their resolv.conf), and each lookup is sent on to the real resolver unchanged
|
|
5
|
+
and answered with whatever it says. Along the way the name and the addresses it
|
|
6
|
+
answered with are written down, which is the only place a host name is seen:
|
|
7
|
+
a program connects to an address, never to a name, and HTTPS is not opened.
|
|
8
|
+
|
|
9
|
+
dns_cache for each sandbox, the name that last answered with an address.
|
|
10
|
+
engine.py reads it to put a name beside a connection. Written
|
|
11
|
+
before the answer goes back, so it is there when the program connects.
|
|
12
|
+
facts one `dns` fact for each name a sandbox looked up, with a count
|
|
13
|
+
|
|
14
|
+
It decides nothing: no name is refused or changed. A lookup it cannot read is
|
|
15
|
+
still forwarded. A sandbox that asks a public resolver directly, or uses DNS
|
|
16
|
+
over HTTPS, is not seen here, and its connections then show bare addresses.
|
|
17
|
+
If this stops, sandboxes cannot resolve names until systemd starts it again.
|
|
18
|
+
"""
|
|
19
|
+
import hashlib
|
|
20
|
+
import ipaddress
|
|
21
|
+
import json
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
import socket
|
|
24
|
+
import socketserver
|
|
25
|
+
import sqlite3
|
|
26
|
+
import struct
|
|
27
|
+
import subprocess
|
|
28
|
+
import threading
|
|
29
|
+
import time
|
|
30
|
+
|
|
31
|
+
import engine
|
|
32
|
+
import network
|
|
33
|
+
|
|
34
|
+
DATABASE = '/var/lib/raytace/runtime.db'
|
|
35
|
+
MAX_NAME = 253
|
|
36
|
+
slots = threading.BoundedSemaphore(64) # lookups in flight; past that a sandbox's packet is dropped and it asks again
|
|
37
|
+
lock = threading.Lock()
|
|
38
|
+
containers = {'read': 0.0, 'by_ip': {}, 'moved': set()}
|
|
39
|
+
facts, dirty = {}, set()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def name_at(message, at):
|
|
43
|
+
"""(the name at `at`, where the next field starts). Follows the pointers
|
|
44
|
+
DNS uses to repeat a name, a bounded number of times."""
|
|
45
|
+
labels, jumps, after = [], 0, None
|
|
46
|
+
while True:
|
|
47
|
+
size = message[at]
|
|
48
|
+
if size & 0xC0 == 0xC0:
|
|
49
|
+
if after is None:
|
|
50
|
+
after = at + 2
|
|
51
|
+
at = ((size & 0x3F) << 8) | message[at + 1]
|
|
52
|
+
jumps += 1
|
|
53
|
+
if jumps > 20:
|
|
54
|
+
raise ValueError('name loops')
|
|
55
|
+
continue
|
|
56
|
+
at += 1
|
|
57
|
+
if size == 0:
|
|
58
|
+
break
|
|
59
|
+
labels.append(message[at:at + size].decode('ascii', errors='replace'))
|
|
60
|
+
at += size
|
|
61
|
+
name = '.'.join(labels).lower()
|
|
62
|
+
if len(name) > MAX_NAME:
|
|
63
|
+
raise ValueError('name too long')
|
|
64
|
+
return name, at if after is None else after
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def lookup(query, answer):
|
|
68
|
+
"""(name asked, record type, result code, addresses answered), or None when either message is not a lookup this reads."""
|
|
69
|
+
try:
|
|
70
|
+
if len(query) < 12 or struct.unpack_from('>H', query, 4)[0] != 1:
|
|
71
|
+
return None
|
|
72
|
+
name, at = name_at(query, 12)
|
|
73
|
+
kind = struct.unpack_from('>H', query, at)[0]
|
|
74
|
+
if len(answer) < 12:
|
|
75
|
+
return name, kind, None, []
|
|
76
|
+
flags, _, count = struct.unpack_from('>HHH', answer, 2)
|
|
77
|
+
_, at = name_at(answer, 12)
|
|
78
|
+
at += 4
|
|
79
|
+
addresses = []
|
|
80
|
+
for _ in range(min(count, 64)):
|
|
81
|
+
_, at = name_at(answer, at)
|
|
82
|
+
record, _, _, size = struct.unpack_from('>HHIH', answer, at)
|
|
83
|
+
at += 10
|
|
84
|
+
if record == 1 and size == 4:
|
|
85
|
+
addresses.append(str(ipaddress.IPv4Address(answer[at:at + 4])))
|
|
86
|
+
elif record == 28 and size == 16:
|
|
87
|
+
addresses.append(str(ipaddress.IPv6Address(answer[at:at + 16])))
|
|
88
|
+
at += size
|
|
89
|
+
return name, kind, flags & 0xF, addresses
|
|
90
|
+
except (IndexError, struct.error, ValueError):
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def container_of(ip):
|
|
95
|
+
"""The sandbox container that holds an address of the sandbox network now,
|
|
96
|
+
as (container id, whether it is a fork), or None. Docker is asked, not a
|
|
97
|
+
record: an address goes to the next container as soon as one stops, and a
|
|
98
|
+
lookup filed by a remembered address would be filed under a sandbox that
|
|
99
|
+
did not make it, which may be another organization's. What Docker said is
|
|
100
|
+
kept for two seconds; a container without a record of projects.py's (a
|
|
101
|
+
throwaway job) is no sandbox."""
|
|
102
|
+
now = time.monotonic()
|
|
103
|
+
if now - containers['read'] > 2:
|
|
104
|
+
try:
|
|
105
|
+
listed = json.loads(subprocess.check_output(
|
|
106
|
+
['docker', 'network', 'inspect', 'raytace-egress', '--format', '{{json .Containers}}'], timeout=5)) or {}
|
|
107
|
+
found = {str(joined.get('IPv4Address', '')).partition('/')[0]: container for container, joined in listed.items()}
|
|
108
|
+
except (subprocess.SubprocessError, OSError, ValueError):
|
|
109
|
+
found = {} # nobody is named when Docker cannot say: a lookup unfiled, never misfiled
|
|
110
|
+
# An address that changed hands: what its last holder looked up is not the new one's.
|
|
111
|
+
before = containers['by_ip']
|
|
112
|
+
containers['moved'].update(address for address in before.keys() | found.keys() if before.get(address) != found.get(address))
|
|
113
|
+
containers.update(read=now, by_ip=found)
|
|
114
|
+
container = containers['by_ip'].get(ip)
|
|
115
|
+
if not container:
|
|
116
|
+
return None
|
|
117
|
+
try:
|
|
118
|
+
record = json.loads((engine.CONTAINERS / (container + '.json')).read_text())
|
|
119
|
+
except (OSError, ValueError):
|
|
120
|
+
return None
|
|
121
|
+
return container, bool(record.get('fork_of'))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def record(db, client, query, answer):
|
|
125
|
+
found = lookup(query, answer)
|
|
126
|
+
if not found or not found[0]:
|
|
127
|
+
return
|
|
128
|
+
name, kind, code, addresses = found
|
|
129
|
+
now = time.time_ns()
|
|
130
|
+
with lock:
|
|
131
|
+
container = container_of(client)
|
|
132
|
+
if containers['moved']:
|
|
133
|
+
db.executemany('DELETE FROM dns_cache WHERE client = ?', [(address,) for address in containers['moved']])
|
|
134
|
+
containers['moved'].clear()
|
|
135
|
+
db.commit()
|
|
136
|
+
if addresses:
|
|
137
|
+
db.executemany('INSERT OR REPLACE INTO dns_cache (client, ip, name, seen_ns) VALUES (?,?,?,?)',
|
|
138
|
+
[(client, address, name, now) for address in addresses])
|
|
139
|
+
db.commit()
|
|
140
|
+
if not container:
|
|
141
|
+
return
|
|
142
|
+
fact_id = hashlib.sha256('\0'.join((container[0], 'dns', name)).encode()).hexdigest()[:32]
|
|
143
|
+
fact = facts.get(fact_id)
|
|
144
|
+
if fact is None:
|
|
145
|
+
fact = facts[fact_id] = {'fact_id': fact_id, 'container_id': container[0], 'kind': 'dns', 'effect': 'looked_up', 'name': name,
|
|
146
|
+
'count': 0, 'first_ns': str(now), 'addresses': [], 'pid': None, 'call_id': None,
|
|
147
|
+
'basis': {'note': 'asked of the sandbox network\'s resolver; which process asked is not known'}}
|
|
148
|
+
if container[1]:
|
|
149
|
+
fact['fork'] = True
|
|
150
|
+
fact['count'] += 1
|
|
151
|
+
fact['last_ns'] = str(now)
|
|
152
|
+
if addresses:
|
|
153
|
+
fact['addresses'] = sorted(set(fact['addresses']) | set(addresses))[:16]
|
|
154
|
+
if code:
|
|
155
|
+
fact['failed'] = {2: 'SERVFAIL', 3: 'NXDOMAIN', 5: 'REFUSED'}.get(code, str(code))
|
|
156
|
+
elif kind in (1, 28):
|
|
157
|
+
fact.pop('failed', None)
|
|
158
|
+
dirty.add(fact_id)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def flush(database):
|
|
162
|
+
"""Writes the lookups that changed, every two seconds: a name asked a
|
|
163
|
+
thousand times is one row written a few times, not a thousand."""
|
|
164
|
+
db = sqlite3.connect(database, timeout=10)
|
|
165
|
+
while True:
|
|
166
|
+
time.sleep(2)
|
|
167
|
+
with lock:
|
|
168
|
+
rows = [(fact_id, facts[fact_id]['container_id'], 'dns', facts[fact_id]['last_ns'], int(time.time() * 1000), json.dumps(facts[fact_id]))
|
|
169
|
+
for fact_id in dirty]
|
|
170
|
+
dirty.clear()
|
|
171
|
+
if len(facts) > 20_000:
|
|
172
|
+
facts.clear()
|
|
173
|
+
if rows:
|
|
174
|
+
try:
|
|
175
|
+
db.executemany('INSERT OR REPLACE INTO facts (fact_id, container_id, kind, at_ns, received_ms, body) VALUES (?,?,?,?,?,?)', rows)
|
|
176
|
+
db.commit()
|
|
177
|
+
except sqlite3.Error as error:
|
|
178
|
+
print(f'RayTrace resolver: lookups not written ({error})', flush=True)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def serve(listen, upstream, database):
|
|
182
|
+
setup = sqlite3.connect(database, timeout=10)
|
|
183
|
+
setup.execute('PRAGMA journal_mode=WAL')
|
|
184
|
+
setup.executescript(engine.SCHEMA)
|
|
185
|
+
setup.close()
|
|
186
|
+
local = threading.local()
|
|
187
|
+
|
|
188
|
+
def db():
|
|
189
|
+
if not hasattr(local, 'db'):
|
|
190
|
+
local.db = sqlite3.connect(database, timeout=10)
|
|
191
|
+
local.db.execute('PRAGMA synchronous=NORMAL')
|
|
192
|
+
return local.db
|
|
193
|
+
|
|
194
|
+
def note(client, query, answer):
|
|
195
|
+
try:
|
|
196
|
+
record(db(), client, query, answer)
|
|
197
|
+
except Exception as error: # writing a lookup down never costs the sandbox its answer
|
|
198
|
+
print(f'RayTrace resolver: {type(error).__name__}: {error}', flush=True)
|
|
199
|
+
|
|
200
|
+
class Datagram(socketserver.BaseRequestHandler):
|
|
201
|
+
def handle(self):
|
|
202
|
+
if not slots.acquire(blocking=False):
|
|
203
|
+
return
|
|
204
|
+
try:
|
|
205
|
+
query, reply_on = self.request
|
|
206
|
+
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as out:
|
|
207
|
+
out.settimeout(5)
|
|
208
|
+
out.sendto(query, (upstream, 53))
|
|
209
|
+
answer = out.recv(65535)
|
|
210
|
+
note(self.client_address[0], query, answer)
|
|
211
|
+
reply_on.sendto(answer, self.client_address)
|
|
212
|
+
except OSError:
|
|
213
|
+
pass # no answer: the sandbox asks again
|
|
214
|
+
finally:
|
|
215
|
+
slots.release()
|
|
216
|
+
|
|
217
|
+
class Stream(socketserver.BaseRequestHandler):
|
|
218
|
+
def read(self, connection):
|
|
219
|
+
head = connection.recv(2, socket.MSG_WAITALL)
|
|
220
|
+
if len(head) < 2:
|
|
221
|
+
return None
|
|
222
|
+
body = connection.recv(struct.unpack('>H', head)[0], socket.MSG_WAITALL)
|
|
223
|
+
return body if len(body) == struct.unpack('>H', head)[0] else None
|
|
224
|
+
|
|
225
|
+
def handle(self):
|
|
226
|
+
if not slots.acquire(blocking=False):
|
|
227
|
+
return
|
|
228
|
+
try:
|
|
229
|
+
self.request.settimeout(5)
|
|
230
|
+
with socket.create_connection((upstream, 53), timeout=5) as out:
|
|
231
|
+
while (query := self.read(self.request)) is not None:
|
|
232
|
+
out.sendall(struct.pack('>H', len(query)) + query)
|
|
233
|
+
answer = self.read(out)
|
|
234
|
+
if answer is None:
|
|
235
|
+
break
|
|
236
|
+
note(self.client_address[0], query, answer)
|
|
237
|
+
self.request.sendall(struct.pack('>H', len(answer)) + answer)
|
|
238
|
+
except OSError:
|
|
239
|
+
pass
|
|
240
|
+
finally:
|
|
241
|
+
slots.release()
|
|
242
|
+
|
|
243
|
+
socketserver.ThreadingUDPServer.allow_reuse_address = socketserver.ThreadingTCPServer.allow_reuse_address = True
|
|
244
|
+
socketserver.ThreadingUDPServer.daemon_threads = socketserver.ThreadingTCPServer.daemon_threads = True
|
|
245
|
+
datagrams = socketserver.ThreadingUDPServer(listen, Datagram)
|
|
246
|
+
streams = socketserver.ThreadingTCPServer(listen, Stream)
|
|
247
|
+
threading.Thread(target=flush, args=(database,), daemon=True).start()
|
|
248
|
+
threading.Thread(target=streams.serve_forever, daemon=True).start()
|
|
249
|
+
print(f'RayTrace resolver ready: {listen[0]}:{listen[1]} -> {upstream}', flush=True)
|
|
250
|
+
datagrams.serve_forever()
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
if __name__ == '__main__':
|
|
254
|
+
Path(DATABASE).parent.mkdir(parents=True, exist_ok=True)
|
|
255
|
+
serve((network.GATEWAY, 53), network.resolver(), DATABASE)
|