raytrace-cli 0.11.1__py3-none-win_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raytrace/__init__.py +1 -0
- raytrace/__main__.py +33 -0
- raytrace/app/LICENSE +21 -0
- raytrace/app/README.md +599 -0
- raytrace/app/bin/raytrace.mjs +297 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-BwCSEQnW.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-CW0RaeGs.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-B9HHJjqV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-Dr3UlScf.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-Aqo67rzb.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-BmdmIIQ2.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-CUSSCpQX.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-DvUfVpUG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BjWJ59Pq.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BtiwyxMk.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-BJfUCQsA.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-DR84L5F-.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-4vooXBpG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-DRtaDpgU.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-BLI3TTWz.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-CJIcxD6K.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-CTOM6hUh.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-fLZuRloM.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-9HEixskS.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-V-xxqcpd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-BgSNZQsw.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-DWFSQ4vo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-D38SheWl.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-DmB0ttJJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-D2EvbN8M.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-iLQfcSjf.woff +0 -0
- raytrace/app/dist/dashboard/assets/index-DepcI8Fi.js +38 -0
- raytrace/app/dist/dashboard/assets/index-H97VLD5l.css +1 -0
- raytrace/app/dist/dashboard/favicon.svg +6 -0
- raytrace/app/dist/dashboard/index.html +15 -0
- raytrace/app/package.json +48 -0
- raytrace/app/proxy/agent-intent-schema.test.mjs +142 -0
- raytrace/app/proxy/call-assessment.mjs +385 -0
- raytrace/app/proxy/call-assessment.test.mjs +167 -0
- raytrace/app/proxy/call-category.mjs +461 -0
- raytrace/app/proxy/capture-health.mjs +30 -0
- raytrace/app/proxy/capture-health.test.mjs +44 -0
- raytrace/app/proxy/capture.mjs +38 -0
- raytrace/app/proxy/capture.test.mjs +49 -0
- raytrace/app/proxy/claude-code-adapter.mjs +301 -0
- raytrace/app/proxy/claude-code-hook.mjs +22 -0
- raytrace/app/proxy/codex-config.mjs +36 -0
- raytrace/app/proxy/codex.mjs +14 -0
- raytrace/app/proxy/evidence-schema.test.mjs +240 -0
- raytrace/app/proxy/evidence-store.mjs +993 -0
- raytrace/app/proxy/evidence-store.test.mjs +301 -0
- raytrace/app/proxy/exec-narrative.mjs +625 -0
- raytrace/app/proxy/exec-narrative.test.mjs +262 -0
- raytrace/app/proxy/execution-correlation.mjs +185 -0
- raytrace/app/proxy/execution-correlation.test.mjs +146 -0
- raytrace/app/proxy/execution-runner.mjs +89 -0
- raytrace/app/proxy/execution-runner.test.mjs +38 -0
- raytrace/app/proxy/experiment-engine.mjs +195 -0
- raytrace/app/proxy/experiment-engine.test.mjs +93 -0
- raytrace/app/proxy/explanations.mjs +36 -0
- raytrace/app/proxy/explanations.test.mjs +20 -0
- raytrace/app/proxy/failure-signature.mjs +56 -0
- raytrace/app/proxy/gvisor-forwarder.mjs +493 -0
- raytrace/app/proxy/gvisor-forwarder.test.mjs +433 -0
- raytrace/app/proxy/init-evidence-db.mjs +227 -0
- raytrace/app/proxy/local-sandboxes.mjs +94 -0
- raytrace/app/proxy/local-token.mjs +30 -0
- raytrace/app/proxy/playground-recovery.test.mjs +130 -0
- raytrace/app/proxy/playground.mjs +439 -0
- raytrace/app/proxy/prompt-context.mjs +278 -0
- raytrace/app/proxy/prompt-traces.mjs +40 -0
- raytrace/app/proxy/prompt-traces.test.mjs +32 -0
- raytrace/app/proxy/providers.mjs +65 -0
- raytrace/app/proxy/providers.test.mjs +76 -0
- raytrace/app/proxy/proxy.integration.test.mjs +135 -0
- raytrace/app/proxy/raytace-proxy.mjs +941 -0
- raytrace/app/proxy/request-metrics.mjs +13 -0
- raytrace/app/proxy/request-metrics.test.mjs +19 -0
- raytrace/app/proxy/request-policy.mjs +45 -0
- raytrace/app/proxy/request-policy.test.mjs +47 -0
- raytrace/app/proxy/run-recap.mjs +139 -0
- raytrace/app/proxy/runtime-attribution.mjs +110 -0
- raytrace/app/proxy/sandbox-bridge.mjs +49 -0
- raytrace/app/proxy/schema/EVIDENCE_DATABASE.md +276 -0
- raytrace/app/proxy/schema/agent_intent.sql +101 -0
- raytrace/app/proxy/schema/auxiliary.sql +90 -0
- raytrace/app/proxy/schema/capture_health.sql +14 -0
- raytrace/app/proxy/schema/classification.sql +89 -0
- raytrace/app/proxy/schema/file_context.sql +53 -0
- raytrace/app/proxy/schema/runtime_evidence.sql +201 -0
- raytrace/app/proxy/schema/search.sql +66 -0
- raytrace/app/proxy/schema/shared_storage.sql +21 -0
- raytrace/app/proxy/search.mjs +418 -0
- raytrace/app/proxy/search.test.mjs +161 -0
- raytrace/app/proxy/smoke.test.mjs +93 -0
- raytrace/app/proxy/step-attribution.mjs +380 -0
- raytrace/app/proxy/step-lab.mjs +78 -0
- raytrace/app/proxy/step-lab.test.mjs +38 -0
- raytrace/app/proxy/summaries.mjs +151 -0
- raytrace/app/proxy/summaries.test.mjs +69 -0
- raytrace/app/proxy/tool-metadata.mjs +111 -0
- raytrace/app/proxy/tool-metadata.test.mjs +38 -0
- raytrace/app/proxy/trace-view.mjs +324 -0
- raytrace/app/proxy/trace-view.test.mjs +92 -0
- raytrace/app/proxy/typesafe.mjs +63 -0
- raytrace/app/proxy/typesafe.test.mjs +39 -0
- raytrace/app/src/agents.mjs +7 -0
- raytrace/app/src/api.mjs +57 -0
- raytrace/app/src/auth.mjs +205 -0
- raytrace/app/src/beta.mjs +96 -0
- raytrace/app/src/claude-code.mjs +93 -0
- raytrace/app/src/commands.mjs +285 -0
- raytrace/app/src/config.mjs +49 -0
- raytrace/app/src/export.mjs +200 -0
- raytrace/app/src/lima.mjs +176 -0
- raytrace/app/src/local-sandbox.mjs +358 -0
- raytrace/app/src/mcp.mjs +688 -0
- raytrace/app/src/paths.mjs +33 -0
- raytrace/app/src/prompt.mjs +42 -0
- raytrace/app/src/proxy-process.mjs +143 -0
- raytrace/app/src/sandbox.mjs +306 -0
- raytrace/app/src/share.mjs +86 -0
- raytrace/app/src/terminal.mjs +68 -0
- raytrace/app/src/upload.mjs +198 -0
- raytrace/app/worker/sandbox/Dockerfile +8 -0
- raytrace/app/worker/sandbox/SYSCALL_CATALOG.md +197 -0
- raytrace/app/worker/sandbox/claude-hook.mjs +364 -0
- raytrace/app/worker/sandbox/collector.py +405 -0
- raytrace/app/worker/sandbox/dns_forwarder.py +255 -0
- raytrace/app/worker/sandbox/engine.py +940 -0
- raytrace/app/worker/sandbox/file_scan.py +491 -0
- raytrace/app/worker/sandbox/lima.yaml +29 -0
- raytrace/app/worker/sandbox/network.py +90 -0
- raytrace/app/worker/sandbox/projects.py +702 -0
- raytrace/app/worker/sandbox/setup.py +159 -0
- raytrace/app/worker/sandbox/syscall_catalog.sql +985 -0
- raytrace/app/worker/sandbox/transcript-delivery.test.mjs +153 -0
- raytrace/app/worker/sandbox/viewer.py +195 -0
- raytrace/node/LICENSE +2946 -0
- raytrace/node/node.exe +4 -0
- raytrace_cli-0.11.1.dist-info/LICENSE +21 -0
- raytrace_cli-0.11.1.dist-info/METADATA +42 -0
- raytrace_cli-0.11.1.dist-info/RECORD +165 -0
- raytrace_cli-0.11.1.dist-info/WHEEL +4 -0
- raytrace_cli-0.11.1.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,940 @@
|
|
|
1
|
+
"""The sandbox host's fact engine. Python stdlib only; never mount into guests.
|
|
2
|
+
|
|
3
|
+
gVisor reports every open, connect, delete and rename a sandbox makes. That is
|
|
4
|
+
far too much to keep and most of it concerns nobody: one `npm install` opens
|
|
5
|
+
tens of thousands of files. The engine looks at each event once, in memory,
|
|
6
|
+
keeps what matters as a fact, counts the rest and stores no event at all.
|
|
7
|
+
|
|
8
|
+
What matters:
|
|
9
|
+
|
|
10
|
+
files a file the project's owner restricted or watches, and a new file
|
|
11
|
+
whose path marks it sensitive (a .env, a key): read, created,
|
|
12
|
+
changed, emptied, appended, deleted, renamed, or an attempt
|
|
13
|
+
the kernel refused
|
|
14
|
+
network every connection out of the sandbox, one fact for each
|
|
15
|
+
destination a step reached, and whether the firewall refuses it
|
|
16
|
+
packages what a package manager left installed once it had run
|
|
17
|
+
identity a change of user or group id, which a sandbox should never make
|
|
18
|
+
|
|
19
|
+
A fact combines three witnesses, and says which it had (`basis`, `certainty`):
|
|
20
|
+
|
|
21
|
+
gVisor who asked for what, and when (collector.py). For an open it
|
|
22
|
+
gives the path and flags; for a delete or rename only the
|
|
23
|
+
number of the call and whether it was refused.
|
|
24
|
+
the host what the file is before and after: size, time, inode, SHA-256,
|
|
25
|
+
read from the sandbox's volume by this process, not the agent.
|
|
26
|
+
a watcher inotify on the folders of those files, which names the file a
|
|
27
|
+
delete or rename hit; it is matched to gVisor's call by time.
|
|
28
|
+
|
|
29
|
+
confirmed a call was seen and the file shows its effect
|
|
30
|
+
attempted the kernel refused the call
|
|
31
|
+
possible opened for writing, and the file is as it was
|
|
32
|
+
one_of more than one process made such a call at that moment
|
|
33
|
+
unattributed the file changed and no call was seen for it
|
|
34
|
+
reported read from files the agent could have written (packages)
|
|
35
|
+
|
|
36
|
+
Each fact names its process and, when the process carries one, the tool call
|
|
37
|
+
(RAYTRACE_CALL_ID, as the forwarder reads it). Facts go to the `facts` table of
|
|
38
|
+
the collector's database, where viewer.py serves them and the control plane
|
|
39
|
+
(facts.py) collects them. Which calls arrive at all is syscall_catalog.sql's
|
|
40
|
+
`capture` column, through setup.py.
|
|
41
|
+
"""
|
|
42
|
+
import collections
|
|
43
|
+
import ctypes
|
|
44
|
+
import hashlib
|
|
45
|
+
import ipaddress
|
|
46
|
+
import json
|
|
47
|
+
import os
|
|
48
|
+
from pathlib import Path, PurePosixPath
|
|
49
|
+
import platform
|
|
50
|
+
import posixpath
|
|
51
|
+
import queue
|
|
52
|
+
import re
|
|
53
|
+
import sqlite3
|
|
54
|
+
import stat
|
|
55
|
+
import struct
|
|
56
|
+
import threading
|
|
57
|
+
import time
|
|
58
|
+
|
|
59
|
+
import file_scan
|
|
60
|
+
import network
|
|
61
|
+
|
|
62
|
+
CATALOG = Path(__file__).with_name('syscall_catalog.sql')
|
|
63
|
+
# Which sandbox a container is, and where its volumes are (projects.py writes one file each).
|
|
64
|
+
CONTAINERS = Path('/var/lib/raytace/containers')
|
|
65
|
+
# Each sandbox's restricted and watched files (projects.py guardrails()).
|
|
66
|
+
GUARDRAILS = Path('/var/lib/raytace/guardrails')
|
|
67
|
+
BRIDGE = Path('/etc/raytace/bridge')
|
|
68
|
+
WORKSPACE = '/workspace/'
|
|
69
|
+
# Large, regenerable folders: the same ones the sandbox hook leaves out of its snapshots.
|
|
70
|
+
SKIPPED = frozenset(['node_modules', '.git', '.next', '.vinext', 'dist', 'build', 'out', 'coverage', '.cache', '.turbo',
|
|
71
|
+
'.venv', 'venv', '__pycache__', '.pytest_cache', '.mypy_cache', 'target', '.gradle', '.wrangler'])
|
|
72
|
+
SCHEMA = '''
|
|
73
|
+
CREATE TABLE IF NOT EXISTS facts (id INTEGER PRIMARY KEY AUTOINCREMENT, fact_id TEXT NOT NULL UNIQUE,
|
|
74
|
+
container_id TEXT NOT NULL, kind TEXT NOT NULL, at_ns TEXT NOT NULL, received_ms INTEGER NOT NULL, body TEXT NOT NULL);
|
|
75
|
+
CREATE INDEX IF NOT EXISTS facts_container ON facts(container_id, id);
|
|
76
|
+
CREATE TABLE IF NOT EXISTS dns_cache (client TEXT NOT NULL, ip TEXT NOT NULL, name TEXT NOT NULL, seen_ns INTEGER NOT NULL,
|
|
77
|
+
PRIMARY KEY (client, ip));
|
|
78
|
+
'''
|
|
79
|
+
AT_FDCWD = -100
|
|
80
|
+
O_ACCMODE, O_CREAT, O_TRUNC, O_APPEND = 3, 0o100, 0o1000, 0o2000
|
|
81
|
+
REFUSED = {1: 'EPERM', 13: 'EACCES', 30: 'EROFS'}
|
|
82
|
+
ERRNO = {**REFUSED, 2: 'ENOENT', 101: 'ENETUNREACH', 110: 'ETIMEDOUT', 111: 'ECONNREFUSED', 113: 'EHOSTUNREACH', 115: 'EINPROGRESS'}
|
|
83
|
+
EINPROGRESS = 115
|
|
84
|
+
MAX_HASHED = 5 * 1024 * 1024 # a larger file is compared by size, time and inode
|
|
85
|
+
HASH_LIMIT = 2000 # with more files than this to keep, none is hashed up front
|
|
86
|
+
SENSITIVE_FROM = 4 # file_scan's level from which a new file's path alone is worth a fact
|
|
87
|
+
SETTLE = 0.3 # seconds between a change seen on disk and looking: gVisor's call may still be on its way
|
|
88
|
+
WRITER_LIFE = 900 # how long an open for writing may explain a later change
|
|
89
|
+
RAW_WINDOW = (2_000_000_000, 1_000_000_000) # ns a delete or rename call may precede, or follow, the watcher's notice
|
|
90
|
+
BULK = 200 # opens from which a step is worth one summary line
|
|
91
|
+
RETAIN_DAYS, RETAIN_ROWS = 14, 50_000
|
|
92
|
+
PACKAGE_TOOLS = frozenset(['npm', 'npx', 'pnpm', 'yarn', 'bun', 'pip', 'pip3', 'uv', 'poetry'])
|
|
93
|
+
REGISTRIES = frozenset(['registry.npmjs.org', 'registry.yarnpkg.com', 'pypi.org', 'files.pythonhosted.org'])
|
|
94
|
+
MAX_PACKAGES = 400
|
|
95
|
+
REJECTED = [ipaddress.ip_network(block) for block in network.REJECTED]
|
|
96
|
+
OWN_KINDS = frozenset(['open', 'connect', 'listen', 'setid'])
|
|
97
|
+
PROCESS_KINDS = frozenset(['exec_succeeded', 'clone', 'process_exit', 'collection_gap', 'collector_disconnected'])
|
|
98
|
+
CALL_MARKER = re.compile(r'(?:^|[\s;])export RAYTRACE_CALL_ID=([A-Za-z0-9_-]{1,128})(?=[\s;]|$)')
|
|
99
|
+
SANDBOX_ID = re.compile(r'rt[pf]-[a-f0-9]{32}')
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def open_inside(root, path, directory=False):
|
|
103
|
+
"""A descriptor for a file below `root`, every step of the way opened
|
|
104
|
+
without following links: the agent names its own folders, and a link out
|
|
105
|
+
of the project is never read."""
|
|
106
|
+
at = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
|
|
107
|
+
try:
|
|
108
|
+
parts = PurePosixPath(path).parts
|
|
109
|
+
for index, part in enumerate(parts):
|
|
110
|
+
last = index == len(parts) - 1
|
|
111
|
+
below = os.open(part, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | (os.O_DIRECTORY if directory or not last else 0), dir_fd=at)
|
|
112
|
+
os.close(at)
|
|
113
|
+
at = below
|
|
114
|
+
return at
|
|
115
|
+
except BaseException:
|
|
116
|
+
os.close(at)
|
|
117
|
+
raise
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def photo(root, path, hashed=True):
|
|
121
|
+
"""What a project file is now: None when it is not there (or is no regular file)."""
|
|
122
|
+
try:
|
|
123
|
+
fd = open_inside(root, path)
|
|
124
|
+
except (OSError, ValueError):
|
|
125
|
+
return None
|
|
126
|
+
try:
|
|
127
|
+
info = os.fstat(fd)
|
|
128
|
+
if not stat.S_ISREG(info.st_mode):
|
|
129
|
+
return None
|
|
130
|
+
found = {'size': info.st_size, 'mtime_ns': info.st_mtime_ns, 'inode': info.st_ino, 'mode': stat.S_IMODE(info.st_mode), 'sha256': None}
|
|
131
|
+
if hashed and info.st_size <= MAX_HASHED:
|
|
132
|
+
digest = hashlib.sha256()
|
|
133
|
+
while chunk := os.read(fd, 1024 * 1024):
|
|
134
|
+
digest.update(chunk)
|
|
135
|
+
found['sha256'] = digest.hexdigest()
|
|
136
|
+
return found
|
|
137
|
+
except OSError:
|
|
138
|
+
return None
|
|
139
|
+
finally:
|
|
140
|
+
os.close(fd)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def read_inside(root, path, limit):
|
|
144
|
+
"""A file's bytes, or None when it is absent, no regular file or over `limit`."""
|
|
145
|
+
try:
|
|
146
|
+
fd = open_inside(root, path)
|
|
147
|
+
except (OSError, ValueError):
|
|
148
|
+
return None
|
|
149
|
+
try:
|
|
150
|
+
info = os.fstat(fd)
|
|
151
|
+
if not stat.S_ISREG(info.st_mode) or info.st_size > limit:
|
|
152
|
+
return None
|
|
153
|
+
return b''.join(iter(lambda: os.read(fd, 1024 * 1024), b''))
|
|
154
|
+
except OSError:
|
|
155
|
+
return None
|
|
156
|
+
finally:
|
|
157
|
+
os.close(fd)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def names_inside(root, path):
|
|
161
|
+
try:
|
|
162
|
+
fd = open_inside(root, path, directory=True)
|
|
163
|
+
except (OSError, ValueError):
|
|
164
|
+
return []
|
|
165
|
+
try:
|
|
166
|
+
return os.listdir(fd)
|
|
167
|
+
except OSError:
|
|
168
|
+
return []
|
|
169
|
+
finally:
|
|
170
|
+
os.close(fd)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def change(before, after, events, flags):
|
|
174
|
+
"""What happened to a file between two photos, or None when nothing did."""
|
|
175
|
+
if after is None:
|
|
176
|
+
return None if before is None else 'renamed' if 'moved_from' in events else 'deleted'
|
|
177
|
+
if before is None:
|
|
178
|
+
return 'created'
|
|
179
|
+
if before['sha256'] and after['sha256']:
|
|
180
|
+
same = before['sha256'] == after['sha256']
|
|
181
|
+
else:
|
|
182
|
+
same = (before['size'], before['mtime_ns'], before['inode']) == (after['size'], after['mtime_ns'], after['inode'])
|
|
183
|
+
if same:
|
|
184
|
+
return 'permissions_changed' if before['mode'] != after['mode'] else None
|
|
185
|
+
if after['size'] == 0:
|
|
186
|
+
return 'emptied'
|
|
187
|
+
if flags & O_APPEND and after['size'] > before['size']:
|
|
188
|
+
return 'appended'
|
|
189
|
+
return 'changed'
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def project_path(event):
|
|
193
|
+
"""The project file an open names, relative to the project, or None when
|
|
194
|
+
it is outside it. Worked out from the text of the path: links inside the
|
|
195
|
+
project are not followed, which a fact says as `resolution: lexical`."""
|
|
196
|
+
name = event.get('pathname') or ''
|
|
197
|
+
if not name:
|
|
198
|
+
return None
|
|
199
|
+
if not name.startswith('/'):
|
|
200
|
+
base = event.get('cwd') if event.get('dirfd', AT_FDCWD) == AT_FDCWD else event.get('fd_path')
|
|
201
|
+
if not base or not base.startswith('/'):
|
|
202
|
+
return None
|
|
203
|
+
name = base + '/' + name
|
|
204
|
+
name = posixpath.normpath(name)
|
|
205
|
+
return name[len(WORKSPACE):] if name.startswith(WORKSPACE) else None
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def installed(workspace, home):
|
|
209
|
+
"""{(manager, name, version): where it came from} as the package managers'
|
|
210
|
+
own records on the volume have it. The agent can write those files, so a
|
|
211
|
+
fact drawn from them is `reported`, not seen."""
|
|
212
|
+
found = {}
|
|
213
|
+
roots = [''] + [name for name in names_inside(workspace, '') if name not in SKIPPED and not name.startswith('.')][:50]
|
|
214
|
+
for root in roots:
|
|
215
|
+
modules = posixpath.join(root, 'node_modules')
|
|
216
|
+
lock = read_inside(workspace, posixpath.join(modules, '.package-lock.json'), 32 * 1024 * 1024)
|
|
217
|
+
try:
|
|
218
|
+
listed = json.loads(lock)['packages'] if lock else None
|
|
219
|
+
except (ValueError, KeyError, TypeError):
|
|
220
|
+
listed = None
|
|
221
|
+
if isinstance(listed, dict):
|
|
222
|
+
for key, entry in listed.items():
|
|
223
|
+
if isinstance(key, str) and isinstance(entry, dict) and isinstance(entry.get('version'), str):
|
|
224
|
+
found[('npm', key.rsplit('node_modules/', 1)[-1][:200], entry['version'][:80])] = str(entry.get('resolved') or '')[:300] or None
|
|
225
|
+
continue
|
|
226
|
+
# No npm record (yarn, pnpm, bun): the packages at the top of node_modules.
|
|
227
|
+
for name in names_inside(workspace, modules)[:1000]:
|
|
228
|
+
for package in ([posixpath.join(name, inner) for inner in names_inside(workspace, posixpath.join(modules, name))[:200]]
|
|
229
|
+
if name.startswith('@') else [name]):
|
|
230
|
+
text = read_inside(workspace, posixpath.join(modules, package, 'package.json'), 1024 * 1024)
|
|
231
|
+
try:
|
|
232
|
+
version = json.loads(text)['version'] if text else None
|
|
233
|
+
except (ValueError, KeyError, TypeError):
|
|
234
|
+
version = None
|
|
235
|
+
if isinstance(version, str):
|
|
236
|
+
found[('node', package[:200], version[:80])] = None
|
|
237
|
+
sites = [(workspace, posixpath.join(env, 'lib')) for env in ('.venv', 'venv')] + ([(home, '.local/lib')] if home else [])
|
|
238
|
+
for volume, lib in sites:
|
|
239
|
+
for python in names_inside(volume, lib)[:10]:
|
|
240
|
+
for name in names_inside(volume, posixpath.join(lib, python, 'site-packages')):
|
|
241
|
+
if name.endswith('.dist-info') and '-' in name:
|
|
242
|
+
package, _, version = name[:-len('.dist-info')].rpartition('-')
|
|
243
|
+
found[('pip', package[:200], version[:80])] = None
|
|
244
|
+
return found
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class Inotify:
|
|
248
|
+
"""The host kernel's file notifications, for the folders of the files
|
|
249
|
+
worth a fact. It names the file a delete or rename hit, which gVisor does not."""
|
|
250
|
+
CLOSE_WRITE, MODIFY, ATTRIB, CREATE, DELETE, MOVED_FROM, MOVED_TO = 0x8, 0x2, 0x4, 0x100, 0x200, 0x40, 0x80
|
|
251
|
+
DELETE_SELF, MOVE_SELF, OVERFLOW, IGNORED, ONLYDIR = 0x400, 0x800, 0x4000, 0x8000, 0x01000000
|
|
252
|
+
WORDS = [(CLOSE_WRITE, 'close_write'), (MODIFY, 'modify'), (ATTRIB, 'attrib'), (CREATE, 'create'), (DELETE, 'delete'),
|
|
253
|
+
(MOVED_FROM, 'moved_from'), (MOVED_TO, 'moved_to')]
|
|
254
|
+
FOLDER = CLOSE_WRITE | MODIFY | ATTRIB | CREATE | DELETE | MOVED_FROM | MOVED_TO | DELETE_SELF | MOVE_SELF | ONLYDIR
|
|
255
|
+
|
|
256
|
+
def __init__(self):
|
|
257
|
+
self.libc = ctypes.CDLL(None, use_errno=True)
|
|
258
|
+
self.fd = self.libc.inotify_init1(os.O_NONBLOCK | os.O_CLOEXEC) # AttributeError where there is no inotify
|
|
259
|
+
if self.fd < 0:
|
|
260
|
+
raise OSError(ctypes.get_errno(), 'inotify_init1')
|
|
261
|
+
|
|
262
|
+
def add(self, root, folder):
|
|
263
|
+
"""Watches a folder of a project, reached without following links."""
|
|
264
|
+
fd = open_inside(root, folder, directory=True)
|
|
265
|
+
try:
|
|
266
|
+
watch = self.libc.inotify_add_watch(self.fd, f'/proc/self/fd/{fd}'.encode(), self.FOLDER)
|
|
267
|
+
if watch < 0:
|
|
268
|
+
raise OSError(ctypes.get_errno(), 'inotify_add_watch')
|
|
269
|
+
return watch
|
|
270
|
+
finally:
|
|
271
|
+
os.close(fd)
|
|
272
|
+
|
|
273
|
+
def remove(self, watch):
|
|
274
|
+
self.libc.inotify_rm_watch(self.fd, watch)
|
|
275
|
+
|
|
276
|
+
def read(self):
|
|
277
|
+
try:
|
|
278
|
+
data = os.read(self.fd, 256 * 1024)
|
|
279
|
+
except BlockingIOError:
|
|
280
|
+
return []
|
|
281
|
+
events, at = [], 0
|
|
282
|
+
while at + 16 <= len(data):
|
|
283
|
+
watch, mask, cookie, size = struct.unpack_from('iIII', data, at)
|
|
284
|
+
events.append((watch, mask, cookie, os.fsdecode(data[at + 16:at + 16 + size].split(b'\0', 1)[0])))
|
|
285
|
+
at += 16 + size
|
|
286
|
+
return events
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
class Sandbox:
|
|
290
|
+
"""What the engine holds of one container, all of it in memory."""
|
|
291
|
+
def __init__(self, container):
|
|
292
|
+
self.container = container
|
|
293
|
+
self.sid = self.workspace = self.home = self.ip = None
|
|
294
|
+
self.fork = False
|
|
295
|
+
self.mapped = self.guarded_at = None # the two files' times, as last read
|
|
296
|
+
self.restricted, self.watched, self.sensitive = set(), set(), set()
|
|
297
|
+
self.guarded = {} # path -> restricted | watched | sensitive
|
|
298
|
+
self.hashing = True
|
|
299
|
+
self.photos = {} # path -> its last photo; None: it was not there
|
|
300
|
+
self.known = None # project paths that exist, to tell a new file from an old one
|
|
301
|
+
self.processes = {} # pid -> {start_ns, ppid, name, argv, call_id}
|
|
302
|
+
self.writers = {} # path -> opens for writing that may explain a change
|
|
303
|
+
self.due = {} # path -> when to look at it next (monotonic)
|
|
304
|
+
self.final = {} # path -> when to stop waiting for a change
|
|
305
|
+
self.causes = {} # path -> what the watcher saw of it since the last look
|
|
306
|
+
self.raw = collections.deque(maxlen=512) # recent delete, rename, resize and mode calls: they carry no path
|
|
307
|
+
self.watches = {} # folder -> inotify watch
|
|
308
|
+
self.counts = {} # step -> opens that were only counted
|
|
309
|
+
self.packages = None # what was installed when last looked
|
|
310
|
+
self.package_due = self.package_who = None
|
|
311
|
+
self.registry_at = float('-inf') # when a package registry was last connected to
|
|
312
|
+
self.gap_at = float('-inf') # when events were last lost
|
|
313
|
+
self.names = {} # ip -> (host name, when read)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
class Engine(threading.Thread):
|
|
317
|
+
def __init__(self, database, arch=None, watch=True, clock=time.monotonic):
|
|
318
|
+
super().__init__(daemon=True, name='raytace-engine')
|
|
319
|
+
self.database = database
|
|
320
|
+
self.clock = clock
|
|
321
|
+
catalog = sqlite3.connect(':memory:')
|
|
322
|
+
try:
|
|
323
|
+
catalog.executescript(CATALOG.read_text())
|
|
324
|
+
# number -> (name, rule) for the calls the catalog captures on this architecture.
|
|
325
|
+
self.calls = {number: (name, rule) for number, name, rule in catalog.execute(
|
|
326
|
+
"SELECT n.sysno, c.syscall_name, c.effect_rule FROM syscall_catalog AS c JOIN syscall_numbers AS n "
|
|
327
|
+
"ON n.syscall_name = c.syscall_name AND n.arch = ? WHERE c.capture IN ('yes', 'watcher')",
|
|
328
|
+
(arch or {'arm64': 'aarch64'}.get(platform.machine(), platform.machine()),))}
|
|
329
|
+
finally:
|
|
330
|
+
catalog.close()
|
|
331
|
+
# Calls gVisor reports by number only: taken from the collector whole.
|
|
332
|
+
self.numbered = frozenset(number for number, (_, rule) in self.calls.items() if rule in ('remove', 'rename', 'resize', 'mkdir', 'metadata'))
|
|
333
|
+
self.inbox = queue.Queue(maxsize=200_000)
|
|
334
|
+
self.overflowed = float('-inf')
|
|
335
|
+
self.bridge = ('', float('-inf')) # RayTrace's own bridge as sandboxes reach it, and when that was read
|
|
336
|
+
self.boxes = {}
|
|
337
|
+
self.facts = {} # fact id -> the fact, while it may still grow
|
|
338
|
+
self.dirty = {} # fact ids to write
|
|
339
|
+
self.inotify = None
|
|
340
|
+
self.watched = {} # inotify watch -> (sandbox, folder)
|
|
341
|
+
if watch:
|
|
342
|
+
try:
|
|
343
|
+
self.inotify = Inotify()
|
|
344
|
+
except (AttributeError, OSError) as error:
|
|
345
|
+
print(f'RayTrace engine: no file watcher ({error}); deletes and renames will not be named', flush=True)
|
|
346
|
+
self.db = None
|
|
347
|
+
self.ticked = self.flushed = self.swept = 0.0
|
|
348
|
+
self.errors = 0
|
|
349
|
+
|
|
350
|
+
# ------------------------------------------------------------ from the collector
|
|
351
|
+
|
|
352
|
+
def accept(self, event):
|
|
353
|
+
"""Called on a sandbox's reader thread for every event: queues it and
|
|
354
|
+
answers whether it is the engine's own (a file or network call), which
|
|
355
|
+
the collector then does not write. Never blocks and never raises."""
|
|
356
|
+
try:
|
|
357
|
+
kind = event.get('kind')
|
|
358
|
+
own = kind in OWN_KINDS or (kind in ('syscall_return', 'syscall_enter') and event.get('sysno') in self.numbered)
|
|
359
|
+
if own or kind in PROCESS_KINDS:
|
|
360
|
+
try:
|
|
361
|
+
self.inbox.put_nowait(event)
|
|
362
|
+
except queue.Full:
|
|
363
|
+
self.overflowed = self.clock() # said on every fact for a minute: coverage has a gap
|
|
364
|
+
return own
|
|
365
|
+
except Exception:
|
|
366
|
+
return False
|
|
367
|
+
|
|
368
|
+
def run(self):
|
|
369
|
+
self.db = sqlite3.connect(self.database, timeout=10)
|
|
370
|
+
self.db.execute('PRAGMA journal_mode=WAL')
|
|
371
|
+
self.db.executescript(SCHEMA)
|
|
372
|
+
while True:
|
|
373
|
+
try:
|
|
374
|
+
busy = self.turn()
|
|
375
|
+
except Exception as error:
|
|
376
|
+
busy = False
|
|
377
|
+
self.errors += 1
|
|
378
|
+
if self.errors <= 20:
|
|
379
|
+
print(f'RayTrace engine: {type(error).__name__}: {error}', flush=True)
|
|
380
|
+
if not busy:
|
|
381
|
+
time.sleep(0.02)
|
|
382
|
+
|
|
383
|
+
def turn(self):
|
|
384
|
+
"""One pass: the events waiting, the watcher, then whatever is due."""
|
|
385
|
+
handled = 0
|
|
386
|
+
while handled < 5000:
|
|
387
|
+
try:
|
|
388
|
+
event = self.inbox.get_nowait()
|
|
389
|
+
except queue.Empty:
|
|
390
|
+
break
|
|
391
|
+
handled += 1
|
|
392
|
+
try:
|
|
393
|
+
self.handle(event)
|
|
394
|
+
except Exception as error:
|
|
395
|
+
self.errors += 1
|
|
396
|
+
if self.errors <= 20:
|
|
397
|
+
print(f'RayTrace engine: {event.get("kind")}: {type(error).__name__}: {error}', flush=True)
|
|
398
|
+
seen = self.watcher_events()
|
|
399
|
+
now = self.clock()
|
|
400
|
+
for box in list(self.boxes.values()):
|
|
401
|
+
for path in [path for path, at in box.due.items() if at <= now]:
|
|
402
|
+
del box.due[path]
|
|
403
|
+
self.look(box, path)
|
|
404
|
+
for path in [path for path, at in box.final.items() if at <= now and path not in box.due]:
|
|
405
|
+
del box.final[path]
|
|
406
|
+
self.look(box, path, final=True)
|
|
407
|
+
if box.package_due and box.package_due <= now:
|
|
408
|
+
box.package_due = None
|
|
409
|
+
self.packages(box)
|
|
410
|
+
if now - self.ticked >= 1:
|
|
411
|
+
self.ticked = now
|
|
412
|
+
for box in list(self.boxes.values()):
|
|
413
|
+
self.refresh(box)
|
|
414
|
+
self.summaries(box, now)
|
|
415
|
+
if now - self.flushed >= 1:
|
|
416
|
+
self.flushed = now
|
|
417
|
+
self.flush()
|
|
418
|
+
if now - self.swept >= 3600:
|
|
419
|
+
self.swept = now
|
|
420
|
+
self.sweep()
|
|
421
|
+
return bool(handled or seen)
|
|
422
|
+
|
|
423
|
+
def handle(self, event):
|
|
424
|
+
container = event.get('container_id')
|
|
425
|
+
if not container:
|
|
426
|
+
return
|
|
427
|
+
kind = event['kind']
|
|
428
|
+
if kind == 'collector_disconnected':
|
|
429
|
+
box = self.boxes.pop(container, None)
|
|
430
|
+
if box:
|
|
431
|
+
self.close(box)
|
|
432
|
+
return
|
|
433
|
+
box = self.boxes.get(container)
|
|
434
|
+
if box is None:
|
|
435
|
+
box = self.boxes[container] = Sandbox(container)
|
|
436
|
+
self.refresh(box)
|
|
437
|
+
if kind == 'open':
|
|
438
|
+
self.opened(box, event)
|
|
439
|
+
elif kind in ('connect', 'listen'):
|
|
440
|
+
self.connected(box, event)
|
|
441
|
+
elif kind == 'setid':
|
|
442
|
+
self.identity(box, event)
|
|
443
|
+
elif kind == 'syscall_return':
|
|
444
|
+
self.numbered_call(box, event)
|
|
445
|
+
elif kind == 'exec_succeeded':
|
|
446
|
+
self.executed(box, event)
|
|
447
|
+
elif kind == 'clone':
|
|
448
|
+
child, parent = event.get('child_pid'), box.processes.get(event.get('pid'))
|
|
449
|
+
if child and child != event.get('pid'):
|
|
450
|
+
box.processes[child] = {**(parent or {'name': event.get('process_name'), 'argv': [], 'call_id': None}),
|
|
451
|
+
'ppid': event.get('pid'), 'start_ns': event.get('child_start_ns')}
|
|
452
|
+
elif kind == 'process_exit':
|
|
453
|
+
self.exited(box, event)
|
|
454
|
+
elif kind == 'collection_gap':
|
|
455
|
+
box.gap_at = self.clock()
|
|
456
|
+
|
|
457
|
+
# ------------------------------------------------------------ sandboxes and their lists
|
|
458
|
+
|
|
459
|
+
def refresh(self, box):
|
|
460
|
+
"""Reads again, when either changed, which sandbox the container is and
|
|
461
|
+
which of its files to keep facts of."""
|
|
462
|
+
mapping = CONTAINERS / (box.container + '.json')
|
|
463
|
+
try:
|
|
464
|
+
mapped = mapping.stat().st_mtime_ns
|
|
465
|
+
except OSError:
|
|
466
|
+
mapped = None
|
|
467
|
+
if mapped != box.mapped:
|
|
468
|
+
box.mapped = mapped
|
|
469
|
+
try:
|
|
470
|
+
found = json.loads(mapping.read_text()) if mapped else {}
|
|
471
|
+
except (OSError, ValueError):
|
|
472
|
+
found = {}
|
|
473
|
+
sid = found.get('id') if SANDBOX_ID.fullmatch(str(found.get('id') or '')) else None
|
|
474
|
+
workspace = found.get('workspace') if sid and os.path.isdir(str(found.get('workspace') or '')) else None
|
|
475
|
+
fresh = workspace != box.workspace
|
|
476
|
+
box.sid, box.workspace, box.home, box.ip = sid, workspace, found.get('home') or None, found.get('ip') or None
|
|
477
|
+
box.fork = bool(found.get('fork_of'))
|
|
478
|
+
if fresh:
|
|
479
|
+
box.guarded_at, box.known, box.packages = None, None, None
|
|
480
|
+
if workspace:
|
|
481
|
+
box.known = set()
|
|
482
|
+
for folder, folders, files in os.walk(workspace):
|
|
483
|
+
folders[:] = [name for name in folders if name not in SKIPPED]
|
|
484
|
+
inside = os.path.relpath(folder, workspace)
|
|
485
|
+
box.known.update(name if inside == '.' else inside + '/' + name for name in files)
|
|
486
|
+
if len(box.known) > 200_000:
|
|
487
|
+
break
|
|
488
|
+
box.packages = installed(workspace, box.home)
|
|
489
|
+
if not box.workspace:
|
|
490
|
+
return
|
|
491
|
+
lists = GUARDRAILS / (box.sid + '.json')
|
|
492
|
+
try:
|
|
493
|
+
at = lists.stat().st_mtime_ns
|
|
494
|
+
except OSError:
|
|
495
|
+
at = 0
|
|
496
|
+
if at != box.guarded_at:
|
|
497
|
+
box.guarded_at = at
|
|
498
|
+
try:
|
|
499
|
+
kept = json.loads(lists.read_text()) if at else {}
|
|
500
|
+
except (OSError, ValueError):
|
|
501
|
+
kept = {}
|
|
502
|
+
box.restricted = {path for path in kept.get('restricted') or [] if isinstance(path, str)}
|
|
503
|
+
box.watched = {path for path in kept.get('watched') or [] if isinstance(path, str)}
|
|
504
|
+
self.guard(box)
|
|
505
|
+
self.arm(box)
|
|
506
|
+
|
|
507
|
+
def guard(self, box):
|
|
508
|
+
"""The files to keep facts of, each with a first photo to compare later ones with."""
|
|
509
|
+
guarded = {path: 'sensitive' for path in box.sensitive}
|
|
510
|
+
guarded.update({path: 'watched' for path in box.watched})
|
|
511
|
+
guarded.update({path: 'restricted' for path in box.restricted})
|
|
512
|
+
box.hashing = len(guarded) <= HASH_LIMIT
|
|
513
|
+
for path in guarded:
|
|
514
|
+
if path not in box.photos:
|
|
515
|
+
box.photos[path] = photo(box.workspace, path, box.hashing)
|
|
516
|
+
for path in [path for path in box.photos if path not in guarded]:
|
|
517
|
+
del box.photos[path]
|
|
518
|
+
box.guarded = guarded
|
|
519
|
+
|
|
520
|
+
def arm(self, box):
|
|
521
|
+
"""Watches the folder of every file to keep facts of; one that is not
|
|
522
|
+
there yet is tried again on the next pass."""
|
|
523
|
+
if not self.inotify:
|
|
524
|
+
return
|
|
525
|
+
wanted = {posixpath.dirname(path) for path in box.guarded}
|
|
526
|
+
for folder in [folder for folder in box.watches if folder not in wanted]:
|
|
527
|
+
self.unwatch(box, folder)
|
|
528
|
+
for folder in wanted - box.watches.keys():
|
|
529
|
+
try:
|
|
530
|
+
watch = self.inotify.add(box.workspace, folder)
|
|
531
|
+
except (OSError, ValueError):
|
|
532
|
+
continue
|
|
533
|
+
box.watches[folder] = watch
|
|
534
|
+
self.watched[watch] = (box, folder)
|
|
535
|
+
|
|
536
|
+
def unwatch(self, box, folder):
|
|
537
|
+
watch = box.watches.pop(folder, None)
|
|
538
|
+
if watch is not None and self.watched.pop(watch, None):
|
|
539
|
+
self.inotify.remove(watch)
|
|
540
|
+
|
|
541
|
+
def close(self, box):
|
|
542
|
+
"""A sandbox's stream ended: settle what was pending and let it go."""
|
|
543
|
+
for path in set(box.due) | set(box.final):
|
|
544
|
+
self.look(box, path, final=True)
|
|
545
|
+
self.summaries(box, self.clock(), closing=True)
|
|
546
|
+
for folder in list(box.watches):
|
|
547
|
+
self.unwatch(box, folder)
|
|
548
|
+
|
|
549
|
+
# ------------------------------------------------------------ processes
|
|
550
|
+
|
|
551
|
+
def executed(self, box, event):
|
|
552
|
+
pid, argv = event.get('pid'), [str(part) for part in event.get('argv') or []]
|
|
553
|
+
known = box.processes.get(pid) or {}
|
|
554
|
+
parent = box.processes.get(event.get('ppid')) or {}
|
|
555
|
+
# The call id travels in the environment, and for the shell that runs a
|
|
556
|
+
# command in its own text (claude-hook.mjs): the forwarder reads both the same way.
|
|
557
|
+
marker = (event.get('markers') or {}).get('RAYTRACE_CALL_ID') or (CALL_MARKER.search(' '.join(argv)) or [None, None])[1]
|
|
558
|
+
box.processes[pid] = {'start_ns': event.get('process_start_ns'), 'ppid': event.get('ppid'),
|
|
559
|
+
'name': posixpath.basename(event.get('executable') or (argv[0] if argv else '')) or event.get('process_name'),
|
|
560
|
+
'argv': [part[:400] for part in argv[:24]],
|
|
561
|
+
'call_id': marker or known.get('call_id') or parent.get('call_id')}
|
|
562
|
+
if len(box.processes) > 20_000: # exits that never arrived: forget the oldest half
|
|
563
|
+
for stale in list(box.processes)[:10_000]:
|
|
564
|
+
del box.processes[stale]
|
|
565
|
+
|
|
566
|
+
def exited(self, box, event):
|
|
567
|
+
gone = box.processes.pop(event.get('pid'), None)
|
|
568
|
+
if gone and box.workspace and PACKAGE_TOOLS & {posixpath.basename(part) for part in gone['argv'][:4]}:
|
|
569
|
+
# A package manager finished: see what it left, once it has settled.
|
|
570
|
+
box.package_due = self.clock() + 1
|
|
571
|
+
box.package_who = self.who(box, event, gone)
|
|
572
|
+
|
|
573
|
+
def who(self, box, event, known=None):
|
|
574
|
+
"""The process an event is from, and the tool call it belongs to."""
|
|
575
|
+
pid = event.get('pid')
|
|
576
|
+
known = known or box.processes.get(pid) or {}
|
|
577
|
+
if known.get('start_ns') and event.get('process_start_ns') and known['start_ns'] != event['process_start_ns']:
|
|
578
|
+
known = {} # the pid was used again
|
|
579
|
+
parent = box.processes.get(event.get('ppid')) or {}
|
|
580
|
+
return {'pid': pid, 'start_ns': known.get('start_ns') or event.get('process_start_ns'),
|
|
581
|
+
'name': known.get('name') or event.get('process_name') or None, 'argv': known.get('argv') or [],
|
|
582
|
+
'call_id': known.get('call_id') or parent.get('call_id')}
|
|
583
|
+
|
|
584
|
+
# ------------------------------------------------------------ facts
|
|
585
|
+
|
|
586
|
+
def fact(self, box, kind, key, who, body, at_ns):
|
|
587
|
+
"""The fact these say, begun or added to: one fact for the same thing
|
|
588
|
+
done by the same step, with a count, not one for each time."""
|
|
589
|
+
step = who.get('call_id') or (f"{who.get('pid')}:{who.get('start_ns')}" if who.get('pid') else '')
|
|
590
|
+
fact_id = hashlib.sha256('\0'.join([box.container, kind, step, *map(str, key)]).encode()).hexdigest()[:32]
|
|
591
|
+
fact = self.facts.get(fact_id)
|
|
592
|
+
if fact is None:
|
|
593
|
+
fact = self.facts[fact_id] = {
|
|
594
|
+
'fact_id': fact_id, 'container_id': box.container, 'kind': kind, 'count': 0, 'first_ns': str(at_ns),
|
|
595
|
+
'pid': who.get('pid'), 'process_start_ns': who.get('start_ns'), 'process': who.get('name'),
|
|
596
|
+
'command': ' '.join(who.get('argv') or [])[:160] or None, 'call_id': who.get('call_id'), **body}
|
|
597
|
+
if box.fork:
|
|
598
|
+
fact['fork'] = True
|
|
599
|
+
fact['count'] += 1
|
|
600
|
+
fact['last_ns'] = str(at_ns)
|
|
601
|
+
now = self.clock()
|
|
602
|
+
if now - box.gap_at < 60 or now - self.overflowed < 60:
|
|
603
|
+
fact['coverage'] = 'gaps' # events were lost about now: this may not be all of it
|
|
604
|
+
fact['touched'] = now
|
|
605
|
+
self.dirty[fact_id] = True
|
|
606
|
+
return fact
|
|
607
|
+
|
|
608
|
+
def flush(self):
|
|
609
|
+
if self.dirty:
|
|
610
|
+
now = int(time.time() * 1000)
|
|
611
|
+
rows = []
|
|
612
|
+
for fact_id in self.dirty:
|
|
613
|
+
fact = {key: value for key, value in self.facts[fact_id].items() if key != 'touched'}
|
|
614
|
+
rows.append((fact_id, fact['container_id'], fact['kind'], fact['last_ns'], now, json.dumps(fact)))
|
|
615
|
+
# Replaced, not updated: a fact that grew gets a new row id, so
|
|
616
|
+
# whoever reads by "newer than" sees it again.
|
|
617
|
+
self.db.executemany('INSERT OR REPLACE INTO facts (fact_id, container_id, kind, at_ns, received_ms, body) VALUES (?,?,?,?,?,?)', rows)
|
|
618
|
+
self.db.commit()
|
|
619
|
+
self.dirty.clear()
|
|
620
|
+
idle = self.clock() - 600
|
|
621
|
+
for fact_id in [fact_id for fact_id, fact in self.facts.items() if fact['touched'] < idle]:
|
|
622
|
+
del self.facts[fact_id]
|
|
623
|
+
|
|
624
|
+
def sweep(self):
|
|
625
|
+
"""Retention on this host: facts are kept two weeks, and no sandbox keeps more than its newest 50,000."""
|
|
626
|
+
self.db.execute('DELETE FROM facts WHERE received_ms < ?', (int(time.time() * 1000) - RETAIN_DAYS * 86_400_000,))
|
|
627
|
+
for (container,) in self.db.execute('SELECT container_id FROM facts GROUP BY container_id HAVING count(*) > ?', (RETAIN_ROWS,)).fetchall():
|
|
628
|
+
self.db.execute('DELETE FROM facts WHERE container_id = ? AND id NOT IN (SELECT id FROM facts WHERE container_id = ? ORDER BY id DESC LIMIT ?)',
|
|
629
|
+
(container, container, RETAIN_ROWS))
|
|
630
|
+
self.db.execute('DELETE FROM dns_cache WHERE seen_ns < ?', (time.time_ns() - 86_400_000_000_000,))
|
|
631
|
+
self.db.commit()
|
|
632
|
+
|
|
633
|
+
def counted(self, box, who, top, at_ns):
|
|
634
|
+
"""An open that is nobody's business: counted for its step, and gone."""
|
|
635
|
+
step = who.get('call_id') or f"{who.get('pid')}:{who.get('start_ns')}"
|
|
636
|
+
count = box.counts.get(step)
|
|
637
|
+
if count is None:
|
|
638
|
+
count = box.counts[step] = {'who': who, 'opens': 0, 'tops': {}, 'first_ns': at_ns, 'written': 0}
|
|
639
|
+
count['opens'] += 1
|
|
640
|
+
count['tops'][top] = count['tops'].get(top, 0) + 1
|
|
641
|
+
count['last_ns'], count['touched'] = at_ns, self.clock()
|
|
642
|
+
|
|
643
|
+
def summaries(self, box, now, closing=False):
|
|
644
|
+
"""One line for a step that opened files in bulk: how many, and where."""
|
|
645
|
+
for step, count in list(box.counts.items()):
|
|
646
|
+
idle = now - count['touched']
|
|
647
|
+
if count['opens'] >= BULK and count['opens'] != count['written'] and (closing or idle > 3 or now - count.get('wrote_at', 0) > 15):
|
|
648
|
+
# A tool call's count is of all its processes: it names the call, not whichever opened first.
|
|
649
|
+
who = {'call_id': count['who']['call_id']} if count['who'].get('call_id') else count['who']
|
|
650
|
+
fact = self.fact(box, 'summary', ('opens',), who, {'effect': 'opened'}, count['last_ns'])
|
|
651
|
+
fact.update(count=1, first_ns=str(count['first_ns']), opens=count['opens'],
|
|
652
|
+
folders=dict(sorted(count['tops'].items(), key=lambda item: -item[1])[:5]))
|
|
653
|
+
count['written'], count['wrote_at'] = count['opens'], now
|
|
654
|
+
if closing or idle > 120:
|
|
655
|
+
del box.counts[step]
|
|
656
|
+
|
|
657
|
+
# ------------------------------------------------------------ files
|
|
658
|
+
|
|
659
|
+
def opened(self, box, event):
|
|
660
|
+
at_ns = int(event.get('timestamp_ns') or 0) or time.time_ns()
|
|
661
|
+
who = self.who(box, event)
|
|
662
|
+
path = project_path(event)
|
|
663
|
+
if path is None or not box.workspace:
|
|
664
|
+
return self.counted(box, who, '(outside the project)', at_ns)
|
|
665
|
+
flags, errno = event.get('flags', 0), event.get('errno', 0)
|
|
666
|
+
writing = (flags & O_ACCMODE) != 0 or bool(flags & (O_CREAT | O_TRUNC))
|
|
667
|
+
guard = box.guarded.get(path)
|
|
668
|
+
top = path.split('/', 1)[0] if '/' in path else '(project root)'
|
|
669
|
+
if guard is None:
|
|
670
|
+
# A file that was not there and now is: worth a fact when its path alone marks it (a .env, a key).
|
|
671
|
+
if flags & O_CREAT and not errno and box.known is not None and path not in box.known and not SKIPPED.intersection(path.split('/')):
|
|
672
|
+
box.known.add(path)
|
|
673
|
+
if file_scan.created_entry(path)['level'] >= SENSITIVE_FROM:
|
|
674
|
+
box.sensitive.add(path)
|
|
675
|
+
box.photos[path] = None
|
|
676
|
+
self.guard(box)
|
|
677
|
+
self.arm(box)
|
|
678
|
+
guard = 'sensitive'
|
|
679
|
+
if guard is None:
|
|
680
|
+
return self.counted(box, who, top, at_ns)
|
|
681
|
+
call = self.calls.get(event.get('sysno'), ('open', 'open'))[0]
|
|
682
|
+
basis = {'syscall': call, 'flags': flags, 'resolution': 'lexical'}
|
|
683
|
+
if errno:
|
|
684
|
+
if errno in REFUSED:
|
|
685
|
+
self.fact(box, 'file', (path, 'attempted', 'write' if writing else 'read'), who,
|
|
686
|
+
{'path': path, 'guard': guard, 'effect': 'attempted', 'certainty': 'attempted', 'intent': 'write' if writing else 'read',
|
|
687
|
+
'basis': {**basis, 'errno': REFUSED[errno]}}, at_ns)
|
|
688
|
+
return
|
|
689
|
+
if not writing:
|
|
690
|
+
self.fact(box, 'file', (path, 'read'), who, {'path': path, 'guard': guard, 'effect': 'read', 'certainty': 'confirmed',
|
|
691
|
+
'basis': {**basis, 'note': 'opened for reading; the bytes read are not captured'}}, at_ns)
|
|
692
|
+
return
|
|
693
|
+
now = self.clock()
|
|
694
|
+
box.writers.setdefault(path, []).append({'who': who, 'at_ns': at_ns, 'flags': flags, 'syscall': call, 'credited': False, 'seen': now})
|
|
695
|
+
box.due[path] = min(box.due.get(path, now + 1), now + 1)
|
|
696
|
+
box.final[path] = now + 30
|
|
697
|
+
|
|
698
|
+
def numbered_call(self, box, event):
|
|
699
|
+
"""A delete, rename, resize, mkdir or mode change: gVisor gives its
|
|
700
|
+
number and result, not its path. Kept a moment, for the watcher to say
|
|
701
|
+
which file; a refusal is a fact at once."""
|
|
702
|
+
name, rule = self.calls.get(event.get('sysno'), (None, None))
|
|
703
|
+
if not name:
|
|
704
|
+
return
|
|
705
|
+
at_ns = int(event.get('timestamp_ns') or 0) or time.time_ns()
|
|
706
|
+
who, errno = self.who(box, event), event.get('errno', 0)
|
|
707
|
+
box.raw.append({'rule': rule, 'syscall': name, 'errno': errno, 'at_ns': at_ns, 'who': who})
|
|
708
|
+
if errno not in REFUSED or rule == 'mkdir':
|
|
709
|
+
return
|
|
710
|
+
# The process's own arguments, as gVisor reported them at exec, may name
|
|
711
|
+
# the file: `rm notes.txt`. Text that only contains the name is weaker.
|
|
712
|
+
named = self.named(box, who, event.get('cwd'))
|
|
713
|
+
basis = {'syscall': name, 'errno': REFUSED[errno], 'note': 'gVisor does not report the path of this call'}
|
|
714
|
+
for path, how in named or [(None, None)]:
|
|
715
|
+
self.fact(box, 'file', (path or '', 'attempted', name), who,
|
|
716
|
+
{'path': path, 'guard': box.guarded.get(path) if path else None, 'effect': 'attempted', 'certainty': 'attempted',
|
|
717
|
+
'intent': {'remove': 'delete', 'rename': 'rename', 'resize': 'write', 'metadata': 'permissions'}[rule],
|
|
718
|
+
'basis': {**basis, **({'path_from': how} if how else {})}}, at_ns)
|
|
719
|
+
|
|
720
|
+
def named(self, box, who, cwd):
|
|
721
|
+
"""Restricted or watched files a process's arguments name: [(path, how)]."""
|
|
722
|
+
found = {}
|
|
723
|
+
for part in who.get('argv') or []:
|
|
724
|
+
if not part or part.startswith('-'):
|
|
725
|
+
continue
|
|
726
|
+
name = posixpath.normpath(part if part.startswith('/') else (cwd or WORKSPACE) + '/' + part)
|
|
727
|
+
if name + '/' != WORKSPACE and not name.startswith(WORKSPACE):
|
|
728
|
+
continue
|
|
729
|
+
at = name[len(WORKSPACE):] # empty for the project folder itself: `rm -r .` names everything
|
|
730
|
+
if at in box.guarded:
|
|
731
|
+
found[at] = 'argv'
|
|
732
|
+
for path in box.guarded: # `rm -r src` reaches every file under src
|
|
733
|
+
if not at or path.startswith(at + '/'):
|
|
734
|
+
found.setdefault(path, 'argv_folder')
|
|
735
|
+
if not found:
|
|
736
|
+
text = ' '.join(who.get('argv') or [])
|
|
737
|
+
found = {path: 'argv_text' for path in box.guarded if path in text}
|
|
738
|
+
return sorted(found.items())[:5]
|
|
739
|
+
|
|
740
|
+
def actors(self, box, rule, at_ns, path):
|
|
741
|
+
"""The processes whose delete, rename, resize or mode call succeeded
|
|
742
|
+
about when the watcher saw `path` change, likeliest first."""
|
|
743
|
+
found = {}
|
|
744
|
+
for call in box.raw:
|
|
745
|
+
if call['rule'] == rule and not call['errno'] and at_ns - RAW_WINDOW[0] <= call['at_ns'] <= at_ns + RAW_WINDOW[1]:
|
|
746
|
+
found.setdefault((call['who']['pid'], call['who']['start_ns']), call)
|
|
747
|
+
names = lambda call: any(posixpath.basename(path) == posixpath.basename(part) for part in call['who']['argv'])
|
|
748
|
+
ranked = sorted(found.values(), key=lambda call: (not names(call), abs(call['at_ns'] - at_ns)))
|
|
749
|
+
own = [call for call in ranked if names(call)]
|
|
750
|
+
return [(call['who'], call['syscall']) for call in (own if len(own) == 1 else ranked)[:5]]
|
|
751
|
+
|
|
752
|
+
def watcher_events(self):
|
|
753
|
+
if not self.inotify:
|
|
754
|
+
return 0
|
|
755
|
+
events = self.inotify.read()
|
|
756
|
+
moved = {cookie: posixpath.join(self.watched[watch][1], name) for watch, mask, cookie, name in events
|
|
757
|
+
if mask & Inotify.MOVED_TO and watch in self.watched}
|
|
758
|
+
for watch, mask, cookie, name in events:
|
|
759
|
+
if mask & Inotify.OVERFLOW: # the kernel dropped notices: look at everything again
|
|
760
|
+
for box in self.boxes.values():
|
|
761
|
+
box.gap_at = self.clock()
|
|
762
|
+
for path in box.guarded:
|
|
763
|
+
self.noticed(box, path, [])
|
|
764
|
+
continue
|
|
765
|
+
target = self.watched.get(watch)
|
|
766
|
+
if not target:
|
|
767
|
+
continue
|
|
768
|
+
box, folder = target
|
|
769
|
+
if mask & (Inotify.IGNORED | Inotify.DELETE_SELF | Inotify.MOVE_SELF):
|
|
770
|
+
# The folder itself went: its files went with it, and nothing named them.
|
|
771
|
+
if mask & Inotify.IGNORED:
|
|
772
|
+
box.watches.pop(folder, None)
|
|
773
|
+
self.watched.pop(watch, None)
|
|
774
|
+
for path in box.guarded:
|
|
775
|
+
if posixpath.dirname(path) == folder:
|
|
776
|
+
self.noticed(box, path, [])
|
|
777
|
+
continue
|
|
778
|
+
path = posixpath.join(folder, name)
|
|
779
|
+
if path in box.guarded:
|
|
780
|
+
self.noticed(box, path, [word for bit, word in Inotify.WORDS if mask & bit],
|
|
781
|
+
moved.get(cookie) if mask & Inotify.MOVED_FROM else None)
|
|
782
|
+
return len(events)
|
|
783
|
+
|
|
784
|
+
def noticed(self, box, path, words, moved_to=None):
|
|
785
|
+
"""The watcher saw something of a file: look at it once things settle."""
|
|
786
|
+
cause = box.causes.setdefault(path, {'events': set(), 'at_ns': time.time_ns()})
|
|
787
|
+
cause['events'].update(words)
|
|
788
|
+
if moved_to:
|
|
789
|
+
cause['to'] = moved_to
|
|
790
|
+
now = self.clock()
|
|
791
|
+
box.due[path] = min(box.due.get(path, now + SETTLE), now + SETTLE)
|
|
792
|
+
|
|
793
|
+
def look(self, box, path, final=False):
|
|
794
|
+
"""Compares a file with its last photo and says what happened to it, and who did it."""
|
|
795
|
+
guard = box.guarded.get(path)
|
|
796
|
+
if guard is None or not box.workspace:
|
|
797
|
+
box.writers.pop(path, None)
|
|
798
|
+
return
|
|
799
|
+
cause = box.causes.pop(path, None) or {'events': set(), 'at_ns': time.time_ns()}
|
|
800
|
+
now = self.clock()
|
|
801
|
+
writers = box.writers[path] = [writer for writer in box.writers.get(path, []) if now - writer['seen'] < WRITER_LIFE]
|
|
802
|
+
before = box.photos.get(path)
|
|
803
|
+
after = photo(box.workspace, path, box.hashing)
|
|
804
|
+
flags = 0
|
|
805
|
+
for writer in writers:
|
|
806
|
+
flags |= writer['flags']
|
|
807
|
+
effect = change(before, after, cause['events'], flags)
|
|
808
|
+
if effect:
|
|
809
|
+
box.photos[path] = after
|
|
810
|
+
if after is None and box.known is not None:
|
|
811
|
+
box.known.discard(path)
|
|
812
|
+
if effect in ('deleted', 'renamed'):
|
|
813
|
+
found = self.actors(box, 'remove' if effect == 'deleted' else 'rename', cause['at_ns'], path)
|
|
814
|
+
elif effect == 'permissions_changed':
|
|
815
|
+
found = self.actors(box, 'metadata', cause['at_ns'], path)
|
|
816
|
+
else:
|
|
817
|
+
found, seen = [], set()
|
|
818
|
+
for writer in reversed(writers): # the latest to open it first
|
|
819
|
+
key = (writer['who']['pid'], writer['who']['start_ns'])
|
|
820
|
+
if key not in seen:
|
|
821
|
+
seen.add(key)
|
|
822
|
+
found.append((writer['who'], writer['syscall']))
|
|
823
|
+
writer['credited'] = True
|
|
824
|
+
if 'moved_to' in cause['events']: # written elsewhere and moved over it, as editors and `sed -i` do
|
|
825
|
+
found += self.actors(box, 'rename', cause['at_ns'], path)
|
|
826
|
+
if not found:
|
|
827
|
+
found = self.actors(box, 'resize', cause['at_ns'], path)
|
|
828
|
+
who, call = found[0] if found else ({}, None)
|
|
829
|
+
certainty = 'unattributed' if not found else 'confirmed' if len(found) == 1 else 'one_of'
|
|
830
|
+
basis = {'syscall': call, 'watcher': sorted(cause['events']), 'before': before, 'after': after, 'resolution': 'lexical'}
|
|
831
|
+
if len(found) > 1:
|
|
832
|
+
basis['candidates'] = [{'pid': other['pid'], 'process': other['name']} for other, _ in found[:5]]
|
|
833
|
+
fact = self.fact(box, 'file', (path, effect, certainty), who,
|
|
834
|
+
{'path': path, 'guard': guard, 'effect': effect, 'certainty': certainty, 'basis': basis}, cause['at_ns'])
|
|
835
|
+
fact['basis']['after'] = after
|
|
836
|
+
if cause.get('to'):
|
|
837
|
+
fact['to'] = cause['to']
|
|
838
|
+
elif final:
|
|
839
|
+
for writer in writers:
|
|
840
|
+
if not writer['credited']:
|
|
841
|
+
writer['credited'] = True
|
|
842
|
+
self.fact(box, 'file', (path, 'opened_for_writing'), writer['who'],
|
|
843
|
+
{'path': path, 'guard': guard, 'effect': 'opened_for_writing', 'certainty': 'possible',
|
|
844
|
+
'basis': {'syscall': writer['syscall'], 'flags': writer['flags'], 'before': before, 'after': after,
|
|
845
|
+
'resolution': 'lexical', 'note': 'opened for writing; the file is as it was'}}, writer['at_ns'])
|
|
846
|
+
if not writers:
|
|
847
|
+
box.writers.pop(path, None)
|
|
848
|
+
|
|
849
|
+
# ------------------------------------------------------------ network, identity, packages
|
|
850
|
+
|
|
851
|
+
def host_name(self, box, ip):
|
|
852
|
+
"""The name last looked up, from this sandbox, that answered with this
|
|
853
|
+
address (dns_forwarder.py). A best match: an address can serve many names."""
|
|
854
|
+
kept = box.names.get(ip)
|
|
855
|
+
now = self.clock()
|
|
856
|
+
if kept and now - kept[1] < 30:
|
|
857
|
+
return kept[0]
|
|
858
|
+
try:
|
|
859
|
+
row = self.db.execute('SELECT name FROM dns_cache WHERE client = ? AND ip = ?', (box.ip or '', ip)).fetchone()
|
|
860
|
+
except sqlite3.Error:
|
|
861
|
+
row = None
|
|
862
|
+
box.names[ip] = (row[0] if row else None, now)
|
|
863
|
+
if len(box.names) > 5000:
|
|
864
|
+
box.names.clear()
|
|
865
|
+
return row[0] if row else None
|
|
866
|
+
|
|
867
|
+
def connected(self, box, event):
|
|
868
|
+
if event.get('family') != 'inet':
|
|
869
|
+
return # a socket file inside the sandbox, or a family that leaves no machine
|
|
870
|
+
ip, port, errno = event['ip'], event['port'], event.get('errno', 0)
|
|
871
|
+
there = ipaddress.ip_address(ip)
|
|
872
|
+
at_ns = int(event.get('timestamp_ns') or 0) or time.time_ns()
|
|
873
|
+
listening = event['kind'] == 'listen'
|
|
874
|
+
if not listening and (there.is_loopback or there.is_unspecified):
|
|
875
|
+
return # the sandbox talking to itself
|
|
876
|
+
if not listening and port == 0:
|
|
877
|
+
# No traffic: the resolver library "connects" a datagram socket to each
|
|
878
|
+
# address a name answered with, port 0, to choose which to use.
|
|
879
|
+
return
|
|
880
|
+
now = self.clock()
|
|
881
|
+
if now - self.bridge[1] > 30:
|
|
882
|
+
try:
|
|
883
|
+
self.bridge = (BRIDGE.read_text().strip().removeprefix('http://'), now)
|
|
884
|
+
except OSError:
|
|
885
|
+
self.bridge = ('', now)
|
|
886
|
+
if f'{ip}:{port}' in (self.bridge[0], f'{network.GATEWAY}:53'):
|
|
887
|
+
return # RayTrace's own: the model and transcript bridge, and the resolver (dns_forwarder.py records lookups)
|
|
888
|
+
who = self.who(box, event)
|
|
889
|
+
if listening:
|
|
890
|
+
outcome = 'failed' if errno else 'listening'
|
|
891
|
+
elif there.version == 4 and any(there in block for block in REJECTED):
|
|
892
|
+
outcome = 'refused_by_policy' # the firewall rejects it, whatever a non-blocking call returned
|
|
893
|
+
else:
|
|
894
|
+
outcome = 'connected' if not errno else 'attempted' if errno == EINPROGRESS else 'failed'
|
|
895
|
+
name = None if listening else self.host_name(box, ip)
|
|
896
|
+
# One fact for a site, not one for each address its name answers with:
|
|
897
|
+
# a registry has a dozen, and a program tries IPv6 before IPv4.
|
|
898
|
+
fact = self.fact(box, 'network', (name or ip, port, event['kind']), who,
|
|
899
|
+
{'ip': ip, 'port': port, 'host': name, 'addresses': [], 'effect': event['kind'], 'outcomes': {},
|
|
900
|
+
'basis': {'syscall': event['kind'], 'note': 'the address is what the program asked for; the name is the lookup that answered with it'}}, at_ns)
|
|
901
|
+
fact['outcomes'][outcome] = fact['outcomes'].get(outcome, 0) + 1
|
|
902
|
+
if ip not in fact['addresses'] and len(fact['addresses']) < 32:
|
|
903
|
+
fact['addresses'].append(ip)
|
|
904
|
+
if outcome == 'connected':
|
|
905
|
+
fact['ip'] = ip # shown first: an address that answered
|
|
906
|
+
if errno and outcome == 'failed':
|
|
907
|
+
fact['errno'] = ERRNO.get(errno, str(errno))
|
|
908
|
+
if name in REGISTRIES:
|
|
909
|
+
box.registry_at = self.clock()
|
|
910
|
+
|
|
911
|
+
def identity(self, box, event):
|
|
912
|
+
errno = event.get('errno', 0)
|
|
913
|
+
call = self.calls.get(event.get('sysno'), ('setid', None))[0]
|
|
914
|
+
at_ns = int(event.get('timestamp_ns') or 0) or time.time_ns()
|
|
915
|
+
self.fact(box, 'identity', (call, *event.get('ids', []), bool(errno)), self.who(box, event),
|
|
916
|
+
{'effect': 'attempted' if errno else 'changed', 'certainty': 'attempted' if errno else 'confirmed',
|
|
917
|
+
'ids': event.get('ids', []), 'basis': {'syscall': call, **({'errno': ERRNO.get(errno, str(errno))} if errno else {})}}, at_ns)
|
|
918
|
+
|
|
919
|
+
def packages(self, box):
|
|
920
|
+
"""What a package manager left that was not there before."""
|
|
921
|
+
if not box.workspace:
|
|
922
|
+
return
|
|
923
|
+
now = installed(box.workspace, box.home)
|
|
924
|
+
before, box.packages = box.packages or {}, now
|
|
925
|
+
fresh = sorted(key for key in now if key not in before)
|
|
926
|
+
if not fresh:
|
|
927
|
+
return
|
|
928
|
+
who = box.package_who or {}
|
|
929
|
+
# Seen reaching a registry about then: the list is the manager's own word, with that beside it.
|
|
930
|
+
certainty = 'corroborated' if self.clock() - box.registry_at < 300 else 'reported'
|
|
931
|
+
for manager in sorted({key[0] for key in fresh}):
|
|
932
|
+
mine = [key for key in fresh if key[0] == manager]
|
|
933
|
+
fact = self.fact(box, 'package', (manager,), who, {'manager': manager, 'packages': [], 'total': 0, 'effect': 'installed',
|
|
934
|
+
'basis': {'note': "read from the package manager's own records in the sandbox after it ran; the agent can write those files"}},
|
|
935
|
+
time.time_ns())
|
|
936
|
+
fact['certainty'] = certainty
|
|
937
|
+
fact['total'] += len(mine)
|
|
938
|
+
room = MAX_PACKAGES - len(fact['packages'])
|
|
939
|
+
fact['packages'] += [{'name': name, 'version': version, **({'source': now[(manager, name, version)]} if now[(manager, name, version)] else {})}
|
|
940
|
+
for _, name, version in mine[:max(0, room)]]
|