raytrace-cli 0.11.1__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raytrace/__init__.py +1 -0
- raytrace/__main__.py +33 -0
- raytrace/app/LICENSE +21 -0
- raytrace/app/README.md +599 -0
- raytrace/app/bin/raytrace.mjs +297 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-BwCSEQnW.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-400-normal-CW0RaeGs.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-B9HHJjqV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-500-normal-Dr3UlScf.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-Aqo67rzb.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-600-normal-BmdmIIQ2.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-CUSSCpQX.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-700-normal-DvUfVpUG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BjWJ59Pq.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-400-normal-BtiwyxMk.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-BJfUCQsA.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-500-normal-DR84L5F-.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-4vooXBpG.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-600-normal-DRtaDpgU.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-BLI3TTWz.woff +0 -0
- raytrace/app/dist/dashboard/assets/dm-sans-latin-ext-700-normal-CJIcxD6K.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-CTOM6hUh.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-600-normal-fLZuRloM.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-9HEixskS.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-cyrillic-ext-600-normal-V-xxqcpd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-BgSNZQsw.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-600-normal-DWFSQ4vo.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-D38SheWl.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-latin-ext-600-normal-DmB0ttJJ.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-D2EvbN8M.woff2 +0 -0
- raytrace/app/dist/dashboard/assets/ibm-plex-mono-vietnamese-600-normal-iLQfcSjf.woff +0 -0
- raytrace/app/dist/dashboard/assets/index-DepcI8Fi.js +38 -0
- raytrace/app/dist/dashboard/assets/index-H97VLD5l.css +1 -0
- raytrace/app/dist/dashboard/favicon.svg +6 -0
- raytrace/app/dist/dashboard/index.html +15 -0
- raytrace/app/package.json +48 -0
- raytrace/app/proxy/agent-intent-schema.test.mjs +142 -0
- raytrace/app/proxy/call-assessment.mjs +385 -0
- raytrace/app/proxy/call-assessment.test.mjs +167 -0
- raytrace/app/proxy/call-category.mjs +461 -0
- raytrace/app/proxy/capture-health.mjs +30 -0
- raytrace/app/proxy/capture-health.test.mjs +44 -0
- raytrace/app/proxy/capture.mjs +38 -0
- raytrace/app/proxy/capture.test.mjs +49 -0
- raytrace/app/proxy/claude-code-adapter.mjs +301 -0
- raytrace/app/proxy/claude-code-hook.mjs +22 -0
- raytrace/app/proxy/codex-config.mjs +36 -0
- raytrace/app/proxy/codex.mjs +14 -0
- raytrace/app/proxy/evidence-schema.test.mjs +240 -0
- raytrace/app/proxy/evidence-store.mjs +993 -0
- raytrace/app/proxy/evidence-store.test.mjs +301 -0
- raytrace/app/proxy/exec-narrative.mjs +625 -0
- raytrace/app/proxy/exec-narrative.test.mjs +262 -0
- raytrace/app/proxy/execution-correlation.mjs +185 -0
- raytrace/app/proxy/execution-correlation.test.mjs +146 -0
- raytrace/app/proxy/execution-runner.mjs +89 -0
- raytrace/app/proxy/execution-runner.test.mjs +38 -0
- raytrace/app/proxy/experiment-engine.mjs +195 -0
- raytrace/app/proxy/experiment-engine.test.mjs +93 -0
- raytrace/app/proxy/explanations.mjs +36 -0
- raytrace/app/proxy/explanations.test.mjs +20 -0
- raytrace/app/proxy/failure-signature.mjs +56 -0
- raytrace/app/proxy/gvisor-forwarder.mjs +493 -0
- raytrace/app/proxy/gvisor-forwarder.test.mjs +433 -0
- raytrace/app/proxy/init-evidence-db.mjs +227 -0
- raytrace/app/proxy/local-sandboxes.mjs +94 -0
- raytrace/app/proxy/local-token.mjs +30 -0
- raytrace/app/proxy/playground-recovery.test.mjs +130 -0
- raytrace/app/proxy/playground.mjs +439 -0
- raytrace/app/proxy/prompt-context.mjs +278 -0
- raytrace/app/proxy/prompt-traces.mjs +40 -0
- raytrace/app/proxy/prompt-traces.test.mjs +32 -0
- raytrace/app/proxy/providers.mjs +65 -0
- raytrace/app/proxy/providers.test.mjs +76 -0
- raytrace/app/proxy/proxy.integration.test.mjs +135 -0
- raytrace/app/proxy/raytace-proxy.mjs +941 -0
- raytrace/app/proxy/request-metrics.mjs +13 -0
- raytrace/app/proxy/request-metrics.test.mjs +19 -0
- raytrace/app/proxy/request-policy.mjs +45 -0
- raytrace/app/proxy/request-policy.test.mjs +47 -0
- raytrace/app/proxy/run-recap.mjs +139 -0
- raytrace/app/proxy/runtime-attribution.mjs +110 -0
- raytrace/app/proxy/sandbox-bridge.mjs +49 -0
- raytrace/app/proxy/schema/EVIDENCE_DATABASE.md +276 -0
- raytrace/app/proxy/schema/agent_intent.sql +101 -0
- raytrace/app/proxy/schema/auxiliary.sql +90 -0
- raytrace/app/proxy/schema/capture_health.sql +14 -0
- raytrace/app/proxy/schema/classification.sql +89 -0
- raytrace/app/proxy/schema/file_context.sql +53 -0
- raytrace/app/proxy/schema/runtime_evidence.sql +201 -0
- raytrace/app/proxy/schema/search.sql +66 -0
- raytrace/app/proxy/schema/shared_storage.sql +21 -0
- raytrace/app/proxy/search.mjs +418 -0
- raytrace/app/proxy/search.test.mjs +161 -0
- raytrace/app/proxy/smoke.test.mjs +93 -0
- raytrace/app/proxy/step-attribution.mjs +380 -0
- raytrace/app/proxy/step-lab.mjs +78 -0
- raytrace/app/proxy/step-lab.test.mjs +38 -0
- raytrace/app/proxy/summaries.mjs +151 -0
- raytrace/app/proxy/summaries.test.mjs +69 -0
- raytrace/app/proxy/tool-metadata.mjs +111 -0
- raytrace/app/proxy/tool-metadata.test.mjs +38 -0
- raytrace/app/proxy/trace-view.mjs +324 -0
- raytrace/app/proxy/trace-view.test.mjs +92 -0
- raytrace/app/proxy/typesafe.mjs +63 -0
- raytrace/app/proxy/typesafe.test.mjs +39 -0
- raytrace/app/src/agents.mjs +7 -0
- raytrace/app/src/api.mjs +57 -0
- raytrace/app/src/auth.mjs +205 -0
- raytrace/app/src/beta.mjs +96 -0
- raytrace/app/src/claude-code.mjs +93 -0
- raytrace/app/src/commands.mjs +285 -0
- raytrace/app/src/config.mjs +49 -0
- raytrace/app/src/export.mjs +200 -0
- raytrace/app/src/lima.mjs +176 -0
- raytrace/app/src/local-sandbox.mjs +358 -0
- raytrace/app/src/mcp.mjs +688 -0
- raytrace/app/src/paths.mjs +33 -0
- raytrace/app/src/prompt.mjs +42 -0
- raytrace/app/src/proxy-process.mjs +143 -0
- raytrace/app/src/sandbox.mjs +306 -0
- raytrace/app/src/share.mjs +86 -0
- raytrace/app/src/terminal.mjs +68 -0
- raytrace/app/src/upload.mjs +198 -0
- raytrace/app/worker/sandbox/Dockerfile +8 -0
- raytrace/app/worker/sandbox/SYSCALL_CATALOG.md +197 -0
- raytrace/app/worker/sandbox/claude-hook.mjs +364 -0
- raytrace/app/worker/sandbox/collector.py +405 -0
- raytrace/app/worker/sandbox/dns_forwarder.py +255 -0
- raytrace/app/worker/sandbox/engine.py +940 -0
- raytrace/app/worker/sandbox/file_scan.py +491 -0
- raytrace/app/worker/sandbox/lima.yaml +29 -0
- raytrace/app/worker/sandbox/network.py +90 -0
- raytrace/app/worker/sandbox/projects.py +702 -0
- raytrace/app/worker/sandbox/setup.py +159 -0
- raytrace/app/worker/sandbox/syscall_catalog.sql +985 -0
- raytrace/app/worker/sandbox/transcript-delivery.test.mjs +153 -0
- raytrace/app/worker/sandbox/viewer.py +195 -0
- raytrace/node/LICENSE +2946 -0
- raytrace/node/node.exe +4 -0
- raytrace_cli-0.11.1.dist-info/LICENSE +21 -0
- raytrace_cli-0.11.1.dist-info/METADATA +42 -0
- raytrace_cli-0.11.1.dist-info/RECORD +165 -0
- raytrace_cli-0.11.1.dist-info/WHEEL +4 -0
- raytrace_cli-0.11.1.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,491 @@
|
|
|
1
|
+
"""The initial scan: how dangerous each file of an uploaded project is to
|
|
2
|
+
change, from 1 (ignore) to 5 (lock down). control_api.create() runs it on
|
|
3
|
+
the screened archive.
|
|
4
|
+
|
|
5
|
+
A file's level is the higher of two independent answers, and both are kept
|
|
6
|
+
with the words that explain them:
|
|
7
|
+
|
|
8
|
+
reach weighted blast radius in the import graph. Direct dependents
|
|
9
|
+
count most; each extra hop decays, and entry or already
|
|
10
|
+
sensitive dependents count more. Files are then banded by
|
|
11
|
+
percentile within their own project, with absolute floors so
|
|
12
|
+
a tiny project cannot manufacture a top level from two or
|
|
13
|
+
three mutual imports.
|
|
14
|
+
hazard sensitivity, from both the path and what the code does.
|
|
15
|
+
Secrets, migrations, deploy and destructive database work
|
|
16
|
+
can be 5; credential, token, encryption, permission and
|
|
17
|
+
payment operations can be 4; environment, subprocess,
|
|
18
|
+
deletion and server code are 3. A Watch-level file with no
|
|
19
|
+
like-named test is lifted to Review carefully, never to 5.
|
|
20
|
+
|
|
21
|
+
Tests, documentation, examples, assets and generated files are held at 2 or
|
|
22
|
+
below whatever imports them. Nothing here calls a model or runs project code.
|
|
23
|
+
The level is only a proposal: the dashboard lets its owner move any file.
|
|
24
|
+
"""
|
|
25
|
+
import bisect
|
|
26
|
+
import io
|
|
27
|
+
import json
|
|
28
|
+
import posixpath
|
|
29
|
+
import re
|
|
30
|
+
import tarfile
|
|
31
|
+
import time
|
|
32
|
+
|
|
33
|
+
VERSION = 'scan-v2'
|
|
34
|
+
REACH_DECAY = 0.55
|
|
35
|
+
MAX_READ = 1024 * 1024 # larger files are scored by path alone
|
|
36
|
+
MAX_MARKED = 256 * 1024 # how far into a file hazard markers are looked for
|
|
37
|
+
LARGE_LINES = 800
|
|
38
|
+
|
|
39
|
+
JS = ('.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx', '.mts', '.cts')
|
|
40
|
+
PY = ('.py',)
|
|
41
|
+
SHELL = ('.sh', '.bash', '.zsh')
|
|
42
|
+
ASSETS = ('png', 'jpg', 'jpeg', 'gif', 'svg', 'ico', 'webp', 'avif', 'bmp', 'woff', 'woff2', 'ttf', 'otf', 'eot',
|
|
43
|
+
'mp3', 'mp4', 'webm', 'mov', 'wav', 'pdf', 'zip', 'gz')
|
|
44
|
+
SETTINGS = '{json,yaml,yml,toml,ini,env,txt,cfg,conf,properties}'
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _pattern(glob):
|
|
48
|
+
"""A glob as a regular expression over a whole project path. Without a
|
|
49
|
+
slash it names a file in any folder; `re:` gives the expression itself."""
|
|
50
|
+
if glob.startswith('re:'):
|
|
51
|
+
return glob[3:]
|
|
52
|
+
out, i = '', 0
|
|
53
|
+
while i < len(glob):
|
|
54
|
+
if glob.startswith('**/', i):
|
|
55
|
+
out, i = out + '(?:.*/)?', i + 3
|
|
56
|
+
elif glob.startswith('**', i):
|
|
57
|
+
out, i = out + '.*', i + 2
|
|
58
|
+
elif glob[i] == '*':
|
|
59
|
+
out, i = out + '[^/]*', i + 1
|
|
60
|
+
elif glob[i] == '{':
|
|
61
|
+
end = glob.index('}', i)
|
|
62
|
+
out, i = out + '(?:' + '|'.join(map(re.escape, glob[i + 1:end].split(','))) + ')', end + 1
|
|
63
|
+
else:
|
|
64
|
+
out, i = out + re.escape(glob[i]), i + 1
|
|
65
|
+
return out if '/' in glob else '(?:.*/)?' + out
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _rules(table):
|
|
69
|
+
return [(level, words, re.compile('|'.join(f'(?:{_pattern(g)})' for g in globs), re.I)) for level, words, globs in table]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# The first that matches is the most a file can be.
|
|
73
|
+
CEILINGS = _rules([
|
|
74
|
+
(1, 'editor and linter settings', ['.gitignore', '.gitattributes', '.editorconfig', '.prettierrc*', '.prettierignore',
|
|
75
|
+
'.oxfmtrc*', '.oxlintrc*', '.eslintrc*', '.eslintignore', 'eslint.config.*',
|
|
76
|
+
'.npmignore', '.DS_Store']),
|
|
77
|
+
(2, 'test', ['*.test.*', '*.spec.*', '*_test.py', '*_test.go', 'test_*.py', 'conftest.py', '**/__tests__/**', '**/test/**',
|
|
78
|
+
'**/tests/**', '**/e2e/**', '**/fixtures/**', '**/__mocks__/**']),
|
|
79
|
+
(2, 'documentation', ['*.md', '*.mdx', '*.rst', '**/docs/**', 'LICENSE*', 'CHANGELOG*', 'NOTICE*',
|
|
80
|
+
r're:(?:.*/)?(?!(?:requirements|constraints)[^/]*\.txt$|CMakeLists\.txt$)[^/]*\.txt']),
|
|
81
|
+
(2, 'example', ['**/examples/**', '**/example/**', '**/samples/**']),
|
|
82
|
+
(2, 'generated file', ['*.min.js', '*.min.css', '*.map', '*.snap', '*.tsbuildinfo', '**/dist/**', '**/build/**']),
|
|
83
|
+
(2, 'static asset', ['*.{' + ','.join(ASSETS) + '}']),
|
|
84
|
+
])
|
|
85
|
+
# The highest that matches is the least a file can be.
|
|
86
|
+
FLOORS = _rules([
|
|
87
|
+
(5, 'holds secrets or credentials', [r're:(?:.*/)?\.env(?:\.(?!(?:example|sample|template|dist|defaults)$)[^/]*)?',
|
|
88
|
+
'*.{pem,key,p12,pfx,keystore,jks}', 'id_rsa*', 'id_ed25519*', 'credentials',
|
|
89
|
+
'credentials.*', '*secret*.' + SETTINGS, '*credentials*.' + SETTINGS,
|
|
90
|
+
'service-account*.json', '.npmrc', '.netrc', '.pypirc']),
|
|
91
|
+
(5, 'database migration', ['**/migrations/**', '**/migrate/**', '**/alembic/versions/**']),
|
|
92
|
+
(5, 'deploy or CI configuration', ['Dockerfile', 'Dockerfile.*', '*.dockerfile', 'docker-compose*.{yml,yaml}',
|
|
93
|
+
'compose.{yml,yaml}', 'vercel.{json,ts}', 'fly.toml', 'netlify.toml', 'render.yaml',
|
|
94
|
+
'Procfile', 'serverless.{yml,yaml}', 'cloudbuild.{yml,yaml}', 'app.yaml',
|
|
95
|
+
'wrangler.{toml,json,jsonc}', '**/k8s/**', '**/kubernetes/**', '**/helm/**',
|
|
96
|
+
'**/.github/workflows/*', '.gitlab-ci.yml', '**/.circleci/**', 'Jenkinsfile',
|
|
97
|
+
'*.tf', '*.tfvars']),
|
|
98
|
+
(4, 'dependency manifest or lockfile', ['package.json', 'package-lock.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml',
|
|
99
|
+
'yarn.lock', 'bun.lock', 'bun.lockb', 'pyproject.toml', 'requirements*.txt',
|
|
100
|
+
'poetry.lock', 'uv.lock', 'Pipfile', 'Pipfile.lock', 'go.mod', 'go.sum',
|
|
101
|
+
'Cargo.toml', 'Cargo.lock', 'Gemfile', 'Gemfile.lock', 'composer.{json,lock}',
|
|
102
|
+
'pom.xml', 'build.gradle*']),
|
|
103
|
+
(4, 'database schema', ['*.sql', '*.prisma', '*.graphql', '*.gql', '*.proto', '**/schema/**']),
|
|
104
|
+
(4, 'build configuration', ['next.config.*', 'vite.config.*', 'tsconfig*.json', 'jsconfig.json', 'webpack*.{js,ts,cjs,mjs}',
|
|
105
|
+
'babel.config.*', '.babelrc*', 'rollup.config.*', 'esbuild.*', 'tailwind.config.*',
|
|
106
|
+
'postcss.config.*', 'turbo.json', 'nx.json', 'Makefile', 'setup.py', 'setup.cfg']),
|
|
107
|
+
(4, 'authentication code', ['**/auth/**', r're:(?:.*/)?[^/]*(?:auth(?!or)|login|session|passw|permission)[^/]*\.[a-z]{2,4}']),
|
|
108
|
+
(4, 'payments code', ['**/payment/**', '**/payments/**', '**/billing/**', '*stripe*.*', '*billing*.*', '*payment*.*']),
|
|
109
|
+
(4, 'database access', ['**/db/**', '**/database/**', '**/models/**', '**/prisma/**']),
|
|
110
|
+
(3, 'configuration', ['middleware.*', '**/config/**', '*.config.*', 'config.*', 'settings.py',
|
|
111
|
+
'.env.{example,sample,template,dist,defaults}']),
|
|
112
|
+
(3, 'script', ['*.{sh,bash,zsh}', '**/scripts/**', '**/bin/**']),
|
|
113
|
+
])
|
|
114
|
+
# What a source file does, wherever it lives. A single dangerous operation
|
|
115
|
+
# makes the whole file sensitive: the guardrails this feeds are per file.
|
|
116
|
+
CONTENT_MARKERS = [(level, words, re.compile(expression, re.M)) for level, words, expression in [
|
|
117
|
+
(5, 'contains credential material', r'-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|\bsk_live_[A-Za-z0-9]|\bAKIA[0-9A-Z]{16}\b|\bgh[pousr]_[A-Za-z0-9_]{20,}\b|"private_key"\s*:\s*"-----BEGIN'),
|
|
118
|
+
(5, 'creates or rotates credentials', r'(?i)\bgenerateKeyPair(?:Sync)?\(|\bcreatePrivateKey\(|\bgenerate_key_pair\b|create_service_account_key|\brotate[A-Za-z0-9_]*\(?.{0,40}(?:credential|secret|api[_ ]?key)'),
|
|
119
|
+
(5, 'destroys database data', r'(?i)\b(?:DROP|TRUNCATE)\s+(?:TABLE|DATABASE|SCHEMA)\b'),
|
|
120
|
+
(4, 'verifies credentials or signatures', r'\bbcrypt\.compare|\bargon2\.verify|\bcheckpw\(|\bverify_password\b|\bcheck_password_hash\b|\bcompare_digest\(|\bverifySignature\b'),
|
|
121
|
+
(4, 'signs or verifies tokens', r'\bjwt\.(?:sign|verify)\b|\bSignJWT\b|\bjwtVerify\b|\bjsonwebtoken\b'),
|
|
122
|
+
(4, 'encrypts or decrypts data', r'\bcreateCipheriv\(|\bcreateDecipheriv\(|crypto\.subtle\.(?:encrypt|decrypt)\b|\bFernet\(|\bAES\.new\('),
|
|
123
|
+
(4, 'charges or refunds payments', r'paymentIntents\.create|charges\.create|refunds\.create|checkout\.sessions\.create'),
|
|
124
|
+
(4, 'checks permissions', r'\brequireAdmin\b|\bhasPermission\(|\bcheckPermission\b|\bpermission_required\b|\blogin_required\b'),
|
|
125
|
+
(4, 'changes database tables', r'(?i)\bALTER\s+TABLE\b|\bDELETE\s+FROM\b'),
|
|
126
|
+
(4, 'uses a security or payments library', r'''(?:\bfrom|\brequire\(|\bimport)\s*['"](?:stripe|bcrypt|bcryptjs|argon2|jsonwebtoken|jose|passport)['"]|^\s*(?:import|from)\s+(?:stripe|bcrypt|argon2|jwt|passlib)\b'''),
|
|
127
|
+
(3, 'reads environment variables', r'process\.env\b|os\.environ\b|os\.getenv\('),
|
|
128
|
+
(3, 'starts other programs', r'child_process|\bexecSync\(|\bspawn(?:Sync)?\(|\bsubprocess\.|os\.system\('),
|
|
129
|
+
(3, 'deletes files', r'\brmSync\(|\bfs\.rm\(|\bunlinkSync\(|shutil\.rmtree\(|os\.remove\(|\brm -r'),
|
|
130
|
+
(3, 'serves network requests', r'\bcreateServer\(|\.listen\(|\bapp\.run\(|serve_forever\(|uvicorn\.run\('),
|
|
131
|
+
]]
|
|
132
|
+
MARKERS = CONTENT_MARKERS
|
|
133
|
+
# Named so by convention: the program starts here.
|
|
134
|
+
ENTRY = re.compile(r'(?:src/)?(?:index|main|server|app|cli)\.(?:[cm]?[jt]sx?)|(?:main|manage|app|wsgi|asgi|__main__)\.py')
|
|
135
|
+
FRAMEWORK_ROOT = re.compile(r'(?:src/)?(?:app/(?:layout|page)|pages/(?:_app|_document|index)|middleware)\.[jt]sx?')
|
|
136
|
+
# Loaded by a framework, by a test runner or by hand, so nothing importing them means nothing.
|
|
137
|
+
FRAMEWORK_LOADED = re.compile(r'(?:.*/)?(?:(?:app|pages)/.*|[^/]*\.config\.[^/]*|[^/]*\.d\.ts)')
|
|
138
|
+
JS_IMPORTS = re.compile(r'''(?:\bimport|\bexport)\s[^'"`;]*?\bfrom\s*['"]([^'"\n]+)['"]'''
|
|
139
|
+
r'''|\bimport\s*['"]([^'"\n]+)['"]|\b(?:require|import)\s*\(\s*['"]([^'"\n]+)['"]\s*\)''')
|
|
140
|
+
PY_FROM = re.compile(r'^[ \t]*from[ \t]+(\.*)([\w.]*)[ \t]+import[ \t]+\(?([^\n#]*)', re.M)
|
|
141
|
+
PY_IMPORT = re.compile(r'^[ \t]*import[ \t]+([^\n#]+)', re.M)
|
|
142
|
+
TEST_STEM = re.compile(r'^(?:test_)?(.+?)(?:[._-](?:test|spec))?$')
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def rules_level(path):
|
|
146
|
+
"""What the path alone says: (level or None, the words for it, 'ceiling' | 'floor' | None).
|
|
147
|
+
Also how a file the agent creates later is scored, before anything imports it."""
|
|
148
|
+
for level, words, pattern in CEILINGS:
|
|
149
|
+
if pattern.fullmatch(path):
|
|
150
|
+
return level, words, 'ceiling'
|
|
151
|
+
best = None
|
|
152
|
+
for level, words, pattern in FLOORS:
|
|
153
|
+
if (best is None or level > best[0]) and pattern.fullmatch(path):
|
|
154
|
+
best = (level, words, 'floor')
|
|
155
|
+
return best or (None, None, None)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _weighted_dependents(path, importers, weights):
|
|
159
|
+
"""(counted dependents, direct counted importers, weighted score).
|
|
160
|
+
|
|
161
|
+
Breadth-first from the file through its importers. Shortest distance is
|
|
162
|
+
what counts: direct dependents contribute their full dependent weight,
|
|
163
|
+
each further hop is discounted by REACH_DECAY. Held files (tests, docs)
|
|
164
|
+
are followed through but contribute nothing, matching the old count.
|
|
165
|
+
"""
|
|
166
|
+
seen = {path: 0}
|
|
167
|
+
queue = [path]
|
|
168
|
+
counted = direct = 0
|
|
169
|
+
score = 0.0
|
|
170
|
+
for current in queue:
|
|
171
|
+
distance = seen[current] + 1
|
|
172
|
+
for importer in sorted(importers.get(current, ())):
|
|
173
|
+
if importer in seen:
|
|
174
|
+
continue
|
|
175
|
+
seen[importer] = distance
|
|
176
|
+
queue.append(importer)
|
|
177
|
+
weight = weights.get(importer, 1.0)
|
|
178
|
+
if weight:
|
|
179
|
+
counted += 1
|
|
180
|
+
direct += distance == 1
|
|
181
|
+
score += weight * REACH_DECAY ** (distance - 1)
|
|
182
|
+
return counted, direct, score
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _reach_level(score, percentile, direct, dependents, counted):
|
|
186
|
+
"""Reach band from a weighted score's percentile in this project.
|
|
187
|
+
|
|
188
|
+
Percentiles spread even a tightly coupled project out; the absolute
|
|
189
|
+
floors keep a handful of mutual imports from reaching the top bands.
|
|
190
|
+
"""
|
|
191
|
+
if not dependents:
|
|
192
|
+
return 1
|
|
193
|
+
if percentile >= 97 and counted >= 10 and direct >= 3 and score >= 6:
|
|
194
|
+
return 5
|
|
195
|
+
if percentile >= 85 and counted >= 5 and (direct >= 2 or score >= 2.5):
|
|
196
|
+
return 4
|
|
197
|
+
if percentile >= 55 and (dependents >= 2 or score >= 1):
|
|
198
|
+
return 3
|
|
199
|
+
return 2
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _loose_json(text):
|
|
203
|
+
"""tsconfig.json and its kind allow comments and trailing commas."""
|
|
204
|
+
try:
|
|
205
|
+
return json.loads(text)
|
|
206
|
+
except ValueError:
|
|
207
|
+
try:
|
|
208
|
+
return json.loads(re.sub(r',(\s*[}\]])', r'\1', re.sub(r'^\s*//.*$|/\*.*?\*/', '', text, flags=re.M | re.S)))
|
|
209
|
+
except ValueError:
|
|
210
|
+
return None
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class _Resolver:
|
|
214
|
+
"""Turns what a file imports into the project files it means."""
|
|
215
|
+
|
|
216
|
+
def __init__(self, paths, texts):
|
|
217
|
+
self.paths = set(paths)
|
|
218
|
+
# Folder -> (base folder, [(prefix, wildcard?, [target prefixes])]) from each tsconfig/jsconfig.
|
|
219
|
+
self.aliases = {}
|
|
220
|
+
for path in paths:
|
|
221
|
+
if posixpath.basename(path) in ('tsconfig.json', 'jsconfig.json') and path in texts:
|
|
222
|
+
options = (_loose_json(texts[path]) or {}).get('compilerOptions') or {}
|
|
223
|
+
folder = posixpath.dirname(path)
|
|
224
|
+
base = posixpath.normpath(posixpath.join(folder, options.get('baseUrl') or '.'))
|
|
225
|
+
mapped = []
|
|
226
|
+
for key, targets in (options.get('paths') or {}).items():
|
|
227
|
+
if isinstance(targets, list):
|
|
228
|
+
mapped.append((key.rstrip('*'), key.endswith('*'), [str(t).rstrip('*') for t in targets]))
|
|
229
|
+
self.aliases[folder] = ('' if base == '.' else base, 'baseUrl' in options, mapped)
|
|
230
|
+
|
|
231
|
+
def _file(self, target):
|
|
232
|
+
target = posixpath.normpath(target)
|
|
233
|
+
if target.startswith('..'):
|
|
234
|
+
return None
|
|
235
|
+
if target in self.paths:
|
|
236
|
+
return target
|
|
237
|
+
stem, extension = posixpath.splitext(target)
|
|
238
|
+
# TypeScript is imported by the name it compiles to: ./store.js is store.ts.
|
|
239
|
+
swapped = {'.js': ('.ts', '.tsx'), '.mjs': ('.mts',), '.cjs': ('.cts',), '.jsx': ('.tsx',)}.get(extension, ())
|
|
240
|
+
for candidate in [stem + e for e in swapped] + [target + e for e in JS] + [f'{target}/index{e}' for e in JS]:
|
|
241
|
+
if candidate in self.paths:
|
|
242
|
+
return candidate
|
|
243
|
+
return None
|
|
244
|
+
|
|
245
|
+
def javascript(self, importer, name):
|
|
246
|
+
folder = posixpath.dirname(importer)
|
|
247
|
+
if name.startswith('.'):
|
|
248
|
+
return self._file(posixpath.join(folder, name))
|
|
249
|
+
if name.startswith('/'):
|
|
250
|
+
return None
|
|
251
|
+
# The nearest tsconfig above the importer decides what a bare name means.
|
|
252
|
+
while True:
|
|
253
|
+
if folder in self.aliases:
|
|
254
|
+
base, has_base, mapped = self.aliases[folder]
|
|
255
|
+
for prefix, wildcard, targets in mapped:
|
|
256
|
+
if name.startswith(prefix) if wildcard else name == prefix:
|
|
257
|
+
for target in targets:
|
|
258
|
+
found = self._file(posixpath.join(base, target + (name[len(prefix):] if wildcard else '')))
|
|
259
|
+
if found:
|
|
260
|
+
return found
|
|
261
|
+
if has_base and (found := self._file(posixpath.join(base, name))):
|
|
262
|
+
return found
|
|
263
|
+
if not folder:
|
|
264
|
+
break
|
|
265
|
+
folder = posixpath.dirname(folder)
|
|
266
|
+
if name[:2] in ('@/', '~/'):
|
|
267
|
+
return self._file(name[2:]) or self._file('src/' + name[2:])
|
|
268
|
+
return None
|
|
269
|
+
|
|
270
|
+
def _module(self, base, dotted):
|
|
271
|
+
target = posixpath.normpath(posixpath.join(base, *dotted.split('.'))) if dotted else posixpath.normpath(base or '.')
|
|
272
|
+
if target.startswith('..'):
|
|
273
|
+
return None
|
|
274
|
+
target = '' if target == '.' else target
|
|
275
|
+
for candidate in (target + '.py', posixpath.join(target, '__init__.py')):
|
|
276
|
+
if candidate in self.paths:
|
|
277
|
+
return candidate
|
|
278
|
+
return None
|
|
279
|
+
|
|
280
|
+
def python(self, importer, dots, module, names):
|
|
281
|
+
"""Every project file `from <dots><module> import <names>` (or `import <module>`) loads."""
|
|
282
|
+
folder = posixpath.dirname(importer)
|
|
283
|
+
if dots:
|
|
284
|
+
bases = [posixpath.normpath(posixpath.join(folder, *['..'] * (len(dots) - 1)))]
|
|
285
|
+
else:
|
|
286
|
+
# A script imports its neighbours by bare name; a package is imported from the root or src/.
|
|
287
|
+
bases, at = [], folder
|
|
288
|
+
while True:
|
|
289
|
+
bases.append(at)
|
|
290
|
+
if not at:
|
|
291
|
+
break
|
|
292
|
+
at = posixpath.dirname(at)
|
|
293
|
+
bases.append('src')
|
|
294
|
+
found = []
|
|
295
|
+
for base in bases:
|
|
296
|
+
base = '' if base == '.' else base
|
|
297
|
+
hit = self._module(base, module) if module else None
|
|
298
|
+
# `from package import module` names a file too.
|
|
299
|
+
parts = [self._module(base, '.'.join(filter(None, [module, name]))) for name in names]
|
|
300
|
+
if hit or any(parts):
|
|
301
|
+
found += [p for p in [hit, *parts] if p]
|
|
302
|
+
break
|
|
303
|
+
return found
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _imports(path, text, resolver):
|
|
307
|
+
if path.endswith(JS):
|
|
308
|
+
for match in JS_IMPORTS.finditer(text):
|
|
309
|
+
target = resolver.javascript(path, match.group(1) or match.group(2) or match.group(3))
|
|
310
|
+
if target:
|
|
311
|
+
yield target
|
|
312
|
+
elif path.endswith(PY):
|
|
313
|
+
for dots, module, rest in PY_FROM.findall(text):
|
|
314
|
+
names = [n.split(' as ')[0].strip() for n in rest.rstrip('\\ ').split(',')]
|
|
315
|
+
yield from resolver.python(path, dots, module, [n for n in names if n.isidentifier()])
|
|
316
|
+
for rest in PY_IMPORT.findall(text):
|
|
317
|
+
for name in rest.split(','):
|
|
318
|
+
name = name.split(' as ')[0].strip()
|
|
319
|
+
if re.fullmatch(r'[\w.]+', name):
|
|
320
|
+
yield from resolver.python(path, '', name, [])
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _entry_points(paths, texts, resolver):
|
|
324
|
+
"""Files a manifest or a Dockerfile starts, and those named as the start by convention."""
|
|
325
|
+
found = {}
|
|
326
|
+
roots = {posixpath.dirname(p) for p in paths if posixpath.basename(p) in ('package.json', 'pyproject.toml')} | {''}
|
|
327
|
+
for path in paths:
|
|
328
|
+
folder, name = posixpath.split(path)
|
|
329
|
+
in_root = next((path[len(root):].lstrip('/') for root in roots if not root or path.startswith(root + '/')
|
|
330
|
+
if ENTRY.fullmatch(path[len(root):].lstrip('/')) or FRAMEWORK_ROOT.fullmatch(path[len(root):].lstrip('/'))), None)
|
|
331
|
+
if in_root:
|
|
332
|
+
found.setdefault(path, 'where the program starts')
|
|
333
|
+
if path not in texts:
|
|
334
|
+
continue
|
|
335
|
+
words = []
|
|
336
|
+
if name == 'package.json':
|
|
337
|
+
manifest = _loose_json(texts[path]) or {}
|
|
338
|
+
binaries = manifest.get('bin')
|
|
339
|
+
words += [manifest.get('main'), manifest.get('module')]
|
|
340
|
+
words += list(binaries.values()) if isinstance(binaries, dict) else [binaries]
|
|
341
|
+
for command in (manifest.get('scripts') or {}).values():
|
|
342
|
+
words += re.split(r'[\s"\'=]+', str(command))
|
|
343
|
+
reason = 'started by package.json'
|
|
344
|
+
elif name.lower().startswith('dockerfile') or name.lower().endswith('.dockerfile'):
|
|
345
|
+
for line in re.findall(r'^\s*(?:CMD|ENTRYPOINT)\s+(.*)$', texts[path], re.M | re.I):
|
|
346
|
+
words += re.split(r'[\s"\',\[\]]+', line)
|
|
347
|
+
reason = 'started by the Dockerfile'
|
|
348
|
+
for word in words:
|
|
349
|
+
if isinstance(word, str) and re.search(r'\.(?:[cm]?[jt]sx?|py|sh)$', word):
|
|
350
|
+
target = resolver._file(posixpath.join(folder, word)) or resolver._file(word.lstrip('/'))
|
|
351
|
+
if target:
|
|
352
|
+
found.setdefault(target, reason)
|
|
353
|
+
return found
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def scan(members):
|
|
357
|
+
"""`members`: (path, size, read) for each project file, where read() gives
|
|
358
|
+
its bytes. Returns {'summary': ..., 'files': [one entry per file, most
|
|
359
|
+
dangerous first]}."""
|
|
360
|
+
started = time.monotonic()
|
|
361
|
+
sizes, texts = {}, {}
|
|
362
|
+
for path, size, read in members:
|
|
363
|
+
sizes[path] = size
|
|
364
|
+
name = posixpath.basename(path).lower()
|
|
365
|
+
readable = path.endswith(JS + PY + SHELL + ('.sql',)) or name in ('package.json', 'tsconfig.json', 'jsconfig.json') \
|
|
366
|
+
or name.startswith('dockerfile') or name.endswith('.dockerfile')
|
|
367
|
+
if readable and size <= MAX_READ:
|
|
368
|
+
texts[path] = read().decode('utf-8', 'replace')
|
|
369
|
+
paths = sorted(sizes)
|
|
370
|
+
ruled = {path: rules_level(path) for path in paths}
|
|
371
|
+
held = {path for path in paths if ruled[path][2] == 'ceiling'}
|
|
372
|
+
source = [path for path in paths if path.endswith(JS + PY)]
|
|
373
|
+
counted = {'js': sum(1 for p in source if p.endswith(JS) and p not in held),
|
|
374
|
+
'py': sum(1 for p in source if p.endswith(PY) and p not in held)}
|
|
375
|
+
|
|
376
|
+
resolver = _Resolver(paths, texts)
|
|
377
|
+
importers = {} # file -> the files that import it
|
|
378
|
+
edges = 0
|
|
379
|
+
for path in source:
|
|
380
|
+
if path in texts:
|
|
381
|
+
for target in set(_imports(path, texts[path], resolver)):
|
|
382
|
+
if target != path:
|
|
383
|
+
importers.setdefault(target, set()).add(path)
|
|
384
|
+
edges += 1
|
|
385
|
+
entries = _entry_points(paths, texts, resolver)
|
|
386
|
+
# A dependent's vote: entry points most, then files whose own path already
|
|
387
|
+
# marks them sensitive. Held files (tests, docs) are followed, not counted.
|
|
388
|
+
weights = {path: 0.0 if path in held else 1.5 if path in entries else 1.25
|
|
389
|
+
if ruled[path][2] == 'floor' and (ruled[path][0] or 0) >= 4 else 1.0
|
|
390
|
+
for path in paths}
|
|
391
|
+
group_of = lambda path: 'py' if path.endswith(PY) else 'js'
|
|
392
|
+
metrics = {}
|
|
393
|
+
for path in importers:
|
|
394
|
+
count, direct, score = _weighted_dependents(path, importers, weights)
|
|
395
|
+
metrics[path] = {'dependents': count, 'direct': direct, 'score': score}
|
|
396
|
+
for group in ('js', 'py'):
|
|
397
|
+
scores = sorted(m['score'] for path, m in metrics.items() if group_of(path) == group and m['dependents'])
|
|
398
|
+
for path, metric in metrics.items():
|
|
399
|
+
if group_of(path) != group:
|
|
400
|
+
continue
|
|
401
|
+
metric['percentile'] = 100 if len(scores) <= 1 else round(
|
|
402
|
+
100 * bisect.bisect_left(scores, metric['score']) / (len(scores) - 1))
|
|
403
|
+
tested = {TEST_STEM.match(posixpath.basename(p).split('.')[0]).group(1) for p in paths if ruled[p][1] == 'test'}
|
|
404
|
+
|
|
405
|
+
files, by_level = [], {level: 0 for level in range(1, 6)}
|
|
406
|
+
for path in paths:
|
|
407
|
+
rule_level, rule_words, rule_kind = ruled[path]
|
|
408
|
+
text = texts.get(path)
|
|
409
|
+
group = 'py' if path.endswith(PY) else 'js'
|
|
410
|
+
reasons, flags, reach, hazard = [], [], None, None
|
|
411
|
+
level = 1
|
|
412
|
+
metric = metrics.get(path)
|
|
413
|
+
if metric:
|
|
414
|
+
count = metric['dependents']
|
|
415
|
+
others = max(counted[group] - (path in source and path not in held), 1)
|
|
416
|
+
reach = {'level': _reach_level(metric['score'], metric.get('percentile', 0), metric['direct'], count, counted[group]),
|
|
417
|
+
'dependents': count, 'of': others, 'direct': metric['direct'],
|
|
418
|
+
'score': round(metric['score'], 3), 'percentile': metric.get('percentile', 0)}
|
|
419
|
+
if count:
|
|
420
|
+
reasons.append(f'{count} of {others} source files depend on this'
|
|
421
|
+
+ (f", {metric['direct']} directly" if metric['direct'] else ''))
|
|
422
|
+
if path in entries:
|
|
423
|
+
reach = {**(reach or {'dependents': 0, 'direct': 0, 'score': 0, 'percentile': 0,
|
|
424
|
+
'of': max(counted[group] - 1, 1)}),
|
|
425
|
+
'level': max(4, (reach or {}).get('level', 1)), 'entry': True}
|
|
426
|
+
reasons.insert(0, entries[path])
|
|
427
|
+
elif starter := next((i for i in sorted(importers.get(path, ())) if i in entries), None):
|
|
428
|
+
# One step from the start: few files import it, and the program does not come up without it.
|
|
429
|
+
reach = {**reach, 'level': max(3, reach['level'])}
|
|
430
|
+
reasons.append(f'loaded at startup by {starter}')
|
|
431
|
+
if reach:
|
|
432
|
+
level = reach['level']
|
|
433
|
+
marked = []
|
|
434
|
+
if text is not None and path.endswith(JS + PY + SHELL + ('.sql',)) and rule_kind != 'ceiling':
|
|
435
|
+
found = [(marker_level, words) for marker_level, words, pattern in CONTENT_MARKERS
|
|
436
|
+
if pattern.search(text, 0, MAX_MARKED)]
|
|
437
|
+
marked = list(dict.fromkeys(words for _, words in found))
|
|
438
|
+
if found:
|
|
439
|
+
content_level, content_words = max(found, key=lambda item: item[0])
|
|
440
|
+
if not hazard and rule_kind != 'floor':
|
|
441
|
+
hazard = {'level': content_level, 'rule': content_words}
|
|
442
|
+
elif rule_kind == 'floor' and content_level > rule_level:
|
|
443
|
+
hazard = {'level': content_level, 'rule': content_words}
|
|
444
|
+
if rule_kind == 'floor':
|
|
445
|
+
hazard = hazard if hazard and hazard['level'] > rule_level else {'level': rule_level, 'rule': rule_words}
|
|
446
|
+
if hazard:
|
|
447
|
+
if marked:
|
|
448
|
+
hazard = {**hazard, 'does': marked}
|
|
449
|
+
level = max(level, hazard['level'])
|
|
450
|
+
reasons.append(hazard['rule'])
|
|
451
|
+
reasons += [words for words in dict.fromkeys([rule_words] if rule_kind == 'floor' else []) if words != hazard['rule']]
|
|
452
|
+
reasons += [words for words in marked if words not in reasons]
|
|
453
|
+
if rule_kind == 'ceiling':
|
|
454
|
+
level = min(level, rule_level)
|
|
455
|
+
reasons.append(rule_words)
|
|
456
|
+
untested = bool(path in source and path not in held and tested
|
|
457
|
+
and posixpath.basename(path).split('.')[0] not in tested)
|
|
458
|
+
# Untested can turn Watch into Review carefully, but never creates a
|
|
459
|
+
# Lock down on its own: most production files have no like-named test.
|
|
460
|
+
if untested and reach and reach['level'] == 3 and rule_kind != 'ceiling' and level == 3:
|
|
461
|
+
level = 4
|
|
462
|
+
reasons.append('widely used and no like-named test covers it')
|
|
463
|
+
if path in source and path not in held:
|
|
464
|
+
if untested:
|
|
465
|
+
flags.append('untested')
|
|
466
|
+
if edges >= 10 and path not in importers and path not in entries and not FRAMEWORK_LOADED.fullmatch(path) \
|
|
467
|
+
and (text is None or '__main__' not in text):
|
|
468
|
+
flags.append('unreferenced')
|
|
469
|
+
if text is not None and text.count('\n') > LARGE_LINES:
|
|
470
|
+
flags.append('large')
|
|
471
|
+
by_level[level] += 1
|
|
472
|
+
files.append({'path': path, 'level': level, 'reach': reach, 'hazard': hazard, 'flags': flags, 'reasons': reasons})
|
|
473
|
+
files.sort(key=lambda f: (-f['level'], -((f['reach'] or {}).get('score', 0)),
|
|
474
|
+
-((f['reach'] or {}).get('dependents', 0)), f['path']))
|
|
475
|
+
return {'files': files, 'summary': {'version': VERSION, 'files': len(files), 'by_level': by_level, 'edges': edges,
|
|
476
|
+
'source_files': counted, 'scanned_ms': round((time.monotonic() - started) * 1000)}}
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def scan_archive(data):
|
|
480
|
+
"""scan() over a project .tar.gz as control_api.screened() leaves it."""
|
|
481
|
+
with tarfile.open(fileobj=io.BytesIO(data), mode='r:gz') as archive:
|
|
482
|
+
members = [item for item in archive if item.isfile()]
|
|
483
|
+
return scan((item.name, item.size, lambda item=item: archive.extractfile(item).read()) for item in members)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def created_entry(path):
|
|
487
|
+
"""The entry for a file that appeared after the scan: scored by its path alone."""
|
|
488
|
+
level, words, kind = rules_level(path)
|
|
489
|
+
return {'path': path, 'level': level or 1, 'reach': None, 'flags': ['created'],
|
|
490
|
+
'hazard': {'level': level, 'rule': words} if kind == 'floor' else None,
|
|
491
|
+
'reasons': ['created after the initial scan'] + ([words] if words else [])}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# The Linux VM that runs local RayTrace sandboxes (`raytrace sandbox create --local`).
|
|
2
|
+
# No vmType: Lima picks the host's own -- vz on macOS, qemu (KVM) on Linux,
|
|
3
|
+
# qemu on Windows (experimental there). Override with `limactl start --vm-type`.
|
|
4
|
+
cpus: 4
|
|
5
|
+
memory: 4GiB
|
|
6
|
+
disk: 30GiB
|
|
7
|
+
mounts: []
|
|
8
|
+
containerd:
|
|
9
|
+
system: false
|
|
10
|
+
user: false
|
|
11
|
+
images:
|
|
12
|
+
- location: https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img
|
|
13
|
+
arch: aarch64
|
|
14
|
+
- location: https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img
|
|
15
|
+
arch: x86_64
|
|
16
|
+
portForwards:
|
|
17
|
+
- guestPort: 8798
|
|
18
|
+
hostPort: 8798
|
|
19
|
+
- guestPortRange: [1, 65535]
|
|
20
|
+
proto: any
|
|
21
|
+
ignore: true
|
|
22
|
+
provision:
|
|
23
|
+
- mode: system
|
|
24
|
+
script: |
|
|
25
|
+
#!/bin/sh
|
|
26
|
+
set -eu
|
|
27
|
+
apt-get update
|
|
28
|
+
DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io python3 ca-certificates curl
|
|
29
|
+
systemctl enable --now docker
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Dedicated VM firewall: agent egress cannot reach private networks or VM services."""
|
|
2
|
+
import os
|
|
3
|
+
import subprocess
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
# Lima's user-mode network provides DNS at this address, and the Mac's model
|
|
7
|
+
# bridge behind it.
|
|
8
|
+
LIMA_HOST = '192.168.5.2'
|
|
9
|
+
# Elsewhere the sandbox manager reaches this host over SSH and listens on the
|
|
10
|
+
# sandbox network's own gateway (a reverse tunnel, see control_api.py)...
|
|
11
|
+
GATEWAY = '172.30.60.1'
|
|
12
|
+
# What sandboxes read as /etc/resolv.conf: the host's own forwarder (dns_forwarder.py) on
|
|
13
|
+
# their gateway, which writes down each lookup and sends it on to UPSTREAM.
|
|
14
|
+
RESOLV = Path('/etc/raytace/resolv.conf')
|
|
15
|
+
UPSTREAM = Path('/etc/raytace/upstream-dns')
|
|
16
|
+
# Where a sandbox may not connect: private, link-local and multicast ranges,
|
|
17
|
+
# and with them the VM's services and the cloud metadata server. engine.py
|
|
18
|
+
# reads this list to call a connection there refused, whatever the call returned.
|
|
19
|
+
REJECTED = ['0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8',
|
|
20
|
+
'169.254.0.0/16', '172.16.0.0/12', '192.168.0.0/16', '224.0.0.0/4']
|
|
21
|
+
# The manager's bridge as sandboxes address it; projects.py puts it in their env.
|
|
22
|
+
BRIDGE = Path('/etc/raytace/bridge')
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def run(*args):
|
|
26
|
+
subprocess.run(args, check=True)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def resolver():
|
|
30
|
+
"""The resolver the host's forwarder sends sandboxes' lookups on to."""
|
|
31
|
+
# RAYTACE_DNS is set once at setup (sudo drops it on later runs from
|
|
32
|
+
# run.py), so the address already written wins after that. A cloud VM
|
|
33
|
+
# sets a public resolver: its own DNS server is the metadata server
|
|
34
|
+
# (169.254.169.254 on GCP), which the rules below must keep blocked.
|
|
35
|
+
if os.environ.get('RAYTACE_DNS'):
|
|
36
|
+
return os.environ['RAYTACE_DNS']
|
|
37
|
+
if UPSTREAM.exists():
|
|
38
|
+
return UPSTREAM.read_text().split()[0]
|
|
39
|
+
# A host set up before the forwarder: its sandboxes asked the resolver themselves.
|
|
40
|
+
if RESOLV.exists() and RESOLV.read_text().split()[1] != GATEWAY:
|
|
41
|
+
return RESOLV.read_text().split()[1]
|
|
42
|
+
return LIMA_HOST
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def private_bridge():
|
|
46
|
+
"""...or, hosted, the control plane's bridge at its private address:port
|
|
47
|
+
(RAYTACE_BRIDGE, set once at setup like RAYTACE_DNS, then kept in BRIDGE)."""
|
|
48
|
+
if os.environ.get('RAYTACE_BRIDGE'):
|
|
49
|
+
return os.environ['RAYTACE_BRIDGE']
|
|
50
|
+
written = BRIDGE.read_text().strip().removeprefix('http://') if BRIDGE.exists() else ''
|
|
51
|
+
return written if written and not written.startswith((LIMA_HOST + ':', GATEWAY + ':')) else None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
if __name__ == '__main__':
|
|
55
|
+
dns = resolver()
|
|
56
|
+
UPSTREAM.write_text(dns + '\n')
|
|
57
|
+
RESOLV.write_text(f'nameserver {GATEWAY}\n')
|
|
58
|
+
if subprocess.run(['docker', 'network', 'inspect', 'raytace-egress'], stdout=subprocess.DEVNULL,
|
|
59
|
+
stderr=subprocess.DEVNULL).returncode:
|
|
60
|
+
run('docker', 'network', 'create', '--driver=bridge', '--subnet=172.30.60.0/24',
|
|
61
|
+
'--opt=com.docker.network.bridge.name=br-raytace', 'raytace-egress')
|
|
62
|
+
def rule(chain, *args):
|
|
63
|
+
if subprocess.run(['iptables', '-C', chain, *args], stderr=subprocess.DEVNULL).returncode:
|
|
64
|
+
run('iptables', '-I', chain, '1', *args)
|
|
65
|
+
for destination in REJECTED:
|
|
66
|
+
rule('DOCKER-USER', '-i', 'br-raytace', '-d', destination, '-j', 'REJECT')
|
|
67
|
+
rule('INPUT', '-i', 'br-raytace', '-j', 'REJECT')
|
|
68
|
+
# Only DNS is exempted, and only at the host's forwarder on the gateway.
|
|
69
|
+
# The rule that let sandboxes ask the resolver itself is taken away: a
|
|
70
|
+
# lookup that went round the forwarder would be one nobody wrote down.
|
|
71
|
+
for protocol in ['udp', 'tcp']:
|
|
72
|
+
rule('INPUT', '-i', 'br-raytace', '-d', GATEWAY, '-p', protocol, '--dport', '53', '-j', 'ACCEPT')
|
|
73
|
+
former = ['DOCKER-USER', '-i', 'br-raytace', '-d', dns, '-p', protocol, '--dport', '53', '-j', 'ACCEPT']
|
|
74
|
+
if not subprocess.run(['iptables', '-C', *former], stderr=subprocess.DEVNULL).returncode:
|
|
75
|
+
run('iptables', '-D', *former)
|
|
76
|
+
# Only the authenticated model and transcript bridge. Its management API rejects this Host.
|
|
77
|
+
bridge = private_bridge()
|
|
78
|
+
if bridge:
|
|
79
|
+
# Above the private-range rejects: this one address and port, nothing else on the VPC.
|
|
80
|
+
address, port = bridge.rsplit(':', 1)
|
|
81
|
+
rule('DOCKER-USER', '-i', 'br-raytace', '-d', address, '-p', 'tcp', '--dport', port, '-j', 'ACCEPT')
|
|
82
|
+
BRIDGE.write_text(f'http://{bridge}\n')
|
|
83
|
+
elif dns == LIMA_HOST:
|
|
84
|
+
rule('DOCKER-USER', '-i', 'br-raytace', '-d', LIMA_HOST,
|
|
85
|
+
'-p', 'tcp', '--dport', '8799', '-j', 'ACCEPT')
|
|
86
|
+
BRIDGE.write_text(f'http://{LIMA_HOST}:8799\n')
|
|
87
|
+
else:
|
|
88
|
+
rule('INPUT', '-i', 'br-raytace', '-d', GATEWAY,
|
|
89
|
+
'-p', 'tcp', '--dport', '8799', '-j', 'ACCEPT')
|
|
90
|
+
BRIDGE.write_text(f'http://{GATEWAY}:8799\n')
|