jscope 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jscope/__init__.py +1 -0
- jscope/__main__.py +10 -0
- jscope/ai/__init__.py +0 -0
- jscope/ai/explain.py +180 -0
- jscope/ai/providers.py +71 -0
- jscope/analyzers/__init__.py +0 -0
- jscope/analyzers/auth_analyzer.py +170 -0
- jscope/analyzers/authorization_analyzer.py +196 -0
- jscope/analyzers/base.py +56 -0
- jscope/analyzers/common.py +78 -0
- jscope/analyzers/dataflow_analyzer.py +192 -0
- jscope/analyzers/endpoint_analyzer.py +150 -0
- jscope/analyzers/framework_detector.py +82 -0
- jscope/analyzers/graphql_analyzer.py +198 -0
- jscope/analyzers/graphql_schema_analyzer.py +132 -0
- jscope/analyzers/html_analyzer.py +196 -0
- jscope/analyzers/manager.py +114 -0
- jscope/analyzers/openapi_analyzer.py +278 -0
- jscope/analyzers/router_analyzer.py +502 -0
- jscope/analyzers/secret_analyzer.py +99 -0
- jscope/analyzers/service_worker_analyzer.py +142 -0
- jscope/analyzers/source_map_analyzer.py +119 -0
- jscope/analyzers/storage_analyzer.py +125 -0
- jscope/analyzers/wasm_analyzer.py +139 -0
- jscope/assessment_engine.py +285 -0
- jscope/assessment_reporting.py +71 -0
- jscope/benchmark/__init__.py +1 -0
- jscope/benchmark/fixtures.py +105 -0
- jscope/benchmark/runner.py +155 -0
- jscope/cli.py +1241 -0
- jscope/config.py +72 -0
- jscope/core/__init__.py +0 -0
- jscope/core/artifacts.py +139 -0
- jscope/core/cache.py +198 -0
- jscope/core/classification.py +56 -0
- jscope/core/evidence.py +411 -0
- jscope/core/graph.py +196 -0
- jscope/core/redact.py +67 -0
- jscope/crawler/__init__.py +0 -0
- jscope/crawler/crawler.py +176 -0
- jscope/dataflow/__init__.py +1 -0
- jscope/dataflow/engine.py +1684 -0
- jscope/dataflow/modules.py +205 -0
- jscope/dataflow/rules.py +118 -0
- jscope/downloader/__init__.py +0 -0
- jscope/downloader/downloader.py +116 -0
- jscope/parsing/__init__.py +0 -0
- jscope/parsing/ast_engine.py +322 -0
- jscope/parsing/graphql_schema.py +153 -0
- jscope/parsing/html.py +55 -0
- jscope/parsing/queries.py +385 -0
- jscope/parsing/wasm.py +146 -0
- jscope/passive_discovery.py +63 -0
- jscope/pipeline.py +196 -0
- jscope/plugin/__init__.py +0 -0
- jscope/plugin/_worker.py +199 -0
- jscope/plugin/loader.py +225 -0
- jscope/plugin/sandbox.py +207 -0
- jscope/reporting/__init__.py +0 -0
- jscope/reporting/formats.py +208 -0
- jscope/reporting/human.py +137 -0
- jscope/reporting/json_report.py +35 -0
- jscope/reporting/priority.py +116 -0
- jscope/runtime/__init__.py +0 -0
- jscope/runtime/observe.py +397 -0
- jscope/runtime/prober.py +259 -0
- jscope/runtime/scope.py +288 -0
- jscope/security_transport.py +246 -0
- jscope/verify/__init__.py +1 -0
- jscope/verify/engine.py +290 -0
- jscope/workspace/__init__.py +0 -0
- jscope/workspace/assessment.py +133 -0
- jscope/workspace/common.py +25 -0
- jscope/workspace/diff.py +143 -0
- jscope/workspace/store.py +206 -0
- jscope-0.4.2.dist-info/METADATA +180 -0
- jscope-0.4.2.dist-info/RECORD +81 -0
- jscope-0.4.2.dist-info/WHEEL +5 -0
- jscope-0.4.2.dist-info/entry_points.txt +2 -0
- jscope-0.4.2.dist-info/licenses/LICENSE +21 -0
- jscope-0.4.2.dist-info/top_level.txt +1 -0
jscope/pipeline.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import fnmatch
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from jscope.analyzers.auth_analyzer import AuthAnalyzer
|
|
8
|
+
from jscope.analyzers.authorization_analyzer import AuthorizationAnalyzer
|
|
9
|
+
from jscope.analyzers.base import Analyzer, ProjectContext
|
|
10
|
+
from jscope.analyzers.dataflow_analyzer import DataFlowAnalyzer
|
|
11
|
+
from jscope.analyzers.endpoint_analyzer import EndpointAnalyzer
|
|
12
|
+
from jscope.analyzers.framework_detector import FrameworkDetector
|
|
13
|
+
from jscope.analyzers.graphql_analyzer import GraphQLAnalyzer
|
|
14
|
+
from jscope.analyzers.graphql_schema_analyzer import GraphQLSchemaAnalyzer
|
|
15
|
+
from jscope.analyzers.html_analyzer import HtmlAnalyzer
|
|
16
|
+
from jscope.analyzers.openapi_analyzer import OpenApiAnalyzer
|
|
17
|
+
from jscope.analyzers.wasm_analyzer import WasmAnalyzer
|
|
18
|
+
from jscope.analyzers.manager import AnalyzerManager
|
|
19
|
+
from jscope.analyzers.router_analyzer import RouterAnalyzer
|
|
20
|
+
from jscope.analyzers.secret_analyzer import SecretAnalyzer
|
|
21
|
+
from jscope.analyzers.service_worker_analyzer import ServiceWorkerAnalyzer
|
|
22
|
+
from jscope.analyzers.source_map_analyzer import SourceMapAnalyzer, expand_source_maps
|
|
23
|
+
from jscope.analyzers.storage_analyzer import StorageAnalyzer
|
|
24
|
+
from jscope.config import Config
|
|
25
|
+
from jscope.core.artifacts import collect_artifacts
|
|
26
|
+
from jscope.core.evidence import Confidence, EvidenceEngine, Finding
|
|
27
|
+
from jscope.core.graph import KnowledgeGraph
|
|
28
|
+
from jscope.parsing.ast_engine import ASTEngine
|
|
29
|
+
from jscope.parsing.html import extract_inline_scripts
|
|
30
|
+
from jscope.core.graph import EdgeKind, NodeKind
|
|
31
|
+
from jscope.plugin.loader import load_plugins
|
|
32
|
+
|
|
33
|
+
CORE_ANALYZERS = [FrameworkDetector, EndpointAnalyzer, SecretAnalyzer, RouterAnalyzer, AuthAnalyzer,
|
|
34
|
+
AuthorizationAnalyzer, StorageAnalyzer, GraphQLAnalyzer, ServiceWorkerAnalyzer,
|
|
35
|
+
SourceMapAnalyzer, HtmlAnalyzer, OpenApiAnalyzer, GraphQLSchemaAnalyzer, WasmAnalyzer,
|
|
36
|
+
DataFlowAnalyzer]
|
|
37
|
+
_EXTS = (".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx", ".mts", ".cts")
|
|
38
|
+
_NOT_JS = (".html", ".htm", ".vue", ".wasm", ".graphql", ".gql", ".graphqls", ".json", ".yaml", ".yml", ".d.ts")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def core_analyzers() -> list[Analyzer]:
|
|
42
|
+
return [cls() for cls in CORE_ANALYZERS]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# kept for backward compatibility with Phase 1 callers
|
|
46
|
+
ALL_ANALYZERS = core_analyzers()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class AnalysisResult:
|
|
51
|
+
findings: list[Finding]
|
|
52
|
+
frameworks: list[str]
|
|
53
|
+
file_count: int
|
|
54
|
+
engine: EvidenceEngine
|
|
55
|
+
graph: KnowledgeGraph
|
|
56
|
+
diagnostics: list[str] = field(default_factory=list)
|
|
57
|
+
analyzer_counts: dict[str, int] = field(default_factory=dict)
|
|
58
|
+
tiers: dict[str, str] = field(default_factory=dict)
|
|
59
|
+
expanded_sources: int = 0
|
|
60
|
+
file_hashes: dict[str, str] = field(default_factory=dict) # path -> content hash (disk files only)
|
|
61
|
+
artifact_counts: dict[str, int] = field(default_factory=dict) # Phase 4: kind -> files analyzed
|
|
62
|
+
dataflow: dict = field(default_factory=dict) # 0.5: every flow incl. sanitized/unreachable + limitations
|
|
63
|
+
file_diagnostics: dict[str, list[str]] = field(default_factory=dict) # 0.5: per-file "skipped / partially analyzed"
|
|
64
|
+
plugin_runs: list[dict] = field(default_factory=list) # 0.5: plugin identity/version/isolation/outcome
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _collect_js_files(paths: list[str], exclude: list[str]) -> list[str]:
|
|
68
|
+
out: list[str] = []
|
|
69
|
+
for p in paths:
|
|
70
|
+
path = Path(p)
|
|
71
|
+
if path.is_file():
|
|
72
|
+
if path.suffix.lower() not in _NOT_JS: # html/vue/wasm/spec files are artifacts, not JS (Phase 4)
|
|
73
|
+
out.append(str(path))
|
|
74
|
+
elif path.is_dir():
|
|
75
|
+
for f in path.rglob("*"):
|
|
76
|
+
if f.suffix not in _EXTS or not f.is_file():
|
|
77
|
+
continue
|
|
78
|
+
rel = str(f)
|
|
79
|
+
if any(fnmatch.fnmatch(rel, pat) for pat in exclude):
|
|
80
|
+
continue
|
|
81
|
+
out.append(str(f))
|
|
82
|
+
return sorted(set(out))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def collect_manifest(paths: list[str], config: Config) -> dict[str, str]:
|
|
86
|
+
"""path -> content hash for every file that can influence an analysis. Cheap (no parsing).
|
|
87
|
+
Includes sibling `.map` files when source-map expansion is on, since they change the result."""
|
|
88
|
+
exclude = config.get("exclude", [])
|
|
89
|
+
files = _collect_js_files(paths, exclude)
|
|
90
|
+
if config.get("source_maps.expand", False):
|
|
91
|
+
for p in paths:
|
|
92
|
+
if Path(p).is_dir():
|
|
93
|
+
files += [str(f) for f in Path(p).rglob("*.map") if f.is_file()]
|
|
94
|
+
out = {f: ASTEngine.hash_bytes(Path(f).read_bytes()) for f in sorted(set(files))}
|
|
95
|
+
if config.get("artifacts.enabled", True):
|
|
96
|
+
for a in collect_artifacts(paths, exclude):
|
|
97
|
+
out[a.path] = a.content_hash
|
|
98
|
+
return out
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def run_analysis(paths: list[str], config: Config) -> AnalysisResult:
|
|
102
|
+
from jscope.analyzers.common import set_analysis_roots
|
|
103
|
+
set_analysis_roots(paths)
|
|
104
|
+
exclude = config.get("exclude", [])
|
|
105
|
+
files = _collect_js_files(paths, exclude)
|
|
106
|
+
|
|
107
|
+
ast_engine = ASTEngine(max_bytes=config.get("max_file_bytes", 5 * 1024 * 1024))
|
|
108
|
+
parsed = [ast_engine.parse_file(f) for f in files]
|
|
109
|
+
|
|
110
|
+
diagnostics: list[str] = []
|
|
111
|
+
artifacts = collect_artifacts(paths, exclude) if config.get("artifacts.enabled", True) else []
|
|
112
|
+
for art in artifacts:
|
|
113
|
+
if art.kind in ("html", "vue"): # inline <script> / .vue <script> become ordinary JS files
|
|
114
|
+
pfs, d = extract_inline_scripts(art, ast_engine)
|
|
115
|
+
parsed += pfs
|
|
116
|
+
diagnostics += d
|
|
117
|
+
expanded = 0
|
|
118
|
+
if config.get("source_maps.expand", False):
|
|
119
|
+
extra, expanded = expand_source_maps(parsed, ast_engine)
|
|
120
|
+
parsed += extra
|
|
121
|
+
|
|
122
|
+
ctx = ProjectContext(files=parsed, ast=ast_engine, config=config.data, artifacts=artifacts)
|
|
123
|
+
if config.get("incremental.cache_dir") and config.get("incremental.enabled", True):
|
|
124
|
+
from jscope.core.cache import AnalysisCache
|
|
125
|
+
ctx.shared["dataflow_cache"] = AnalysisCache(config.get("incremental.cache_dir"))
|
|
126
|
+
|
|
127
|
+
analyzers = core_analyzers()
|
|
128
|
+
core_names = {a.name for a in analyzers}
|
|
129
|
+
from jscope.plugin.sandbox import SandboxPolicy
|
|
130
|
+
plugin_res = load_plugins(config.get("plugins.paths", []), core_names,
|
|
131
|
+
set(config.get("plugins.verified", [])),
|
|
132
|
+
use_entry_points=config.get("plugins.entry_points", True),
|
|
133
|
+
isolation=config.get("plugins.isolation", "auto"),
|
|
134
|
+
sandbox_policy=SandboxPolicy(timeout_seconds=float(config.get("plugins.timeout_seconds", 30)),
|
|
135
|
+
memory_mb=int(config.get("plugins.memory_mb", 1024)),
|
|
136
|
+
grant_read=list(config.get("plugins.grant_read", []))))
|
|
137
|
+
analyzers += plugin_res.analyzers
|
|
138
|
+
diagnostics += [f"[plugin-loader] {e}" for e in plugin_res.errors]
|
|
139
|
+
diagnostics += [f"[plugin-isolation] {w}" for w in plugin_res.warnings]
|
|
140
|
+
|
|
141
|
+
enabled = set(config.get("analyzers.enabled", [a.name for a in analyzers]))
|
|
142
|
+
enabled |= {a.name for a in plugin_res.analyzers} # loaded plugins are opt-in by being configured
|
|
143
|
+
tiers = {a.name: a.tier for a in analyzers}
|
|
144
|
+
|
|
145
|
+
min_conf = Confidence[config.get("min_confidence", "LOW")]
|
|
146
|
+
cap = Confidence[config.get("plugins.community_confidence_cap", "MEDIUM")]
|
|
147
|
+
engine = EvidenceEngine(min_display_confidence=min_conf, analyzer_tiers=tiers, community_cap=cap)
|
|
148
|
+
|
|
149
|
+
manager = AnalyzerManager(analyzers)
|
|
150
|
+
counts = manager.run(ctx, engine, enabled=enabled)
|
|
151
|
+
findings = engine.promote()
|
|
152
|
+
_attach_findings_to_graph(ctx.graph, findings)
|
|
153
|
+
for pf in parsed: # surface per-file parse problems even if no analyzer needed them
|
|
154
|
+
if pf.skipped_reason:
|
|
155
|
+
ctx.add_file_diagnostic(pf.path, f"not analyzed: {pf.skipped_reason}")
|
|
156
|
+
elif pf.syntax_errors:
|
|
157
|
+
ctx.add_file_diagnostic(pf.path, f"partially analyzed: {pf.syntax_errors} syntax error node(s)")
|
|
158
|
+
|
|
159
|
+
return AnalysisResult(
|
|
160
|
+
findings=findings, frameworks=sorted(ctx.framework_hints.keys()),
|
|
161
|
+
file_count=sum(1 for p in parsed if p.origin not in ("html-inline", "vue-sfc")) + len(artifacts),
|
|
162
|
+
engine=engine, graph=ctx.graph, diagnostics=diagnostics + ctx.diagnostics,
|
|
163
|
+
analyzer_counts=counts, tiers=tiers, expanded_sources=expanded,
|
|
164
|
+
file_hashes={**{p.path: p.content_hash for p in parsed if p.origin == "disk"}, **{a.path: a.content_hash for a in artifacts}},
|
|
165
|
+
artifact_counts={k: sum(1 for a in artifacts if a.kind == k) for k in sorted({a.kind for a in artifacts})},
|
|
166
|
+
dataflow=ctx.shared.get("dataflow", {}), file_diagnostics=ctx.file_diagnostics,
|
|
167
|
+
plugin_runs=[r.to_dict() for r in ctx.shared.get("plugin_runs", [])] + [
|
|
168
|
+
{"plugin": plugin_res.sources.get(a.name, "?"), "analyzer": a.name, "version": a.version, "tier": a.tier,
|
|
169
|
+
"isolation": "none", "outcome": "ok" if a.name in enabled else "skipped"}
|
|
170
|
+
for a in plugin_res.analyzers if a.name not in plugin_res.isolated])
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def analyze_paths(paths: list[str], config: Config) -> tuple[list[Finding], list[str], int, EvidenceEngine]:
|
|
174
|
+
"""Phase 1 signature, preserved."""
|
|
175
|
+
r = run_analysis(paths, config)
|
|
176
|
+
return r.findings, r.frameworks, r.file_count, r.engine
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _attach_findings_to_graph(graph, findings) -> None:
|
|
180
|
+
"""FR-05: link each promoted Finding back into the graph as a first-class node, so a UI/CLI consumer can walk
|
|
181
|
+
finding -> evidence files -> related data-flow / route / endpoint context in one structure."""
|
|
182
|
+
for f in findings:
|
|
183
|
+
fid = f"finding:{f.id}"
|
|
184
|
+
graph.add_node(fid, NodeKind.FINDING, f.subject, category=f.category, confidence=f.confidence.value, score=f.score)
|
|
185
|
+
flow = f.metadata.get("flow")
|
|
186
|
+
if flow:
|
|
187
|
+
sink_id = f"dsink:{flow['sink']['file']}:{flow['sink']['line']}:{flow['sink']['name']}"
|
|
188
|
+
if sink_id in graph.nodes:
|
|
189
|
+
graph.add_edge(sink_id, fid, EdgeKind.DESCRIBES, "evidence_engine", "DATA_FLOW")
|
|
190
|
+
seen_files = set()
|
|
191
|
+
for e in f.evidence:
|
|
192
|
+
if e.file and e.file not in seen_files:
|
|
193
|
+
seen_files.add(e.file)
|
|
194
|
+
file_node = f"file:{e.file}"
|
|
195
|
+
graph.add_node(file_node, NodeKind.FILE, e.file.rsplit("/", 1)[-1])
|
|
196
|
+
graph.add_edge(fid, file_node, EdgeKind.SUPPORTED_BY, "evidence_engine", e.type.value, e.file, e.line)
|
|
File without changes
|
jscope/plugin/_worker.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Sandbox worker (FR-09). Runs in a SEPARATE Python process started with `-I` by jscope.plugin.sandbox.
|
|
2
|
+
|
|
3
|
+
Defence-in-depth, not a hard security boundary (see docs/PLUGIN_ISOLATION.md):
|
|
4
|
+
1. scrubbed environment, empty temp cwd, request on stdin, result on the original stdout as one JSON line
|
|
5
|
+
2. POSIX resource limits: CPU seconds, address space, open files, file size 0 (no file can be written)
|
|
6
|
+
3. a Python audit hook (cannot be removed once installed) that denies: new sockets / DNS, process creation, ctypes,
|
|
7
|
+
any write/delete/rename/chmod, reads outside an allow-list of roots, and imports of network/process modules.
|
|
8
|
+
Every denied attempt is recorded and returned to the parent as a `violation`.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import base64
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
|
|
18
|
+
_REAL_STDOUT = sys.stdout
|
|
19
|
+
_violations: list[str] = []
|
|
20
|
+
_BLOCKED_IMPORTS = {"ctypes", "_ctypes", "subprocess", "_posixsubprocess", "socket", "_socket", "ssl", "_ssl", "multiprocessing",
|
|
21
|
+
"http.client", "urllib.request", "ftplib", "smtplib", "telnetlib", "poplib", "imaplib", "asyncio.subprocess",
|
|
22
|
+
"requests", "urllib3", "httpx", "aiohttp", "pty", "webbrowser", "mmap", "_multiprocessing", "ensurepip"}
|
|
23
|
+
_DENY_ALWAYS = {
|
|
24
|
+
"socket.__new__", "socket.connect", "socket.bind", "socket.getaddrinfo", "socket.gethostbyname", "socket.gethostbyaddr",
|
|
25
|
+
"socket.sendto", "socket.sendmsg", "subprocess.Popen", "os.system", "os.exec", "os.posix_spawn", "os.spawn", "os.fork",
|
|
26
|
+
"os.forkpty", "os.startfile", "ctypes.dlopen", "ctypes.dlsym", "ctypes.cdata", "ctypes.call_function",
|
|
27
|
+
"os.remove", "os.rename", "os.mkdir", "os.rmdir", "os.chmod", "os.chown", "os.link", "os.symlink", "os.truncate",
|
|
28
|
+
"os.mkfifo", "os.mknod", "os.chroot", "os.kill", "os.killpg", "os.setuid", "os.setgid", "shutil.copyfile", "shutil.copytree",
|
|
29
|
+
"shutil.rmtree", "shutil.move", "shutil.make_archive", "shutil.unpack_archive", "tempfile.mkstemp", "tempfile.mkdtemp",
|
|
30
|
+
"winreg.OpenKey", "webbrowser.open", "pty.spawn", "sys.addaudithook_removed",
|
|
31
|
+
}
|
|
32
|
+
_READ_ROOTS: list[str] = []
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _deny(event: str, detail: str = "") -> None:
|
|
36
|
+
msg = f"{event}{(' ' + detail) if detail else ''}"
|
|
37
|
+
if msg not in _violations:
|
|
38
|
+
_violations.append(msg)
|
|
39
|
+
raise PermissionError(f"JScope sandbox: {msg} is not permitted for plugins")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _under(path: str) -> bool:
|
|
43
|
+
try:
|
|
44
|
+
p = os.path.realpath(path)
|
|
45
|
+
except (OSError, ValueError):
|
|
46
|
+
return False
|
|
47
|
+
return any(p == r or p.startswith(r.rstrip(os.sep) + os.sep) for r in _READ_ROOTS)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _hook(event: str, args: tuple) -> None:
|
|
51
|
+
if event in _DENY_ALWAYS:
|
|
52
|
+
_deny(event, repr(args[:1])[:80])
|
|
53
|
+
if event == "import":
|
|
54
|
+
name = str(args[0])
|
|
55
|
+
if name in _BLOCKED_IMPORTS or name.split(".")[0] in {"ctypes", "_ctypes", "subprocess", "socket", "multiprocessing"}:
|
|
56
|
+
_deny("import", name)
|
|
57
|
+
return
|
|
58
|
+
if event == "open":
|
|
59
|
+
path, mode, flags = args[0], args[1], args[2] if len(args) > 2 else 0
|
|
60
|
+
if isinstance(path, int):
|
|
61
|
+
return # already-open fd
|
|
62
|
+
m = mode or "r"
|
|
63
|
+
if any(c in str(m) for c in "wax+") or (isinstance(flags, int) and flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC | os.O_APPEND)):
|
|
64
|
+
_deny("open-for-write", str(path)[:80])
|
|
65
|
+
if not _under(os.fsdecode(path)):
|
|
66
|
+
_deny("read-outside-allowed-roots", str(path)[:80])
|
|
67
|
+
return
|
|
68
|
+
if event in ("os.listdir", "os.scandir", "glob.glob", "os.walk"):
|
|
69
|
+
p = args[0]
|
|
70
|
+
if p is not None and not isinstance(p, int) and not _under(os.fsdecode(p)):
|
|
71
|
+
_deny("list-outside-allowed-roots", str(p)[:80])
|
|
72
|
+
return
|
|
73
|
+
if event in ("os.putenv", "os.unsetenv"):
|
|
74
|
+
return
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _limits(cfg: dict) -> None:
|
|
78
|
+
try:
|
|
79
|
+
import resource
|
|
80
|
+
except ImportError: # Windows: only the audit hook + parent timeout apply
|
|
81
|
+
return
|
|
82
|
+
cpu = int(cfg.get("cpu_seconds", 30))
|
|
83
|
+
mem = int(cfg.get("memory_mb", 1024)) * 1024 * 1024
|
|
84
|
+
for res, val in ((resource.RLIMIT_CPU, (cpu, cpu + 1)), (resource.RLIMIT_AS, (mem, mem)),
|
|
85
|
+
(resource.RLIMIT_NOFILE, (128, 128)), (resource.RLIMIT_FSIZE, (0, 0)), (resource.RLIMIT_CORE, (0, 0))):
|
|
86
|
+
try:
|
|
87
|
+
resource.setrlimit(res, val)
|
|
88
|
+
except (ValueError, OSError):
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _serialize_candidate(c) -> dict:
|
|
93
|
+
from jscope.core.evidence import Candidate
|
|
94
|
+
if not isinstance(c, Candidate):
|
|
95
|
+
return {"_rejected": type(c).__name__}
|
|
96
|
+
return {
|
|
97
|
+
"category": str(c.category)[:200], "subject": str(c.subject)[:1000], "analyzer": str(c.analyzer)[:200],
|
|
98
|
+
"attack_surface": c.attack_surface if c.attack_surface is None else str(c.attack_surface)[:100],
|
|
99
|
+
"suggested_test": c.suggested_test if c.suggested_test is None else str(c.suggested_test)[:2000],
|
|
100
|
+
"related": [str(x)[:500] for x in (c.related or [])][:50],
|
|
101
|
+
"metadata": _json_safe(c.metadata),
|
|
102
|
+
"evidence": [{"type": getattr(e.type, "value", str(e.type)), "description": str(e.description)[:2000], "analyzer": str(e.analyzer)[:200],
|
|
103
|
+
"file": None if e.file is None else str(e.file)[:1000], "line": e.line if isinstance(e.line, int) else None,
|
|
104
|
+
"snippet": None if e.snippet is None else str(e.snippet)[:2000], "metadata": _json_safe(e.metadata)}
|
|
105
|
+
for e in (c.evidence or [])][:200],
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _json_safe(obj, depth=0):
|
|
110
|
+
if depth > 6:
|
|
111
|
+
return None
|
|
112
|
+
if isinstance(obj, (str, int, float, bool)) or obj is None:
|
|
113
|
+
return obj
|
|
114
|
+
if isinstance(obj, dict):
|
|
115
|
+
return {str(k)[:100]: _json_safe(v, depth + 1) for k, v in list(obj.items())[:200]}
|
|
116
|
+
if isinstance(obj, (list, tuple, set)):
|
|
117
|
+
return [_json_safe(v, depth + 1) for v in list(obj)[:200]]
|
|
118
|
+
return str(obj)[:200]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _find_analyzers(plugin_path: str):
|
|
122
|
+
import importlib.util
|
|
123
|
+
import inspect
|
|
124
|
+
from jscope.analyzers.base import Analyzer
|
|
125
|
+
spec = importlib.util.spec_from_file_location("jscope_sandboxed_plugin", plugin_path)
|
|
126
|
+
mod = importlib.util.module_from_spec(spec)
|
|
127
|
+
spec.loader.exec_module(mod)
|
|
128
|
+
items = list(getattr(mod, "ANALYZERS", []) or [])
|
|
129
|
+
if not items and callable(getattr(mod, "register", None)):
|
|
130
|
+
items = list(mod.register() or [])
|
|
131
|
+
out = []
|
|
132
|
+
for it in items:
|
|
133
|
+
inst = it() if inspect.isclass(it) else it
|
|
134
|
+
if isinstance(inst, Analyzer):
|
|
135
|
+
out.append(inst)
|
|
136
|
+
return out, bool(items)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def main() -> int:
|
|
140
|
+
req = json.loads(sys.stdin.read())
|
|
141
|
+
sys.path.insert(0, req["jscope_root"])
|
|
142
|
+
# Import everything the plugin API needs BEFORE the hook goes in; the hook then only polices what the plugin does.
|
|
143
|
+
import jscope.analyzers.base # noqa: F401
|
|
144
|
+
import jscope.core.evidence # noqa: F401
|
|
145
|
+
import jscope.parsing.ast_engine # noqa: F401
|
|
146
|
+
import jscope.parsing.queries # noqa: F401
|
|
147
|
+
import jscope.core.artifacts # noqa: F401
|
|
148
|
+
import jscope.core.graph # noqa: F401
|
|
149
|
+
import jscope.parsing.html # noqa: F401
|
|
150
|
+
import importlib.util, inspect, glob, tempfile, shutil, re, collections, itertools, functools, dataclasses, typing, enum # noqa: F401,E401
|
|
151
|
+
|
|
152
|
+
roots = {os.path.realpath(p) for p in (sys.prefix, sys.base_prefix, sys.exec_prefix, req["jscope_root"], "/dev/null", "/dev/urandom")}
|
|
153
|
+
roots |= {os.path.realpath(p) for p in sys.path if p and os.path.exists(p)}
|
|
154
|
+
roots |= {os.path.realpath(req["plugin_path"]), os.path.realpath(os.path.dirname(req["plugin_path"]))}
|
|
155
|
+
roots |= {os.path.realpath(p) for p in req.get("grant_read", [])}
|
|
156
|
+
_READ_ROOTS.extend(sorted(roots))
|
|
157
|
+
sys.stdout = sys.stderr # plugin prints must not corrupt the result channel
|
|
158
|
+
_limits(req.get("limits", {}))
|
|
159
|
+
sys.addaudithook(_hook)
|
|
160
|
+
|
|
161
|
+
result: dict = {"ok": True}
|
|
162
|
+
try:
|
|
163
|
+
analyzers, declared = _find_analyzers(req["plugin_path"])
|
|
164
|
+
if req["op"] == "discover":
|
|
165
|
+
result["declared"] = declared
|
|
166
|
+
result["analyzers"] = [{"name": a.name, "version": getattr(a, "version", "0.1.0"), "class": type(a).__name__,
|
|
167
|
+
"depends_on": list(a.depends_on), "run_after": list(getattr(a, "run_after", [])),
|
|
168
|
+
"produces_categories": list(a.produces_categories)} for a in analyzers]
|
|
169
|
+
else:
|
|
170
|
+
target = next((a for a in analyzers if a.name == req["analyzer"]), None)
|
|
171
|
+
if target is None:
|
|
172
|
+
raise RuntimeError(f"analyzer {req['analyzer']!r} not found in plugin")
|
|
173
|
+
from jscope.analyzers.base import ProjectContext
|
|
174
|
+
from jscope.core.artifacts import Artifact
|
|
175
|
+
from jscope.parsing.ast_engine import ASTEngine
|
|
176
|
+
ast = ASTEngine(max_bytes=req.get("max_file_bytes", 5 * 1024 * 1024))
|
|
177
|
+
files = [ast.parse_bytes(base64.b64decode(f["data"]), path=f["path"]) for f in req["files"]]
|
|
178
|
+
for pf, f in zip(files, req["files"]):
|
|
179
|
+
pf.origin = f.get("origin", "disk")
|
|
180
|
+
arts = [Artifact(kind=a["kind"], path=a["path"], data=base64.b64decode(a["data"]), content_hash=a["content_hash"],
|
|
181
|
+
origin=a.get("origin", "disk")) for a in req.get("artifacts", [])]
|
|
182
|
+
ctx = ProjectContext(files=files, ast=ast, framework_hints=req.get("framework_hints", {}), config=req.get("config", {}),
|
|
183
|
+
artifacts=arts, shared=req.get("shared", {}))
|
|
184
|
+
out = target.analyze_project(ctx)
|
|
185
|
+
if not isinstance(out, list):
|
|
186
|
+
result["returned"] = type(out).__name__
|
|
187
|
+
out = []
|
|
188
|
+
result["candidates"] = [_serialize_candidate(c) for c in out[: req.get("max_candidates", 5000)]]
|
|
189
|
+
result["diagnostics"] = [str(d)[:500] for d in ctx.diagnostics][:100]
|
|
190
|
+
except BaseException as exc: # noqa: BLE001 - includes MemoryError, PermissionError
|
|
191
|
+
result = {"ok": False, "error": f"{type(exc).__name__}: {str(exc)[:500]}"}
|
|
192
|
+
result["violations"] = list(_violations)
|
|
193
|
+
_REAL_STDOUT.write("\n@@JSCOPE-RESULT@@" + json.dumps(result) + "\n")
|
|
194
|
+
_REAL_STDOUT.flush()
|
|
195
|
+
return 0
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
if __name__ == "__main__":
|
|
199
|
+
sys.exit(main())
|
jscope/plugin/loader.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""Plugin loader.
|
|
2
|
+
|
|
3
|
+
A plugin is a Python module that exposes either
|
|
4
|
+
ANALYZERS = [MyAnalyzer, ...] (classes or instances), or
|
|
5
|
+
def register() -> list[Analyzer | type]
|
|
6
|
+
Plugins are discovered from (1) directories listed in `plugins.paths` and
|
|
7
|
+
(2) installed packages exposing the `jscope.analyzers` entry-point group.
|
|
8
|
+
|
|
9
|
+
Trust model (see docs/PLUGINS.md):
|
|
10
|
+
* plugins can only PROPOSE evidence; the AnalyzerManager enforces that at runtime
|
|
11
|
+
* every plugin analyzer gets tier "community" unless its name appears in
|
|
12
|
+
`plugins.verified`; core analyzers are tier "core"
|
|
13
|
+
* a finding supported ONLY by community-tier analyzers is capped at
|
|
14
|
+
`plugins.community_confidence_cap` (default MEDIUM)
|
|
15
|
+
|
|
16
|
+
Loading a plugin executes its Python code. Only load plugins you trust --
|
|
17
|
+
the tier system limits what a plugin can do to the *report*, not to your machine.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import importlib.metadata
|
|
23
|
+
import importlib.util
|
|
24
|
+
import inspect
|
|
25
|
+
import sys
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from jscope.analyzers.base import Analyzer
|
|
30
|
+
|
|
31
|
+
PLUGIN_API_VERSION = "0.2"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class LoadResult:
|
|
36
|
+
analyzers: list[Analyzer] = field(default_factory=list)
|
|
37
|
+
errors: list[str] = field(default_factory=list)
|
|
38
|
+
sources: dict[str, str] = field(default_factory=dict) # analyzer name -> where it came from
|
|
39
|
+
warnings: list[str] = field(default_factory=list) # FR-09: plugins running WITHOUT isolation, policy notes
|
|
40
|
+
isolated: set[str] = field(default_factory=set) # analyzer names running in the sandbox
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _coerce(obj, origin: str, res: LoadResult, taken: set[str], verified: set[str]) -> None:
|
|
44
|
+
inst = obj() if inspect.isclass(obj) else obj
|
|
45
|
+
if not isinstance(inst, Analyzer):
|
|
46
|
+
res.errors.append(f"{origin}: {inst!r} is not an Analyzer subclass; skipped")
|
|
47
|
+
return
|
|
48
|
+
if not inst.name or inst.name == "base":
|
|
49
|
+
res.errors.append(f"{origin}: analyzer {type(inst).__name__} must define a unique `name`; skipped")
|
|
50
|
+
return
|
|
51
|
+
if inst.name in taken:
|
|
52
|
+
res.errors.append(f"{origin}: analyzer name '{inst.name}' already registered; skipped")
|
|
53
|
+
return
|
|
54
|
+
inst.tier = "verified" if inst.name in verified else "community"
|
|
55
|
+
taken.add(inst.name)
|
|
56
|
+
res.analyzers.append(inst)
|
|
57
|
+
res.sources[inst.name] = origin
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _from_module(mod, origin: str, res: LoadResult, taken: set[str], verified: set[str]) -> None: # noqa: C901
|
|
61
|
+
items = list(getattr(mod, "ANALYZERS", []) or [])
|
|
62
|
+
if not items and callable(getattr(mod, "register", None)):
|
|
63
|
+
items = list(mod.register() or [])
|
|
64
|
+
if not items:
|
|
65
|
+
res.errors.append(f"{origin}: defines neither ANALYZERS nor register(); skipped")
|
|
66
|
+
return
|
|
67
|
+
for it in items:
|
|
68
|
+
try:
|
|
69
|
+
_coerce(it, origin, res, taken, verified)
|
|
70
|
+
except Exception as exc:
|
|
71
|
+
res.errors.append(f"{origin}: could not instantiate {it!r}: {type(exc).__name__}: {exc}")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _load_isolated(f: Path, res: LoadResult, taken: set[str], verified: set[str], isolation: str, policy) -> bool:
|
|
75
|
+
"""FR-09: discover the plugin INSIDE the sandbox (its code never runs in this process unless it is verified and
|
|
76
|
+
isolation == "auto"). Returns True when the file was handled here, False to fall back to in-process import."""
|
|
77
|
+
from jscope.plugin.sandbox import SandboxedAnalyzer, discover
|
|
78
|
+
metas, err, violations = discover(str(f), policy)
|
|
79
|
+
if err:
|
|
80
|
+
res.errors.append(f"{f}: {err}")
|
|
81
|
+
return True
|
|
82
|
+
if violations:
|
|
83
|
+
res.warnings.append(f"plugin {f.name} attempted forbidden operations while loading (blocked): {'; '.join(violations)}")
|
|
84
|
+
wants_in_process = isolation == "auto" and any(m["name"] in verified for m in metas)
|
|
85
|
+
if wants_in_process:
|
|
86
|
+
return False # operator explicitly trusts this analyzer by name
|
|
87
|
+
for m in metas:
|
|
88
|
+
if not m.get("name") or m["name"] == "base":
|
|
89
|
+
res.errors.append(f"{f}: analyzer {m.get('class')} must define a unique `name`; skipped")
|
|
90
|
+
continue
|
|
91
|
+
if m["name"] in taken:
|
|
92
|
+
res.errors.append(f"{f}: analyzer name '{m['name']}' already registered; skipped")
|
|
93
|
+
continue
|
|
94
|
+
taken.add(m["name"])
|
|
95
|
+
an = SandboxedAnalyzer(m, str(f), policy, tier="verified" if m["name"] in verified else "community")
|
|
96
|
+
res.analyzers.append(an)
|
|
97
|
+
res.sources[an.name] = str(f)
|
|
98
|
+
res.isolated.add(an.name)
|
|
99
|
+
return True
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def load_plugins(paths: list[str], core_names: set[str], verified: set[str] | None = None,
|
|
103
|
+
use_entry_points: bool = True, isolation: str = "none", sandbox_policy=None) -> LoadResult:
|
|
104
|
+
"""isolation: "none" (in-process, full trust), "auto" (community plugins sandboxed, verified-by-name in-process),
|
|
105
|
+
"always" (everything from `paths` sandboxed). Entry-point plugins are installed packages and always run in-process."""
|
|
106
|
+
res = LoadResult()
|
|
107
|
+
taken = set(core_names)
|
|
108
|
+
verified = verified or set()
|
|
109
|
+
if isolation not in ("none", "auto", "always"):
|
|
110
|
+
res.errors.append(f"unknown plugins.isolation {isolation!r}; falling back to 'always'")
|
|
111
|
+
isolation = "always"
|
|
112
|
+
if isolation != "none" and sandbox_policy is None:
|
|
113
|
+
from jscope.plugin.sandbox import SandboxPolicy
|
|
114
|
+
sandbox_policy = SandboxPolicy()
|
|
115
|
+
|
|
116
|
+
for p in paths:
|
|
117
|
+
base = Path(p).expanduser()
|
|
118
|
+
files = [base] if base.is_file() else sorted(base.glob("*.py")) if base.is_dir() else []
|
|
119
|
+
if not files:
|
|
120
|
+
res.errors.append(f"plugin path not found or empty: {p}")
|
|
121
|
+
for f in files:
|
|
122
|
+
if f.name.startswith("_"):
|
|
123
|
+
continue
|
|
124
|
+
if isolation != "none" and _load_isolated(f, res, taken, verified, isolation, sandbox_policy):
|
|
125
|
+
continue
|
|
126
|
+
modname = f"jscope_plugin_{f.stem}"
|
|
127
|
+
try:
|
|
128
|
+
spec = importlib.util.spec_from_file_location(modname, f)
|
|
129
|
+
mod = importlib.util.module_from_spec(spec)
|
|
130
|
+
sys.modules[modname] = mod
|
|
131
|
+
spec.loader.exec_module(mod) # executes plugin code
|
|
132
|
+
except Exception as exc:
|
|
133
|
+
res.errors.append(f"{f}: import failed: {type(exc).__name__}: {exc}")
|
|
134
|
+
continue
|
|
135
|
+
before = len(res.analyzers)
|
|
136
|
+
_from_module(mod, str(f), res, taken, verified)
|
|
137
|
+
for a in res.analyzers[before:]:
|
|
138
|
+
res.warnings.append(f"plugin '{a.name}' ({f.name}) runs WITHOUT isolation"
|
|
139
|
+
+ (" (plugins.isolation=none)" if isolation == "none" else " (verified by name)")
|
|
140
|
+
+ ": it has the same access to this machine as JScope itself")
|
|
141
|
+
|
|
142
|
+
if use_entry_points:
|
|
143
|
+
try:
|
|
144
|
+
eps = importlib.metadata.entry_points(group="jscope.analyzers")
|
|
145
|
+
except TypeError: # very old importlib.metadata
|
|
146
|
+
eps = []
|
|
147
|
+
for ep in eps:
|
|
148
|
+
try:
|
|
149
|
+
obj = ep.load()
|
|
150
|
+
except Exception as exc:
|
|
151
|
+
res.errors.append(f"entry-point {ep.name}: import failed: {type(exc).__name__}: {exc}")
|
|
152
|
+
continue
|
|
153
|
+
before = len(res.analyzers)
|
|
154
|
+
if inspect.ismodule(obj):
|
|
155
|
+
_from_module(obj, f"entry-point:{ep.name}", res, taken, verified)
|
|
156
|
+
else:
|
|
157
|
+
try:
|
|
158
|
+
_coerce(obj, f"entry-point:{ep.name}", res, taken, verified)
|
|
159
|
+
except Exception as exc:
|
|
160
|
+
res.errors.append(f"entry-point {ep.name}: {type(exc).__name__}: {exc}")
|
|
161
|
+
return res
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
TEMPLATE = '''"""{name}: a JScope analyzer plugin.
|
|
165
|
+
|
|
166
|
+
Rules of the road (docs/EVIDENCE_SCHEMA.md, docs/PLUGINS.md):
|
|
167
|
+
* you PROPOSE Candidates backed by typed Evidence; the Evidence Engine decides
|
|
168
|
+
what becomes a finding and at what confidence
|
|
169
|
+
* evidence.analyzer must be your analyzer's name (it is enforced)
|
|
170
|
+
* you cannot emit CROSS_ANALYZER_AGREEMENT -- the engine adds it when a
|
|
171
|
+
DIFFERENT analyzer independently proposes the same (category, subject)
|
|
172
|
+
* prefer AST evidence over bare string matches; always add negative evidence
|
|
173
|
+
(DEAD_CODE, TEST_FIXTURE_CONTEXT) when you can see it
|
|
174
|
+
"""
|
|
175
|
+
|
|
176
|
+
from jscope.analyzers.base import Analyzer, ProjectContext
|
|
177
|
+
from jscope.core.evidence import Candidate, Evidence, EvidenceType
|
|
178
|
+
from jscope.parsing import queries as Q
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class {cls}(Analyzer):
|
|
182
|
+
name = "{name}"
|
|
183
|
+
depends_on = ["framework_detector"]
|
|
184
|
+
produces_categories = ["{name}"]
|
|
185
|
+
|
|
186
|
+
def analyze_project(self, ctx: ProjectContext) -> list[Candidate]:
|
|
187
|
+
out = []
|
|
188
|
+
for pf in ctx.files:
|
|
189
|
+
if pf.skipped_reason or pf.tree is None:
|
|
190
|
+
continue
|
|
191
|
+
for site in Q.iter_calls(ctx.ast, pf):
|
|
192
|
+
if site.callee != "dangerousThing": # <- your detection here
|
|
193
|
+
continue
|
|
194
|
+
evidence = [Evidence(
|
|
195
|
+
type=EvidenceType.AST_PATTERN_MATCH,
|
|
196
|
+
description="call to dangerousThing()",
|
|
197
|
+
analyzer=self.name, file=pf.path, line=site.line,
|
|
198
|
+
snippet=(site.args_raw[0][:80] if site.args_raw else None),
|
|
199
|
+
)]
|
|
200
|
+
if ctx.ast.is_in_dead_branch(pf, site.node):
|
|
201
|
+
evidence.append(Evidence(EvidenceType.DEAD_CODE, "in a dead branch", self.name))
|
|
202
|
+
out.append(Candidate(
|
|
203
|
+
category="{name}", subject=f"dangerousThing@{{pf.path}}:{{site.line}}",
|
|
204
|
+
analyzer=self.name, evidence=evidence,
|
|
205
|
+
suggested_test="What should a human try next?",
|
|
206
|
+
))
|
|
207
|
+
return out
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
ANALYZERS = [{cls}]
|
|
211
|
+
'''
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def scaffold(name: str, out_dir: str) -> Path:
|
|
215
|
+
ident = "".join(ch if ch.isalnum() else "_" for ch in name).strip("_").lower()
|
|
216
|
+
if not ident or ident[0].isdigit():
|
|
217
|
+
raise ValueError("plugin name must start with a letter and contain letters/digits/underscores")
|
|
218
|
+
cls = "".join(p.capitalize() for p in ident.split("_")) + "Analyzer"
|
|
219
|
+
d = Path(out_dir)
|
|
220
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
221
|
+
target = d / f"{ident}.py"
|
|
222
|
+
if target.exists():
|
|
223
|
+
raise FileExistsError(str(target))
|
|
224
|
+
target.write_text(TEMPLATE.format(name=ident, cls=cls))
|
|
225
|
+
return target
|