attackmap 0.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- attackmap/__init__.py +2 -0
- attackmap/analyzer.py +449 -0
- attackmap/analyzer_contracts.py +194 -0
- attackmap/analyzers.py +1003 -0
- attackmap/anomalies.py +612 -0
- attackmap/asset_model.py +245 -0
- attackmap/attack_taxonomy.py +133 -0
- attackmap/auth_protocol.py +702 -0
- attackmap/authz.py +1695 -0
- attackmap/bench.py +467 -0
- attackmap/cli.py +1474 -0
- attackmap/config_scanner.py +389 -0
- attackmap/context_pack.py +82 -0
- attackmap/contracts.py +201 -0
- attackmap/control_model.py +280 -0
- attackmap/crossrepo.py +356 -0
- attackmap/crypto.py +240 -0
- attackmap/cve.py +717 -0
- attackmap/defensive_review.py +647 -0
- attackmap/detection_opportunities.py +1221 -0
- attackmap/detections/__init__.py +491 -0
- attackmap/detections/bola.yml +57 -0
- attackmap/detections/cloud.yml +609 -0
- attackmap/detections/injection.yml +167 -0
- attackmap/detections/secrets.yml +48 -0
- attackmap/detections/sigma.py +511 -0
- attackmap/detections/ssrf.yml +98 -0
- attackmap/detections/weakness.yml +62 -0
- attackmap/detections/webhook.yml +45 -0
- attackmap/detections/workflow.yml +121 -0
- attackmap/diagrams.py +237 -0
- attackmap/diff.py +301 -0
- attackmap/exploitability.py +493 -0
- attackmap/fingerprint.py +91 -0
- attackmap/fleet.py +370 -0
- attackmap/gitref.py +129 -0
- attackmap/graph.py +26 -0
- attackmap/handler_auth.py +639 -0
- attackmap/handler_span.py +62 -0
- attackmap/hunt_harness.py +505 -0
- attackmap/insights.py +701 -0
- attackmap/llm_review.py +846 -0
- attackmap/lockfiles.py +521 -0
- attackmap/mass_assignment.py +543 -0
- attackmap/md.py +175 -0
- attackmap/merge.py +333 -0
- attackmap/models.py +1284 -0
- attackmap/plugins_lock.py +114 -0
- attackmap/popular_packages.py +193 -0
- attackmap/progress.py +258 -0
- attackmap/recon_models.py +31 -0
- attackmap/recon_to_analysis.py +149 -0
- attackmap/redact.py +220 -0
- attackmap/report.py +424 -0
- attackmap/review_eval.py +335 -0
- attackmap/review_json.py +308 -0
- attackmap/review_prompts.py +796 -0
- attackmap/route_auth_fusion.py +474 -0
- attackmap/safe_fs.py +285 -0
- attackmap/sarif.py +373 -0
- attackmap/sbom.py +418 -0
- attackmap/scanner.py +1379 -0
- attackmap/sdk/__init__.py +278 -0
- attackmap/sdk/contracts.py +17 -0
- attackmap/sdk/fs.py +156 -0
- attackmap/sdk/models.py +59 -0
- attackmap/secrets_history.py +161 -0
- attackmap/security_overlay.py +71 -0
- attackmap/srcpaths.py +301 -0
- attackmap/suggest.py +394 -0
- attackmap/supply_chain.py +688 -0
- attackmap/suppress.py +740 -0
- attackmap/taint.py +1540 -0
- attackmap/taint_flow.py +1419 -0
- attackmap/taint_sinks.py +431 -0
- attackmap/taint_sinks.yaml +308 -0
- attackmap/taint_sources.py +396 -0
- attackmap/taxonomy.py +465 -0
- attackmap/threat_model.py +3441 -0
- attackmap/topology.py +316 -0
- attackmap/triage.py +151 -0
- attackmap/weaknesses.py +365 -0
- attackmap/webhardening.py +479 -0
- attackmap/webhook_verification.py +1067 -0
- attackmap/workflow_scanner.py +964 -0
- attackmap-0.6.4.dist-info/METADATA +226 -0
- attackmap-0.6.4.dist-info/RECORD +91 -0
- attackmap-0.6.4.dist-info/WHEEL +5 -0
- attackmap-0.6.4.dist-info/entry_points.txt +2 -0
- attackmap-0.6.4.dist-info/licenses/LICENSE +21 -0
- attackmap-0.6.4.dist-info/top_level.txt +1 -0
attackmap/taint_flow.py
ADDED
|
@@ -0,0 +1,1419 @@
|
|
|
1
|
+
"""Intra-procedural def-use taint propagation (#239).
|
|
2
|
+
|
|
3
|
+
The import-graph walk in :mod:`attackmap.taint` decides *which* sinks a route
|
|
4
|
+
can reach. This module decides, for one sink call, whether a request-derived
|
|
5
|
+
value actually flows into the dangerous argument **inside the enclosing
|
|
6
|
+
function**, so the overwhelmingly common two-line shape is caught::
|
|
7
|
+
|
|
8
|
+
target = request.args.get("url") # source
|
|
9
|
+
r = requests.get(target, timeout=3) # sink — fires, source_line = 1st line
|
|
10
|
+
|
|
11
|
+
How it works:
|
|
12
|
+
|
|
13
|
+
- **Enclosing function.** Python via the stdlib ``ast``; JS/TS, Go, PHP and
|
|
14
|
+
Java via a single linear brace-balance pass over the file with string
|
|
15
|
+
literals and comments blanked out (interpolations such as ``${x}`` and PHP
|
|
16
|
+
``"$x"`` are kept, because they carry data).
|
|
17
|
+
- **Def-use map.** Simple assignments in source order — ``x = <expr>``,
|
|
18
|
+
tuple targets, JS destructuring (``const {u} = req.query``), Go ``:=`` and
|
|
19
|
+
``&out`` decode targets, PHP ``$x = …`` / ``.=``, Java ``Type x = …`` — plus
|
|
20
|
+
container mutators (``parts.append(x)``). A variable is tainted when its
|
|
21
|
+
right-hand side matches the per-language source catalog
|
|
22
|
+
(:mod:`attackmap.taint_sources`) or references a tainted variable, so taint
|
|
23
|
+
follows f-strings, ``+``, ``.format``, ``%``, ``path.join`` and any other
|
|
24
|
+
expression to the sink. Framework-bound handler parameters (Flask/FastAPI
|
|
25
|
+
route params, Spring ``@RequestParam``, NestJS ``@Query()``) are sources.
|
|
26
|
+
- **Argument-aware sinks.** Only the dangerous argument counts: the URL of an
|
|
27
|
+
HTTP call (not its JSON body), the query of an ``execute`` (not its bound
|
|
28
|
+
params). A URL whose literal prefix pins scheme and host
|
|
29
|
+
(``f"https://api.example.com/{id}"``) is not SSRF.
|
|
30
|
+
- **Flow-bound sanitizers.** A sanitizer counts only when the tainted value
|
|
31
|
+
passes through it on the way to the sink (``u = secure_filename(u)``), and a
|
|
32
|
+
guard only when it checks a value derived from the same source and dominates
|
|
33
|
+
the sink — an early exit before it (``if ip.is_private: abort(400)``) or an
|
|
34
|
+
``if`` that encloses it. Nothing is file-granular any more.
|
|
35
|
+
|
|
36
|
+
Documented limits (precision-first, heuristic — evidence, not proof):
|
|
37
|
+
|
|
38
|
+
- **Intra-procedural only.** Taint does not cross a call boundary: a value
|
|
39
|
+
passed to a helper that runs the sink is not followed into the helper (the
|
|
40
|
+
helper's own parameters are treated as *unknown*, never as request input).
|
|
41
|
+
Cross-file reach is still the file-granular import walk in ``taint.py``;
|
|
42
|
+
there is no function-level call graph. Cross-repo taint is #196.
|
|
43
|
+
- **Flow-insensitive within a function.** Assignments are replayed in source
|
|
44
|
+
order up to the sink; branches are not distinguished, loops are a single
|
|
45
|
+
pass, and a guard's polarity (``if ok`` vs ``if not ok``) is not checked —
|
|
46
|
+
an early-exit or enclosing ``if`` on a derived value counts as a guard.
|
|
47
|
+
- **Only simple names propagate.** Attribute stores (``self.x = …``,
|
|
48
|
+
``obj.prop = …``), aliasing through containers other than the listed
|
|
49
|
+
mutators, closures, generators and ``global`` state are not tracked.
|
|
50
|
+
- **Regex languages are approximate.** JS/Go/PHP/Java use a brace-balance pass,
|
|
51
|
+
not a parser: heredocs, regex literals containing
|
|
52
|
+
braces, and statements split across lines without a trailing operator can
|
|
53
|
+
be mis-scoped. A JS concise arrow body (``x => expr``) is scoped as its own
|
|
54
|
+
function up to the expression's end (#398). When the enclosing function can't be found the caller falls
|
|
55
|
+
back to the legacy same-call request-token gate.
|
|
56
|
+
- **Raising validators are not guards.** A bare ``validate_url(u)`` statement
|
|
57
|
+
that raises on bad input is not recognized; only ``if``/``assert`` checks.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
from __future__ import annotations
|
|
61
|
+
|
|
62
|
+
import ast
|
|
63
|
+
import bisect
|
|
64
|
+
import re
|
|
65
|
+
from dataclasses import dataclass, field
|
|
66
|
+
|
|
67
|
+
from .srcpaths import line_number
|
|
68
|
+
from .taint_sources import (
|
|
69
|
+
ALL_SANITIZER_KINDS,
|
|
70
|
+
CAST_SANITIZERS,
|
|
71
|
+
CSHARP,
|
|
72
|
+
EARLY_EXIT,
|
|
73
|
+
GO,
|
|
74
|
+
JAVA,
|
|
75
|
+
JS,
|
|
76
|
+
PARAM_ANNOTATIONS,
|
|
77
|
+
PHP,
|
|
78
|
+
PY,
|
|
79
|
+
PY_PLUMBING_PARAMS,
|
|
80
|
+
PY_ROUTE_DECORATOR_NAMES,
|
|
81
|
+
PY_SAFE_ANNOTATION,
|
|
82
|
+
PY_SAFE_DEFAULT,
|
|
83
|
+
PY_SCALAR_ANNOTATION,
|
|
84
|
+
SANITIZERS,
|
|
85
|
+
SOURCES,
|
|
86
|
+
guard_patterns,
|
|
87
|
+
lang_for_suffix,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
_MAX_STEPS = 8 # flow steps kept per value
|
|
91
|
+
_MAX_ATOMS = 6 # distinct sources tracked per variable
|
|
92
|
+
_MAX_RHS = 1500 # chars scanned for one right-hand side
|
|
93
|
+
_MAX_ARGS = 4000 # chars scanned for one sink call's arguments
|
|
94
|
+
_MAX_HEADER = 600 # chars scanned back for a block's header
|
|
95
|
+
_MAX_EVAL_DEPTH = 60 # Python expression recursion bound
|
|
96
|
+
|
|
97
|
+
Step = tuple[int, str, str] # (line, kind, note); kind: source|propagation|sanitizer|guard|sink
|
|
98
|
+
|
|
99
|
+
# `args_spec` sentinels: the call's receiver (`url.openStream()`), and the rest
|
|
100
|
+
# of the statement after the match (keyword / assignment sinks such as PHP
|
|
101
|
+
# `include $f;` or `response.headers["X"] = v`).
|
|
102
|
+
RECEIVER: tuple[int, ...] = (-1,)
|
|
103
|
+
REST: tuple[int, ...] = (-2,)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass(frozen=True)
|
|
107
|
+
class Atom:
|
|
108
|
+
"""One source-derived value: where it came from, the steps it took, and
|
|
109
|
+
the sink kinds a sanitizer/guard on its path has neutralized."""
|
|
110
|
+
|
|
111
|
+
kind: str
|
|
112
|
+
line: int
|
|
113
|
+
weight: str # "high" (request input) | "low" (env/argv) | "param" (unknown helper arg)
|
|
114
|
+
label: str
|
|
115
|
+
sanitized: tuple[tuple[str, str], ...] = () # (sink_kind, sanitizer label)
|
|
116
|
+
steps: tuple[Step, ...] = ()
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def ident(self) -> tuple[str, int, str]:
|
|
120
|
+
return (self.kind, self.line, self.label)
|
|
121
|
+
|
|
122
|
+
def sanitizer_for(self, kind: str) -> str | None:
|
|
123
|
+
for k, label in self.sanitized:
|
|
124
|
+
if k == kind:
|
|
125
|
+
return label
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
def with_step(self, line: int, kind: str, note: str) -> "Atom":
|
|
129
|
+
steps = self.steps
|
|
130
|
+
if len(steps) < _MAX_STEPS and (not steps or steps[-1][:2] != (line, kind)):
|
|
131
|
+
steps = steps + ((line, kind, note[:120]),)
|
|
132
|
+
return Atom(self.kind, self.line, self.weight, self.label, self.sanitized, steps)
|
|
133
|
+
|
|
134
|
+
def with_sanitizer(self, kinds: frozenset[str] | tuple[str, ...], label: str, line: int, step_kind: str = "sanitizer") -> "Atom":
|
|
135
|
+
have = {k for k, _ in self.sanitized}
|
|
136
|
+
add = tuple((k, label) for k in kinds if k not in have)
|
|
137
|
+
if not add:
|
|
138
|
+
return self
|
|
139
|
+
a = Atom(self.kind, self.line, self.weight, self.label, self.sanitized + add, self.steps)
|
|
140
|
+
return a.with_step(line, step_kind, label)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _dedupe(atoms: list[Atom]) -> list[Atom]:
|
|
144
|
+
seen: set[tuple] = set()
|
|
145
|
+
out: list[Atom] = []
|
|
146
|
+
for a in atoms:
|
|
147
|
+
key = (a.ident, a.sanitized)
|
|
148
|
+
if key in seen:
|
|
149
|
+
continue
|
|
150
|
+
seen.add(key)
|
|
151
|
+
out.append(a)
|
|
152
|
+
if len(out) >= _MAX_ATOMS:
|
|
153
|
+
break
|
|
154
|
+
return out
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@dataclass
|
|
158
|
+
class SinkFlow:
|
|
159
|
+
"""Flow verdict for one sink call."""
|
|
160
|
+
|
|
161
|
+
tainted: bool = False # a request (high-weight) value reaches the dangerous arg unsanitized
|
|
162
|
+
source_kind: str | None = None
|
|
163
|
+
source_line: int | None = None
|
|
164
|
+
sanitized: bool = False # every source-derived value reaching the arg passed a sanitizer/guard
|
|
165
|
+
sanitizer: str | None = None
|
|
166
|
+
steps: list[Step] = field(default_factory=list)
|
|
167
|
+
span: tuple[int, int] | None = None # enclosing function (start_line, end_line), decorators included
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _verdict(kind: str, atoms: list[Atom], sink_line: int, sink_note: str, span: tuple[int, int]) -> SinkFlow:
|
|
171
|
+
flow = SinkFlow(span=span)
|
|
172
|
+
live = [a for a in atoms if a.weight == "high" and a.sanitizer_for(kind) is None]
|
|
173
|
+
if live:
|
|
174
|
+
top = min(live, key=lambda a: a.line)
|
|
175
|
+
flow.tainted = True
|
|
176
|
+
flow.source_kind, flow.source_line = top.kind, top.line
|
|
177
|
+
flow.steps = sorted(top.steps, key=lambda st: st[0]) + [(sink_line, "sink", sink_note[:120])]
|
|
178
|
+
return flow
|
|
179
|
+
neutral = [a for a in atoms if a.sanitizer_for(kind) is not None]
|
|
180
|
+
unknown = [a for a in atoms if a.sanitizer_for(kind) is None and a.weight != "low"]
|
|
181
|
+
if neutral and not unknown:
|
|
182
|
+
top = min(neutral, key=lambda a: (a.weight != "high", a.line))
|
|
183
|
+
flow.sanitized = True
|
|
184
|
+
flow.sanitizer = top.sanitizer_for(kind)
|
|
185
|
+
if top.weight == "high":
|
|
186
|
+
flow.source_kind, flow.source_line = top.kind, top.line
|
|
187
|
+
flow.steps = sorted(top.steps, key=lambda st: st[0]) + [(sink_line, "sink", sink_note[:120])]
|
|
188
|
+
return flow
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
# --- shared literal-prefix checks --------------------------------------------
|
|
192
|
+
|
|
193
|
+
# A URL literal that pins scheme + host before any interpolation: tainted data
|
|
194
|
+
# after it only controls the path/query, which is not SSRF.
|
|
195
|
+
# The leading lookbehind pins match starts to a scheme boundary (linear, #236).
|
|
196
|
+
_FIXED_HOST_RE = re.compile(r"(?<![\w+.-])[A-Za-z][\w+.-]*://[^/\s?#{}$`'\"%]+/")
|
|
197
|
+
# The same for a redirect target (#327). The literal must close the authority
|
|
198
|
+
# with "/", "?" or "#", so "https://site.example${x}" (→ ".evil.com", "@evil")
|
|
199
|
+
# stays tainted. Script schemes run the rest of the URL, so they never pin it.
|
|
200
|
+
_FIXED_REDIRECT_HOST_RE = re.compile(
|
|
201
|
+
r"(?<![\w+.-])(?!(?i:javascript|vbscript|data):)[A-Za-z][\w+.-]*://[^/\\\s?#{}$`'\"%]+[/?#]"
|
|
202
|
+
)
|
|
203
|
+
# A redirect literal that pins a same-site path ("/orders/…", not "//evil").
|
|
204
|
+
_FIXED_PATH_RE = re.compile(r"/[^/\\\s{}$`'\"%]")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _fixed_prefix(kind: str, literal: str | None) -> bool:
|
|
208
|
+
if not literal:
|
|
209
|
+
return False
|
|
210
|
+
if kind == "ssrf":
|
|
211
|
+
return _FIXED_HOST_RE.match(literal) is not None
|
|
212
|
+
if kind == "open_redirect":
|
|
213
|
+
return _FIXED_PATH_RE.match(literal) is not None or _FIXED_REDIRECT_HOST_RE.match(literal) is not None
|
|
214
|
+
return False
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
_INTERNAL_URL_FUNCS = frozenset({"reverse", "reverse_lazy", "url_for", "resolve_url"})
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _internal_url_node(node: ast.AST) -> bool:
|
|
221
|
+
"""A redirect target built from `reverse("view")` / `url_for("view", …)`
|
|
222
|
+
(possibly followed by `+ "?q=" + x` or inside an f-string's first part):
|
|
223
|
+
the view name pins an internal path, so request data in its kwargs or the
|
|
224
|
+
query can't send the user off-site (#375)."""
|
|
225
|
+
for _ in range(8):
|
|
226
|
+
if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.Add, ast.Mod)):
|
|
227
|
+
node = node.left
|
|
228
|
+
elif isinstance(node, ast.JoinedStr) and node.values and isinstance(node.values[0], ast.FormattedValue):
|
|
229
|
+
node = node.values[0].value
|
|
230
|
+
else:
|
|
231
|
+
break
|
|
232
|
+
if not isinstance(node, ast.Call) or not node.args:
|
|
233
|
+
return False
|
|
234
|
+
func = node.func
|
|
235
|
+
name = func.id if isinstance(func, ast.Name) else func.attr if isinstance(func, ast.Attribute) else None
|
|
236
|
+
if name not in _INTERNAL_URL_FUNCS:
|
|
237
|
+
return False
|
|
238
|
+
first = node.args[0]
|
|
239
|
+
return (isinstance(first, ast.Constant) and isinstance(first.value, str)) or isinstance(first, (ast.Name, ast.Attribute))
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _sanitizer_table() -> list[tuple[frozenset[str], str, re.Pattern[str]]]:
|
|
243
|
+
table: list[tuple[frozenset[str], str, re.Pattern[str]]] = []
|
|
244
|
+
for kind, entries in SANITIZERS.items():
|
|
245
|
+
for label, pat in entries:
|
|
246
|
+
table.append((frozenset({kind}), label, pat))
|
|
247
|
+
every = frozenset(ALL_SANITIZER_KINDS)
|
|
248
|
+
for label, pat in CAST_SANITIZERS:
|
|
249
|
+
table.append((every, label, pat))
|
|
250
|
+
return table
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
_SANITIZER_TABLE = _sanitizer_table()
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
# =============================================================================
|
|
257
|
+
# Python (ast)
|
|
258
|
+
# =============================================================================
|
|
259
|
+
|
|
260
|
+
_PY_MUTATORS = frozenset({"append", "extend", "insert", "add", "update", "appendleft", "write", "setdefault"})
|
|
261
|
+
_PY_ARG_KEYWORDS = frozenset(
|
|
262
|
+
{
|
|
263
|
+
"url", "sql", "query", "statement", "stmt", "operation", "cmd", "command", "args",
|
|
264
|
+
"path", "file", "filename", "source", "template", "location", "string", "expression", "filter",
|
|
265
|
+
}
|
|
266
|
+
)
|
|
267
|
+
_PY_EXIT_CALLS = frozenset({"abort", "exit", "quit", "_exit", "fail", "deny"})
|
|
268
|
+
_FLASK_PATH_PARAM_RE = re.compile(r"<(?:(\w+):)?(\w+)>")
|
|
269
|
+
_BRACE_PATH_PARAM_RE = re.compile(r"\{(\w+)(?::[^}]*)?\}")
|
|
270
|
+
_SAFE_CONVERTERS = frozenset({"int", "float", "uuid"})
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
class _PyFlow:
|
|
274
|
+
def __init__(self, content: str) -> None:
|
|
275
|
+
self.content = content
|
|
276
|
+
self.tree = ast.parse(content)
|
|
277
|
+
self.blines = content.encode("utf-8", "surrogatepass").split(b"\n")
|
|
278
|
+
self.funcs: list[tuple[int, int, ast.AST]] = []
|
|
279
|
+
for node in ast.walk(self.tree):
|
|
280
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
281
|
+
start = min([d.lineno for d in node.decorator_list] + [node.lineno])
|
|
282
|
+
self.funcs.append((start, node.end_lineno or node.lineno, node))
|
|
283
|
+
self._fn: dict[int, tuple[list, dict]] = {}
|
|
284
|
+
# Functions by start line (walk order breaks ties), each with the index
|
|
285
|
+
# of the function that encloses it, so `_enclosing` climbs the nesting
|
|
286
|
+
# instead of scanning every function per sink (quadratic on a file
|
|
287
|
+
# with thousands of handlers).
|
|
288
|
+
order = sorted(range(len(self.funcs)), key=lambda i: (self.funcs[i][0], i))
|
|
289
|
+
self._by_start = [self.funcs[i] for i in order]
|
|
290
|
+
self._starts = [start for start, _, _ in self._by_start]
|
|
291
|
+
self._parent: list[int] = []
|
|
292
|
+
stack: list[int] = []
|
|
293
|
+
for index, (start, end, _) in enumerate(self._by_start):
|
|
294
|
+
while stack and self._by_start[stack[-1]][1] < start:
|
|
295
|
+
stack.pop()
|
|
296
|
+
self._parent.append(stack[-1] if stack else -1)
|
|
297
|
+
stack.append(index)
|
|
298
|
+
|
|
299
|
+
# -- text helpers --
|
|
300
|
+
def seg(self, node: ast.AST, cap: int = 2000) -> str:
|
|
301
|
+
l1, c1 = node.lineno, node.col_offset # type: ignore[attr-defined]
|
|
302
|
+
l2, c2 = node.end_lineno or l1, node.end_col_offset or c1 # type: ignore[attr-defined]
|
|
303
|
+
try:
|
|
304
|
+
if l1 == l2:
|
|
305
|
+
b = self.blines[l1 - 1][c1:c2]
|
|
306
|
+
else:
|
|
307
|
+
if l2 - l1 > 40:
|
|
308
|
+
l2, c2 = l1 + 40, len(self.blines[l1 + 39])
|
|
309
|
+
parts = [self.blines[l1 - 1][c1:]] + self.blines[l1 : l2 - 1] + [self.blines[l2 - 1][:c2]]
|
|
310
|
+
b = b"\n".join(parts)
|
|
311
|
+
except IndexError:
|
|
312
|
+
return ""
|
|
313
|
+
return b[:cap].decode("utf-8", "replace")
|
|
314
|
+
|
|
315
|
+
def _enclosing(self, line: int) -> ast.AST | None:
|
|
316
|
+
"""The innermost function containing ``line`` (the first in walk
|
|
317
|
+
order among functions starting on the same line)."""
|
|
318
|
+
index = bisect.bisect_right(self._starts, line) - 1
|
|
319
|
+
while index >= 0:
|
|
320
|
+
start, end, _ = self._by_start[index]
|
|
321
|
+
if line <= end:
|
|
322
|
+
while index > 0 and self._starts[index - 1] == start and self._by_start[index - 1][1] >= line:
|
|
323
|
+
index -= 1
|
|
324
|
+
return self._by_start[index][2]
|
|
325
|
+
# Closed before `line`: only a function enclosing it can contain it.
|
|
326
|
+
index = self._parent[index]
|
|
327
|
+
return None
|
|
328
|
+
|
|
329
|
+
# -- sources --
|
|
330
|
+
def _source(self, text: str) -> tuple[str, str] | None:
|
|
331
|
+
for kind, pat, weight in SOURCES[PY]:
|
|
332
|
+
if pat.match(text):
|
|
333
|
+
return kind, weight
|
|
334
|
+
return None
|
|
335
|
+
|
|
336
|
+
def _eval(self, node: ast.AST, env: dict[str, list[Atom]], depth: int = 0) -> list[Atom]:
|
|
337
|
+
if depth > _MAX_EVAL_DEPTH or isinstance(node, (ast.Constant, ast.Lambda)):
|
|
338
|
+
return []
|
|
339
|
+
if isinstance(node, (ast.Attribute, ast.Subscript, ast.Call, ast.Await)):
|
|
340
|
+
text = self.seg(node, cap=300)
|
|
341
|
+
hit = self._source(text)
|
|
342
|
+
if hit is not None:
|
|
343
|
+
kind, weight = hit
|
|
344
|
+
label = text.split("\n", 1)[0][:80]
|
|
345
|
+
return [Atom(kind, node.lineno, weight, label, (), ((node.lineno, "source", f"{kind}: {label}"),))] # type: ignore[attr-defined]
|
|
346
|
+
if isinstance(node, ast.Name):
|
|
347
|
+
return list(env.get(node.id, ()))
|
|
348
|
+
if isinstance(node, ast.Call):
|
|
349
|
+
inner: list[Atom] = []
|
|
350
|
+
if isinstance(node.func, ast.Attribute):
|
|
351
|
+
inner += self._eval(node.func.value, env, depth + 1)
|
|
352
|
+
for a in node.args:
|
|
353
|
+
inner += self._eval(a, env, depth + 1)
|
|
354
|
+
for kw in node.keywords:
|
|
355
|
+
inner += self._eval(kw.value, env, depth + 1)
|
|
356
|
+
ftxt = self.seg(node.func, cap=200) + "("
|
|
357
|
+
for kinds, label, pat in _SANITIZER_TABLE:
|
|
358
|
+
if any(m.end() == len(ftxt) for m in pat.finditer(ftxt)):
|
|
359
|
+
inner = [a.with_sanitizer(kinds, label, node.lineno) for a in inner]
|
|
360
|
+
return _dedupe(inner)
|
|
361
|
+
if isinstance(node, ast.Subscript):
|
|
362
|
+
# `MAP[user]` is a lookup keyed by the value, not the value itself.
|
|
363
|
+
return self._eval(node.value, env, depth + 1)
|
|
364
|
+
if isinstance(node, ast.Attribute):
|
|
365
|
+
return self._eval(node.value, env, depth + 1)
|
|
366
|
+
out: list[Atom] = []
|
|
367
|
+
for child in ast.iter_child_nodes(node):
|
|
368
|
+
if isinstance(child, ast.expr):
|
|
369
|
+
out += self._eval(child, env, depth + 1)
|
|
370
|
+
elif isinstance(child, ast.comprehension):
|
|
371
|
+
out += self._eval(child.iter, env, depth + 1)
|
|
372
|
+
return _dedupe(out)
|
|
373
|
+
|
|
374
|
+
# -- function setup --
|
|
375
|
+
def _handler_params(self, fn: ast.AST) -> dict[str, list[Atom]]:
|
|
376
|
+
env: dict[str, list[Atom]] = {}
|
|
377
|
+
route_path = None
|
|
378
|
+
is_handler = False
|
|
379
|
+
for dec in fn.decorator_list: # type: ignore[attr-defined]
|
|
380
|
+
if not isinstance(dec, ast.Call):
|
|
381
|
+
continue
|
|
382
|
+
f = dec.func
|
|
383
|
+
name = f.attr if isinstance(f, ast.Attribute) else f.id if isinstance(f, ast.Name) else None
|
|
384
|
+
if name in PY_ROUTE_DECORATOR_NAMES:
|
|
385
|
+
is_handler = True
|
|
386
|
+
if dec.args and isinstance(dec.args[0], ast.Constant) and isinstance(dec.args[0].value, str):
|
|
387
|
+
route_path = dec.args[0].value
|
|
388
|
+
path_params: dict[str, str | None] = {}
|
|
389
|
+
if route_path:
|
|
390
|
+
for conv, pname in _FLASK_PATH_PARAM_RE.findall(route_path):
|
|
391
|
+
path_params[pname] = conv or None
|
|
392
|
+
for pname in _BRACE_PATH_PARAM_RE.findall(route_path):
|
|
393
|
+
path_params.setdefault(pname, None)
|
|
394
|
+
args = fn.args # type: ignore[attr-defined]
|
|
395
|
+
params = list(args.posonlyargs) + list(args.args) + list(args.kwonlyargs)
|
|
396
|
+
defaults: dict[str, ast.AST] = {}
|
|
397
|
+
pos = list(args.posonlyargs) + list(args.args)
|
|
398
|
+
for p, d in zip(pos[len(pos) - len(args.defaults):], args.defaults):
|
|
399
|
+
defaults[p.arg] = d
|
|
400
|
+
for p, d in zip(args.kwonlyargs, args.kw_defaults):
|
|
401
|
+
if d is not None:
|
|
402
|
+
defaults[p.arg] = d
|
|
403
|
+
line = fn.lineno # type: ignore[attr-defined]
|
|
404
|
+
for p in params:
|
|
405
|
+
if p.arg in ("self", "cls"):
|
|
406
|
+
continue
|
|
407
|
+
ann = self.seg(p.annotation, cap=200) if p.annotation is not None else ""
|
|
408
|
+
dflt = self.seg(defaults[p.arg], cap=200) if p.arg in defaults else ""
|
|
409
|
+
if not is_handler:
|
|
410
|
+
env[p.arg] = [Atom("param", line, "param", p.arg)]
|
|
411
|
+
continue
|
|
412
|
+
if p.arg in PY_PLUMBING_PARAMS or PY_SAFE_ANNOTATION.search(ann) or PY_SAFE_DEFAULT.search(dflt):
|
|
413
|
+
continue
|
|
414
|
+
if p.arg in path_params:
|
|
415
|
+
if path_params[p.arg] in _SAFE_CONVERTERS:
|
|
416
|
+
continue
|
|
417
|
+
kind = "path_param"
|
|
418
|
+
elif not ann or PY_SCALAR_ANNOTATION.search(ann):
|
|
419
|
+
kind = "query"
|
|
420
|
+
else:
|
|
421
|
+
kind = "body"
|
|
422
|
+
label = f"handler parameter `{p.arg}`"
|
|
423
|
+
env[p.arg] = [Atom(kind, line, "high", label, (), ((line, "source", f"{kind}: {label}"),))]
|
|
424
|
+
return env
|
|
425
|
+
|
|
426
|
+
def _names(self, target: ast.AST) -> tuple[list[str], list[str]]:
|
|
427
|
+
"""(rebound names, names a subscript store merges into)."""
|
|
428
|
+
if isinstance(target, ast.Name):
|
|
429
|
+
return [target.id], []
|
|
430
|
+
if isinstance(target, (ast.Tuple, ast.List)):
|
|
431
|
+
bound: list[str] = []
|
|
432
|
+
merged: list[str] = []
|
|
433
|
+
for elt in target.elts:
|
|
434
|
+
b, m = self._names(elt)
|
|
435
|
+
bound += b
|
|
436
|
+
merged += m
|
|
437
|
+
return bound, merged
|
|
438
|
+
if isinstance(target, ast.Starred):
|
|
439
|
+
return self._names(target.value)
|
|
440
|
+
if isinstance(target, ast.Subscript) and isinstance(target.value, ast.Name):
|
|
441
|
+
return [], [target.value.id]
|
|
442
|
+
return [], []
|
|
443
|
+
|
|
444
|
+
def _stmts(self, body: list[ast.stmt]):
|
|
445
|
+
for st in body:
|
|
446
|
+
if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
447
|
+
continue
|
|
448
|
+
yield st
|
|
449
|
+
for attr in ("body", "orelse", "finalbody"):
|
|
450
|
+
sub = getattr(st, attr, None)
|
|
451
|
+
if isinstance(sub, list) and sub and isinstance(sub[0], ast.stmt):
|
|
452
|
+
yield from self._stmts(sub)
|
|
453
|
+
for h in getattr(st, "handlers", ()) or ():
|
|
454
|
+
yield from self._stmts(h.body)
|
|
455
|
+
for case in getattr(st, "cases", ()) or ():
|
|
456
|
+
yield from self._stmts(case.body)
|
|
457
|
+
|
|
458
|
+
def _function_state(self, fn: ast.AST) -> tuple[list, dict]:
|
|
459
|
+
key = id(fn)
|
|
460
|
+
if key in self._fn:
|
|
461
|
+
return self._fn[key]
|
|
462
|
+
base = self._handler_params(fn)
|
|
463
|
+
env = {k: list(v) for k, v in base.items()}
|
|
464
|
+
history: list[tuple[int, str, list[Atom]]] = []
|
|
465
|
+
|
|
466
|
+
def bind(line: int, names: list[str], merged: list[str], atoms: list[Atom], note: str, aug: bool = False) -> None:
|
|
467
|
+
for n in names:
|
|
468
|
+
new = [a.with_step(line, "propagation", note) for a in atoms]
|
|
469
|
+
if aug:
|
|
470
|
+
new = _dedupe(env.get(n, []) + new)
|
|
471
|
+
env[n] = new
|
|
472
|
+
history.append((line, n, new))
|
|
473
|
+
for n in merged:
|
|
474
|
+
new = _dedupe(env.get(n, []) + [a.with_step(line, "propagation", note) for a in atoms])
|
|
475
|
+
env[n] = new
|
|
476
|
+
history.append((line, n, new))
|
|
477
|
+
|
|
478
|
+
for st in self._stmts(fn.body): # type: ignore[attr-defined]
|
|
479
|
+
line = st.lineno
|
|
480
|
+
if isinstance(st, ast.Assign):
|
|
481
|
+
atoms = self._eval(st.value, env)
|
|
482
|
+
note = self.seg(st, cap=120).split("\n", 1)[0]
|
|
483
|
+
for t in st.targets:
|
|
484
|
+
b, m = self._names(t)
|
|
485
|
+
bind(line, b, m, atoms, note)
|
|
486
|
+
elif isinstance(st, ast.AnnAssign) and st.value is not None:
|
|
487
|
+
b, m = self._names(st.target)
|
|
488
|
+
bind(line, b, m, self._eval(st.value, env), self.seg(st, cap=120).split("\n", 1)[0])
|
|
489
|
+
elif isinstance(st, ast.AugAssign):
|
|
490
|
+
b, m = self._names(st.target)
|
|
491
|
+
bind(line, b, m, self._eval(st.value, env), self.seg(st, cap=120).split("\n", 1)[0], aug=True)
|
|
492
|
+
elif isinstance(st, (ast.For, ast.AsyncFor)):
|
|
493
|
+
b, m = self._names(st.target)
|
|
494
|
+
bind(line, b, m, self._eval(st.iter, env), f"for … in {self.seg(st.iter, cap=80)}")
|
|
495
|
+
elif isinstance(st, (ast.With, ast.AsyncWith)):
|
|
496
|
+
for item in st.items:
|
|
497
|
+
if item.optional_vars is not None:
|
|
498
|
+
b, m = self._names(item.optional_vars)
|
|
499
|
+
bind(line, b, m, self._eval(item.context_expr, env), f"with … as {self.seg(item.optional_vars, cap=40)}")
|
|
500
|
+
elif (
|
|
501
|
+
isinstance(st, ast.Expr)
|
|
502
|
+
and isinstance(st.value, ast.Call)
|
|
503
|
+
and isinstance(st.value.func, ast.Attribute)
|
|
504
|
+
and st.value.func.attr in _PY_MUTATORS
|
|
505
|
+
and isinstance(st.value.func.value, ast.Name)
|
|
506
|
+
):
|
|
507
|
+
call = st.value
|
|
508
|
+
atoms: list[Atom] = []
|
|
509
|
+
for a in call.args:
|
|
510
|
+
atoms += self._eval(a, env)
|
|
511
|
+
for kw in call.keywords:
|
|
512
|
+
atoms += self._eval(kw.value, env)
|
|
513
|
+
if atoms:
|
|
514
|
+
bind(line, [], [call.func.value.id], atoms, self.seg(st, cap=120).split("\n", 1)[0])
|
|
515
|
+
state = (history, base)
|
|
516
|
+
self._fn[key] = state
|
|
517
|
+
return state
|
|
518
|
+
|
|
519
|
+
def _env_at(self, fn: ast.AST, line: int) -> dict[str, list[Atom]]:
|
|
520
|
+
history, base = self._function_state(fn)
|
|
521
|
+
env = {k: list(v) for k, v in base.items()}
|
|
522
|
+
for hline, name, atoms in history:
|
|
523
|
+
if hline < line:
|
|
524
|
+
env[name] = atoms
|
|
525
|
+
return env
|
|
526
|
+
|
|
527
|
+
# -- sink --
|
|
528
|
+
def _find_call(self, fn: ast.AST, paren: int) -> ast.Call | None:
|
|
529
|
+
content = self.content
|
|
530
|
+
line_start = content.rfind("\n", 0, paren) + 1
|
|
531
|
+
lp = line_number(content, paren)
|
|
532
|
+
bcol = len(content[line_start:paren].encode("utf-8", "surrogatepass"))
|
|
533
|
+
best: ast.Call | None = None
|
|
534
|
+
best_col = -1
|
|
535
|
+
for node in ast.walk(fn):
|
|
536
|
+
if isinstance(node, ast.Call):
|
|
537
|
+
f = node.func
|
|
538
|
+
if f.end_lineno == lp and f.end_col_offset is not None and best_col < f.end_col_offset <= bcol:
|
|
539
|
+
best, best_col = node, f.end_col_offset
|
|
540
|
+
return best
|
|
541
|
+
|
|
542
|
+
def _leading_literal(self, node: ast.AST) -> str | None:
|
|
543
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
544
|
+
return node.value
|
|
545
|
+
if isinstance(node, ast.JoinedStr) and node.values:
|
|
546
|
+
first = node.values[0]
|
|
547
|
+
if isinstance(first, ast.Constant) and isinstance(first.value, str):
|
|
548
|
+
return first.value
|
|
549
|
+
return None
|
|
550
|
+
if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.Add, ast.Mod)):
|
|
551
|
+
return self._leading_literal(node.left)
|
|
552
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "format":
|
|
553
|
+
return self._leading_literal(node.func.value)
|
|
554
|
+
return None
|
|
555
|
+
|
|
556
|
+
def _stmt_path(self, body: list[ast.stmt], line: int) -> list[tuple[list[ast.stmt], int]]:
|
|
557
|
+
path: list[tuple[list[ast.stmt], int]] = []
|
|
558
|
+
current = body
|
|
559
|
+
while True:
|
|
560
|
+
for i, st in enumerate(current):
|
|
561
|
+
if st.lineno <= line <= (st.end_lineno or st.lineno):
|
|
562
|
+
path.append((current, i))
|
|
563
|
+
break
|
|
564
|
+
else:
|
|
565
|
+
return path
|
|
566
|
+
st = current[path[-1][1]]
|
|
567
|
+
if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
568
|
+
return path
|
|
569
|
+
subs: list[list[ast.stmt]] = []
|
|
570
|
+
for attr in ("body", "orelse", "finalbody"):
|
|
571
|
+
sub = getattr(st, attr, None)
|
|
572
|
+
if isinstance(sub, list) and sub and isinstance(sub[0], ast.stmt):
|
|
573
|
+
subs.append(sub)
|
|
574
|
+
subs += [h.body for h in getattr(st, "handlers", ()) or ()]
|
|
575
|
+
subs += [c.body for c in getattr(st, "cases", ()) or ()]
|
|
576
|
+
nxt = None
|
|
577
|
+
for sub in subs:
|
|
578
|
+
if sub[0].lineno <= line <= (sub[-1].end_lineno or sub[-1].lineno):
|
|
579
|
+
nxt = sub
|
|
580
|
+
break
|
|
581
|
+
if nxt is None:
|
|
582
|
+
return path
|
|
583
|
+
current = nxt
|
|
584
|
+
|
|
585
|
+
def _terminates(self, body: list[ast.stmt]) -> bool:
|
|
586
|
+
if not body:
|
|
587
|
+
return False
|
|
588
|
+
last = body[-1]
|
|
589
|
+
if isinstance(last, (ast.Return, ast.Raise, ast.Continue, ast.Break)):
|
|
590
|
+
return True
|
|
591
|
+
if isinstance(last, ast.Expr) and isinstance(last.value, ast.Call):
|
|
592
|
+
f = last.value.func
|
|
593
|
+
name = f.attr if isinstance(f, ast.Attribute) else f.id if isinstance(f, ast.Name) else ""
|
|
594
|
+
return name in _PY_EXIT_CALLS
|
|
595
|
+
return False
|
|
596
|
+
|
|
597
|
+
def _guard_tests(self, fn: ast.AST, line: int) -> list[ast.expr]:
|
|
598
|
+
tests: list[ast.expr] = []
|
|
599
|
+
for lst, i in self._stmt_path(fn.body, line): # type: ignore[attr-defined]
|
|
600
|
+
for prior in lst[:i]:
|
|
601
|
+
if isinstance(prior, ast.If) and self._terminates(prior.body):
|
|
602
|
+
tests.append(prior.test)
|
|
603
|
+
elif isinstance(prior, ast.Assert):
|
|
604
|
+
tests.append(prior.test)
|
|
605
|
+
st = lst[i]
|
|
606
|
+
if isinstance(st, ast.If) and st.body and st.body[0].lineno <= line <= (st.body[-1].end_lineno or 0):
|
|
607
|
+
tests.append(st.test)
|
|
608
|
+
return tests
|
|
609
|
+
|
|
610
|
+
def _rest_nodes(self, fn: ast.AST, line: int) -> tuple[int, list[ast.AST]] | None:
|
|
611
|
+
"""The value of the simple statement on ``line`` (an assignment's
|
|
612
|
+
right-hand side, or an expression statement) for statement sinks."""
|
|
613
|
+
best: ast.stmt | None = None
|
|
614
|
+
for node in ast.walk(fn):
|
|
615
|
+
if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign, ast.Expr)) and node.lineno == line:
|
|
616
|
+
best = node
|
|
617
|
+
break
|
|
618
|
+
if best is None or getattr(best, "value", None) is None:
|
|
619
|
+
return None
|
|
620
|
+
return best.lineno, [best.value] # type: ignore[attr-defined]
|
|
621
|
+
|
|
622
|
+
def sink_flow(
|
|
623
|
+
self,
|
|
624
|
+
kind: str,
|
|
625
|
+
paren: int,
|
|
626
|
+
args_spec: tuple[int, ...] | None,
|
|
627
|
+
snippet: str,
|
|
628
|
+
match_start: int,
|
|
629
|
+
match_end: int,
|
|
630
|
+
) -> SinkFlow | None:
|
|
631
|
+
anchor = match_start if args_spec == REST else paren
|
|
632
|
+
line = line_number(self.content, anchor)
|
|
633
|
+
fn = self._enclosing(line)
|
|
634
|
+
if fn is None:
|
|
635
|
+
return None
|
|
636
|
+
if args_spec == REST:
|
|
637
|
+
found = self._rest_nodes(fn, line)
|
|
638
|
+
if found is None:
|
|
639
|
+
return None
|
|
640
|
+
call_line, nodes = found
|
|
641
|
+
return self._finish(kind, fn, call_line, nodes, snippet)
|
|
642
|
+
call = self._find_call(fn, paren)
|
|
643
|
+
if call is None:
|
|
644
|
+
return None
|
|
645
|
+
if args_spec is None:
|
|
646
|
+
nodes: list[ast.AST] = list(call.args) + [kw.value for kw in call.keywords]
|
|
647
|
+
elif args_spec == RECEIVER:
|
|
648
|
+
nodes = [call.func.value] if isinstance(call.func, ast.Attribute) else []
|
|
649
|
+
else:
|
|
650
|
+
nodes = [call.args[i] for i in args_spec if 0 <= i < len(call.args)]
|
|
651
|
+
nodes += [kw.value for kw in call.keywords if kw.arg in _PY_ARG_KEYWORDS]
|
|
652
|
+
if not nodes:
|
|
653
|
+
nodes = [kw.value for kw in call.keywords]
|
|
654
|
+
return self._finish(kind, fn, call.lineno, nodes, snippet)
|
|
655
|
+
|
|
656
|
+
def _finish(self, kind: str, fn: ast.AST, call_line: int, nodes: list[ast.AST], snippet: str) -> SinkFlow:
|
|
657
|
+
env = self._env_at(fn, call_line)
|
|
658
|
+
atoms: list[Atom] = []
|
|
659
|
+
for node in nodes:
|
|
660
|
+
if _fixed_prefix(kind, self._leading_literal(node)):
|
|
661
|
+
continue
|
|
662
|
+
if kind == "open_redirect" and _internal_url_node(node):
|
|
663
|
+
continue
|
|
664
|
+
atoms += self._eval(node, env)
|
|
665
|
+
atoms = _dedupe(atoms)
|
|
666
|
+
atoms = self._apply_guards(kind, fn, call_line, env, atoms)
|
|
667
|
+
start = min([d.lineno for d in fn.decorator_list] + [fn.lineno]) # type: ignore[attr-defined]
|
|
668
|
+
return _verdict(kind, atoms, call_line, snippet, (start, fn.end_lineno or start)) # type: ignore[attr-defined]
|
|
669
|
+
|
|
670
|
+
def _apply_guards(self, kind: str, fn: ast.AST, line: int, env: dict, atoms: list[Atom]) -> list[Atom]:
|
|
671
|
+
live = {a.ident for a in atoms if a.sanitizer_for(kind) is None}
|
|
672
|
+
if not live:
|
|
673
|
+
return atoms
|
|
674
|
+
for test in self._guard_tests(fn, line):
|
|
675
|
+
text = self.seg(test, cap=400)
|
|
676
|
+
label = next((lbl for lbl, pat in guard_patterns(kind) if pat.search(text)), None)
|
|
677
|
+
if label is None:
|
|
678
|
+
continue
|
|
679
|
+
bound = {a.ident for a in self._eval(test, env)} & live
|
|
680
|
+
if bound:
|
|
681
|
+
atoms = [
|
|
682
|
+
a.with_sanitizer((kind,), label, test.lineno, "guard") if a.ident in bound else a
|
|
683
|
+
for a in atoms
|
|
684
|
+
]
|
|
685
|
+
return atoms
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
# =============================================================================
|
|
689
|
+
# JS/TS, Go, PHP, Java (masked text + brace balance)
|
|
690
|
+
# =============================================================================
|
|
691
|
+
|
|
692
|
+
_MASK_SPECIAL = {
|
|
693
|
+
JS: re.compile(r"[\"'`/]"),
|
|
694
|
+
GO: re.compile(r"[\"'`/]"),
|
|
695
|
+
JAVA: re.compile(r"[\"'/]"),
|
|
696
|
+
PHP: re.compile(r"[\"'/#]"),
|
|
697
|
+
}
|
|
698
|
+
_MASK_SPECIAL[CSHARP] = _MASK_SPECIAL[JAVA]
|
|
699
|
+
_PHP_VAR_RE = re.compile(r"\$[A-Za-z_]\w*")
|
|
700
|
+
# String-literal bodies (unambiguous alternation → linear).
|
|
701
|
+
_QUOTED_BODY = {
|
|
702
|
+
('"', False): re.compile(r'(?:[^"\\\n]|\\.)*'),
|
|
703
|
+
("'", False): re.compile(r"(?:[^'\\\n]|\\.)*"),
|
|
704
|
+
('"', True): re.compile(r'(?:[^"\\]|\\.)*', re.S),
|
|
705
|
+
("'", True): re.compile(r"(?:[^'\\]|\\.)*", re.S),
|
|
706
|
+
}
|
|
707
|
+
_BRACES_RE = re.compile(r"[{}]")
|
|
708
|
+
# A JS arrow and the whitespace after it; a body not starting with `{` is concise (#398).
|
|
709
|
+
_CONCISE_ARROW_RE = re.compile(r"=>[ \t\r\n]*")
|
|
710
|
+
_PARENS_RE = re.compile(r"[()]")
|
|
711
|
+
_STMT_SPECIAL_RE = re.compile(r"[()\[\]{};\n]")
|
|
712
|
+
_COMMA_SPECIAL_RE = re.compile(r"[()\[\]{},]")
|
|
713
|
+
_ASSIGN_OP = {
|
|
714
|
+
JS: re.compile(r"\+=|(?<![=!<>+\-*/%&|^:~?])=(?![=>~])"),
|
|
715
|
+
JAVA: re.compile(r"\+=|(?<![=!<>+\-*/%&|^:~?])=(?![=>~])"),
|
|
716
|
+
GO: re.compile(r":=|\+=|(?<![=!<>+\-*/%&|^:~])=(?![=>~])"),
|
|
717
|
+
PHP: re.compile(r"\.=|\+=|\?\?=|(?<![=!<>+\-*/%&|^:~.?])=(?![=>~])"),
|
|
718
|
+
}
|
|
719
|
+
_IDENT_RE = {
|
|
720
|
+
JS: re.compile(r"(?<![\w$.])[A-Za-z_$][\w$]*"),
|
|
721
|
+
JAVA: re.compile(r"(?<![\w$.])[A-Za-z_$][\w$]*"),
|
|
722
|
+
GO: re.compile(r"(?<![\w.])[A-Za-z_]\w*"),
|
|
723
|
+
PHP: re.compile(r"\$[A-Za-z_]\w*"),
|
|
724
|
+
}
|
|
725
|
+
_ASSIGN_OP[CSHARP] = _ASSIGN_OP[JAVA]
|
|
726
|
+
_IDENT_RE[CSHARP] = _IDENT_RE[JAVA]
|
|
727
|
+
_GO_OUTPARAM_RE = re.compile(r"&\s*([A-Za-z_]\w*)")
|
|
728
|
+
_IF_RE = re.compile(r"\bif\b")
|
|
729
|
+
_FN_KEYWORD_RE = re.compile(r"\b(?:function|func)\b")
|
|
730
|
+
_NEW_CLASS_RE = re.compile(r"\bnew\s+[\w.<>]+\s*$")
|
|
731
|
+
_TRAILING_IDENT_RE = re.compile(r"[A-Za-z_$][\w$]*$")
|
|
732
|
+
_FMT_SPRINTF_RE = re.compile(r"(?:fmt\.Sprintf|String\.format|sprintf)\s*\(\s*")
|
|
733
|
+
_CONTROL = frozenset(
|
|
734
|
+
{
|
|
735
|
+
"if", "for", "while", "switch", "catch", "with", "foreach", "elseif", "synchronized",
|
|
736
|
+
"using", "lock", "return", "typeof", "await", "yield", "else", "do", "try", "case", "in", "of",
|
|
737
|
+
"and", "or", "not", "sizeof", "defer", "go", "select", "range",
|
|
738
|
+
}
|
|
739
|
+
)
|
|
740
|
+
_KEYWORDS = frozenset(
|
|
741
|
+
{
|
|
742
|
+
"const", "let", "var", "final", "static", "public", "private", "protected", "if", "for", "range",
|
|
743
|
+
"return", "new", "this", "true", "false", "null", "nil", "undefined", "await", "async", "typeof",
|
|
744
|
+
"instanceof", "in", "of", "else", "string", "int", "bool", "byte", "error", "func", "go", "defer",
|
|
745
|
+
}
|
|
746
|
+
)
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def _mask(src: str, lang: str) -> str:
|
|
750
|
+
"""Blank string-literal contents and comments (offsets and newlines kept).
|
|
751
|
+
|
|
752
|
+
Interpolations stay visible: JS template ``${…}`` and PHP double-quoted
|
|
753
|
+
``$var`` carry data into the string."""
|
|
754
|
+
out = list(src)
|
|
755
|
+
n = len(src)
|
|
756
|
+
special = _MASK_SPECIAL[lang]
|
|
757
|
+
|
|
758
|
+
def blank(a: int, b: int) -> None:
|
|
759
|
+
for k in range(a, min(b, n)):
|
|
760
|
+
if out[k] != "\n":
|
|
761
|
+
out[k] = " "
|
|
762
|
+
|
|
763
|
+
def quoted(i: int, q: str, multiline: bool) -> int:
|
|
764
|
+
# Index of the closing quote (or the newline / EOF ending an
|
|
765
|
+
# unterminated single-line literal).
|
|
766
|
+
return _QUOTED_BODY[(q, multiline)].match(src, i + 1).end() # type: ignore[union-attr]
|
|
767
|
+
|
|
768
|
+
def template(i: int) -> int:
|
|
769
|
+
# JS template literal starting at backtick i; returns index of closing backtick.
|
|
770
|
+
j = i + 1
|
|
771
|
+
lit_start = j
|
|
772
|
+
while j < n:
|
|
773
|
+
c = src[j]
|
|
774
|
+
if c == "\\":
|
|
775
|
+
j += 2
|
|
776
|
+
continue
|
|
777
|
+
if c == "`":
|
|
778
|
+
blank(lit_start, j)
|
|
779
|
+
return j
|
|
780
|
+
if c == "$" and j + 1 < n and src[j + 1] == "{":
|
|
781
|
+
blank(lit_start, j)
|
|
782
|
+
j = code(j + 2, stop_on_brace=True)
|
|
783
|
+
lit_start = j + 1
|
|
784
|
+
j += 1
|
|
785
|
+
blank(lit_start, n)
|
|
786
|
+
return n
|
|
787
|
+
|
|
788
|
+
def code(i: int, stop_on_brace: bool = False) -> int:
|
|
789
|
+
depth = 0
|
|
790
|
+
while i < n:
|
|
791
|
+
if stop_on_brace:
|
|
792
|
+
# Track braces only inside an interpolation; scan char-wise.
|
|
793
|
+
c = src[i]
|
|
794
|
+
if c == "{":
|
|
795
|
+
depth += 1
|
|
796
|
+
elif c == "}":
|
|
797
|
+
if depth == 0:
|
|
798
|
+
return i
|
|
799
|
+
depth -= 1
|
|
800
|
+
elif c in "\"'`/":
|
|
801
|
+
i = handle(i)
|
|
802
|
+
i += 1
|
|
803
|
+
continue
|
|
804
|
+
m = special.search(src, i)
|
|
805
|
+
if m is None:
|
|
806
|
+
return n
|
|
807
|
+
i = handle(m.start()) + 1
|
|
808
|
+
return n
|
|
809
|
+
|
|
810
|
+
def handle(i: int) -> int:
|
|
811
|
+
c = src[i]
|
|
812
|
+
nxt = src[i + 1] if i + 1 < n else ""
|
|
813
|
+
if c == "/" and nxt == "/" or (c == "#" and lang == PHP and nxt != "["):
|
|
814
|
+
j = src.find("\n", i)
|
|
815
|
+
j = n if j == -1 else j
|
|
816
|
+
blank(i, j)
|
|
817
|
+
return j - 1 if j < n else n
|
|
818
|
+
if c == "/" and nxt == "*":
|
|
819
|
+
j = src.find("*/", i + 2)
|
|
820
|
+
j = n if j == -1 else j + 2
|
|
821
|
+
blank(i, j)
|
|
822
|
+
return j - 1
|
|
823
|
+
if c == "/":
|
|
824
|
+
return i
|
|
825
|
+
if c == "`" and lang == JS:
|
|
826
|
+
return template(i)
|
|
827
|
+
if c == "`" and lang == GO:
|
|
828
|
+
j = src.find("`", i + 1)
|
|
829
|
+
j = n if j == -1 else j
|
|
830
|
+
blank(i + 1, j)
|
|
831
|
+
return j
|
|
832
|
+
if c == '"' and lang == JAVA and src.startswith('"""', i):
|
|
833
|
+
j = src.find('"""', i + 3)
|
|
834
|
+
j = n if j == -1 else j
|
|
835
|
+
blank(i + 3, j)
|
|
836
|
+
return j + 2
|
|
837
|
+
if c in "\"'":
|
|
838
|
+
j = quoted(i, c, multiline=(lang == PHP))
|
|
839
|
+
blank(i + 1, j)
|
|
840
|
+
if lang == PHP and c == '"':
|
|
841
|
+
for m in _PHP_VAR_RE.finditer(src, i + 1, j):
|
|
842
|
+
out[m.start() : m.end()] = src[m.start() : m.end()]
|
|
843
|
+
return j
|
|
844
|
+
return i
|
|
845
|
+
|
|
846
|
+
code(0)
|
|
847
|
+
return "".join(out)
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
class _TextFlow:
|
|
851
|
+
def __init__(self, content: str, lang: str) -> None:
|
|
852
|
+
self.raw = content
|
|
853
|
+
self.lang = lang
|
|
854
|
+
self.m = _mask(content, lang)
|
|
855
|
+
opens: list[int] = []
|
|
856
|
+
closes: list[int] = []
|
|
857
|
+
parent: list[int] = []
|
|
858
|
+
stack: list[int] = []
|
|
859
|
+
for mt in _BRACES_RE.finditer(self.m):
|
|
860
|
+
if mt.group() == "{":
|
|
861
|
+
parent.append(stack[-1] if stack else -1)
|
|
862
|
+
opens.append(mt.start())
|
|
863
|
+
closes.append(len(self.m))
|
|
864
|
+
stack.append(len(opens) - 1)
|
|
865
|
+
elif stack:
|
|
866
|
+
closes[stack.pop()] = mt.start()
|
|
867
|
+
# Concise arrow bodies (`(req, res) => db.query(...)`, #398) have no
|
|
868
|
+
# braces; each becomes a synthetic block that opens on the `>` of its
|
|
869
|
+
# `=>`, so `_body` starts right after it and the function machinery
|
|
870
|
+
# treats the expression like a braced body.
|
|
871
|
+
self._concise: set[int] = set()
|
|
872
|
+
concise = self._concise_bodies() if lang == JS else []
|
|
873
|
+
if concise:
|
|
874
|
+
blocks = sorted([(o, c, False) for o, c in zip(opens, closes)] + [(o, c, True) for o, c in concise])
|
|
875
|
+
opens, closes, parent = [], [], []
|
|
876
|
+
stack = []
|
|
877
|
+
for o, c, synthetic in blocks:
|
|
878
|
+
while stack and closes[stack[-1]] < o:
|
|
879
|
+
stack.pop()
|
|
880
|
+
parent.append(stack[-1] if stack else -1)
|
|
881
|
+
opens.append(o)
|
|
882
|
+
closes.append(c)
|
|
883
|
+
if synthetic:
|
|
884
|
+
self._concise.add(len(opens) - 1)
|
|
885
|
+
stack.append(len(opens) - 1)
|
|
886
|
+
self.opens, self.closes, self.parent = opens, closes, parent
|
|
887
|
+
self.open_index = {o: k for k, o in enumerate(opens) if k not in self._concise}
|
|
888
|
+
self._is_fn: dict[int, tuple[int, int, int] | None] = {}
|
|
889
|
+
self._fn_state: dict[int, tuple[list, dict]] = {}
|
|
890
|
+
|
|
891
|
+
# -- structure --
|
|
892
|
+
def _concise_bodies(self) -> list[tuple[int, int]]:
|
|
893
|
+
"""(open, close) of every concise arrow body: ``open`` is the ``>`` of
|
|
894
|
+
its ``=>``, ``close`` the first depth-0 ``,`` / ``;`` / closing bracket,
|
|
895
|
+
or the statement's end (``_stmt_end``: a line break ends it unless the
|
|
896
|
+
expression continues)."""
|
|
897
|
+
m = self.m
|
|
898
|
+
out: list[tuple[int, int]] = []
|
|
899
|
+
for mt in _CONCISE_ARROW_RE.finditer(m):
|
|
900
|
+
body = mt.end()
|
|
901
|
+
if body >= len(m) or m[body] == "{":
|
|
902
|
+
continue
|
|
903
|
+
end = self._stmt_end(body, len(m))
|
|
904
|
+
depth = 0
|
|
905
|
+
for sp in _COMMA_SPECIAL_RE.finditer(m, body, end):
|
|
906
|
+
c = sp.group()
|
|
907
|
+
if c in "([{":
|
|
908
|
+
depth += 1
|
|
909
|
+
elif c in ")]}":
|
|
910
|
+
depth -= 1
|
|
911
|
+
elif depth == 0:
|
|
912
|
+
end = sp.start()
|
|
913
|
+
break
|
|
914
|
+
if end > body:
|
|
915
|
+
out.append((mt.start() + 1, end))
|
|
916
|
+
return out
|
|
917
|
+
|
|
918
|
+
def _innermost(self, off: int) -> int:
|
|
919
|
+
k = bisect.bisect_right(self.opens, off) - 1
|
|
920
|
+
while k >= 0 and not (self.opens[k] < off < self.closes[k]):
|
|
921
|
+
k = self.parent[k]
|
|
922
|
+
return k
|
|
923
|
+
|
|
924
|
+
def _function_header(self, k: int) -> tuple[int, int, int] | None:
|
|
925
|
+
"""(header_start, params_open, params_close) when block ``k`` is a
|
|
926
|
+
function body, else None. params_* are -1 when not located."""
|
|
927
|
+
if k in self._is_fn:
|
|
928
|
+
return self._is_fn[k]
|
|
929
|
+
o = self.opens[k]
|
|
930
|
+
# A concise body opens on the `>` of `=>`: the header includes it.
|
|
931
|
+
hend = o + 1 if k in self._concise else o
|
|
932
|
+
lo = max(0, o - _MAX_HEADER)
|
|
933
|
+
seg = self.m[lo:hend]
|
|
934
|
+
cut = max(seg.rfind(";"), seg.rfind("{"), seg.rfind("}"))
|
|
935
|
+
start = lo + cut + 1
|
|
936
|
+
header = self.m[start:hend]
|
|
937
|
+
h = header.rstrip()
|
|
938
|
+
result: tuple[int, int, int] | None = None
|
|
939
|
+
hs = start + (len(header) - len(header.lstrip()))
|
|
940
|
+
p = h.rfind(")")
|
|
941
|
+
po = -1
|
|
942
|
+
if p != -1:
|
|
943
|
+
depth = 0
|
|
944
|
+
for q in range(p, -1, -1):
|
|
945
|
+
ch = h[q]
|
|
946
|
+
if ch == ")":
|
|
947
|
+
depth += 1
|
|
948
|
+
elif ch == "(":
|
|
949
|
+
depth -= 1
|
|
950
|
+
if depth == 0:
|
|
951
|
+
po = q
|
|
952
|
+
break
|
|
953
|
+
params = (start + po, start + p) if po != -1 else (-1, -1)
|
|
954
|
+
if h.endswith("=>") or h.endswith("->") or _FN_KEYWORD_RE.search(h):
|
|
955
|
+
result = (hs, *params)
|
|
956
|
+
elif p != -1 and po != -1:
|
|
957
|
+
tail = h[p + 1 :].strip()
|
|
958
|
+
before = h[:po].rstrip()
|
|
959
|
+
word = _TRAILING_IDENT_RE.search(before)
|
|
960
|
+
if (
|
|
961
|
+
(tail == "" or tail.startswith(":") or tail.startswith("throws"))
|
|
962
|
+
and word is not None
|
|
963
|
+
and word.group() not in _CONTROL
|
|
964
|
+
and not _NEW_CLASS_RE.search(before)
|
|
965
|
+
):
|
|
966
|
+
result = (hs, *params)
|
|
967
|
+
self._is_fn[k] = result
|
|
968
|
+
return result
|
|
969
|
+
|
|
970
|
+
def _enclosing_fn(self, off: int) -> tuple[int, tuple[int, int, int]] | None:
|
|
971
|
+
k = self._innermost(off)
|
|
972
|
+
while k >= 0:
|
|
973
|
+
hdr = self._function_header(k)
|
|
974
|
+
if hdr is not None:
|
|
975
|
+
return k, hdr
|
|
976
|
+
k = self.parent[k]
|
|
977
|
+
return None
|
|
978
|
+
|
|
979
|
+
def _match_paren(self, open_idx: int, cap: int) -> int:
|
|
980
|
+
depth = 0
|
|
981
|
+
end = min(len(self.m), open_idx + cap)
|
|
982
|
+
for mt in _PARENS_RE.finditer(self.m, open_idx, end):
|
|
983
|
+
if mt.group() == "(":
|
|
984
|
+
depth += 1
|
|
985
|
+
else:
|
|
986
|
+
depth -= 1
|
|
987
|
+
if depth == 0:
|
|
988
|
+
return mt.start()
|
|
989
|
+
return end
|
|
990
|
+
|
|
991
|
+
def _stmt_end(self, i: int, limit: int) -> int:
|
|
992
|
+
m = self.m
|
|
993
|
+
depth = 0
|
|
994
|
+
end = min(limit, i + _MAX_RHS)
|
|
995
|
+
pos = i
|
|
996
|
+
while True:
|
|
997
|
+
mt = _STMT_SPECIAL_RE.search(m, pos, end)
|
|
998
|
+
if mt is None:
|
|
999
|
+
return end
|
|
1000
|
+
c = mt.group()
|
|
1001
|
+
j = mt.start()
|
|
1002
|
+
if c in "([{":
|
|
1003
|
+
depth += 1
|
|
1004
|
+
elif c in ")]}":
|
|
1005
|
+
if depth == 0:
|
|
1006
|
+
return j
|
|
1007
|
+
depth -= 1
|
|
1008
|
+
elif c == ";" and depth == 0:
|
|
1009
|
+
return j
|
|
1010
|
+
elif c == "\n" and depth == 0:
|
|
1011
|
+
prev = m[max(i, j - 200) : j].rstrip()
|
|
1012
|
+
rest = m[j + 1 : min(end, j + 200)].lstrip()
|
|
1013
|
+
cont_prev = bool(prev) and prev[-1] in "+,(.=?:&|[{-*/%"
|
|
1014
|
+
cont_next = bool(rest) and rest[0] in ".+?:&|-*/%" and not rest.startswith("//")
|
|
1015
|
+
if not (cont_prev or cont_next) and prev:
|
|
1016
|
+
return j
|
|
1017
|
+
pos = j + 1
|
|
1018
|
+
|
|
1019
|
+
def _split_args(self, a: int, b: int) -> list[tuple[int, int]]:
|
|
1020
|
+
regions: list[tuple[int, int]] = []
|
|
1021
|
+
depth = 0
|
|
1022
|
+
start = a
|
|
1023
|
+
for mt in _COMMA_SPECIAL_RE.finditer(self.m, a, b):
|
|
1024
|
+
c = mt.group()
|
|
1025
|
+
if c in "([{":
|
|
1026
|
+
depth += 1
|
|
1027
|
+
elif c in ")]}":
|
|
1028
|
+
depth -= 1
|
|
1029
|
+
elif depth == 0:
|
|
1030
|
+
regions.append((start, mt.start()))
|
|
1031
|
+
start = mt.end()
|
|
1032
|
+
regions.append((start, b))
|
|
1033
|
+
return [(s, e) for s, e in regions if self.m[s:e].strip()]
|
|
1034
|
+
|
|
1035
|
+
# -- evaluation --
|
|
1036
|
+
def _is_code(self, off: int) -> bool:
|
|
1037
|
+
return self.m[off] == self.raw[off]
|
|
1038
|
+
|
|
1039
|
+
def _eval(self, a: int, b: int, env: dict[str, list[Atom]]) -> list[Atom]:
|
|
1040
|
+
if b <= a:
|
|
1041
|
+
return []
|
|
1042
|
+
spans: list[tuple[int, int, frozenset[str], str]] = []
|
|
1043
|
+
for kinds, label, pat in _SANITIZER_TABLE:
|
|
1044
|
+
for mt in pat.finditer(self.m, a, b):
|
|
1045
|
+
if self.m[mt.end() - 1] == "(":
|
|
1046
|
+
close = self._match_paren(mt.end() - 1, _MAX_RHS)
|
|
1047
|
+
else: # PHP `(int)$x` cast: covers the following variable
|
|
1048
|
+
tail = re.match(r"\w*", self.m[mt.end() : mt.end() + 64])
|
|
1049
|
+
close = mt.end() + (tail.end() if tail else 0)
|
|
1050
|
+
spans.append((mt.start(), close, kinds, label))
|
|
1051
|
+
|
|
1052
|
+
def sanitize(atom: Atom, off: int) -> Atom:
|
|
1053
|
+
for s, e, kinds, label in spans:
|
|
1054
|
+
if s <= off <= e:
|
|
1055
|
+
atom = atom.with_sanitizer(kinds, label, line_number(self.raw, s))
|
|
1056
|
+
return atom
|
|
1057
|
+
|
|
1058
|
+
atoms: list[Atom] = []
|
|
1059
|
+
for kind, pat, weight in SOURCES[self.lang]:
|
|
1060
|
+
for mt in pat.finditer(self.raw, a, b):
|
|
1061
|
+
if not self._is_code(mt.start()) and "php:" not in pat.pattern:
|
|
1062
|
+
continue
|
|
1063
|
+
line = line_number(self.raw, mt.start())
|
|
1064
|
+
label = self.raw[mt.start() : min(b, mt.end() + 24)].split("\n", 1)[0].strip()[:80]
|
|
1065
|
+
atom = Atom(kind, line, weight, label, (), ((line, "source", f"{kind}: {label}"),))
|
|
1066
|
+
atoms.append(sanitize(atom, mt.start()))
|
|
1067
|
+
for mt in _IDENT_RE[self.lang].finditer(self.m, a, b):
|
|
1068
|
+
got = env.get(mt.group())
|
|
1069
|
+
if got:
|
|
1070
|
+
atoms += [sanitize(x, mt.start()) for x in got]
|
|
1071
|
+
return _dedupe(atoms)
|
|
1072
|
+
|
|
1073
|
+
def _param_env(self, hdr: tuple[int, int, int]) -> dict[str, list[Atom]]:
|
|
1074
|
+
env: dict[str, list[Atom]] = {}
|
|
1075
|
+
_, po, pc = hdr
|
|
1076
|
+
if po < 0 or self.lang == GO:
|
|
1077
|
+
return env
|
|
1078
|
+
line = line_number(self.raw, po)
|
|
1079
|
+
annotations = PARAM_ANNOTATIONS.get(self.lang, ())
|
|
1080
|
+
for s, e in self._split_args(po + 1, pc):
|
|
1081
|
+
raw_seg = self.raw[s:e]
|
|
1082
|
+
kind = next((k for k, pat in annotations if pat.search(raw_seg)), None)
|
|
1083
|
+
masked = self.m[s:e]
|
|
1084
|
+
if self.lang == PHP:
|
|
1085
|
+
names = _PHP_VAR_RE.findall(masked)
|
|
1086
|
+
name = names[-1] if names else None
|
|
1087
|
+
else:
|
|
1088
|
+
# strip annotations/decorators and their arguments
|
|
1089
|
+
cleaned = re.sub(r"@\w+(?:\s*\([^()]*\))?", " ", masked)
|
|
1090
|
+
cleaned = cleaned.split("=", 1)[0]
|
|
1091
|
+
if self.lang == JS:
|
|
1092
|
+
cleaned = cleaned.split(":", 1)[0]
|
|
1093
|
+
ids = [x for x in _IDENT_RE[self.lang].findall(cleaned) if x not in _KEYWORDS]
|
|
1094
|
+
name = ids[-1] if ids else None
|
|
1095
|
+
if not name:
|
|
1096
|
+
continue
|
|
1097
|
+
if kind is not None:
|
|
1098
|
+
label = f"handler parameter `{name}`"
|
|
1099
|
+
env[name] = [Atom(kind, line, "high", label, (), ((line, "source", f"{kind}: {label}"),))]
|
|
1100
|
+
else:
|
|
1101
|
+
env[name] = [Atom("param", line, "param", name)]
|
|
1102
|
+
return env
|
|
1103
|
+
|
|
1104
|
+
def _lhs_names(self, op: int, floor: int) -> tuple[list[str], list[str]]:
|
|
1105
|
+
m = self.m
|
|
1106
|
+
back = m[max(floor, op - 200) : op].rstrip()
|
|
1107
|
+
if not back:
|
|
1108
|
+
return [], []
|
|
1109
|
+
if back[-1] in "}]":
|
|
1110
|
+
closer = back[-1]
|
|
1111
|
+
opener = "{" if closer == "}" else "["
|
|
1112
|
+
depth = 0
|
|
1113
|
+
q = len(back) - 1
|
|
1114
|
+
while q >= 0:
|
|
1115
|
+
if back[q] == closer:
|
|
1116
|
+
depth += 1
|
|
1117
|
+
elif back[q] == opener:
|
|
1118
|
+
depth -= 1
|
|
1119
|
+
if depth == 0:
|
|
1120
|
+
break
|
|
1121
|
+
q -= 1
|
|
1122
|
+
if q < 0:
|
|
1123
|
+
return [], []
|
|
1124
|
+
before = back[:q].rstrip()
|
|
1125
|
+
inner = back[q + 1 : -1]
|
|
1126
|
+
if closer == "]" and before and (before[-1].isalnum() or before[-1] in "_$)]"):
|
|
1127
|
+
# subscript store: `x[k] = v` merges into x
|
|
1128
|
+
base = _IDENT_RE[self.lang].findall(before[-80:])
|
|
1129
|
+
return [], base[-1:] if base else []
|
|
1130
|
+
if self.lang == PHP:
|
|
1131
|
+
return _PHP_VAR_RE.findall(inner), []
|
|
1132
|
+
names = []
|
|
1133
|
+
for part in inner.split(","):
|
|
1134
|
+
part = part.split("=", 1)[0]
|
|
1135
|
+
ids = _IDENT_RE[self.lang].findall(part)
|
|
1136
|
+
if ids:
|
|
1137
|
+
names.append(ids[-1])
|
|
1138
|
+
return [n for n in names if n not in _KEYWORDS], []
|
|
1139
|
+
cut_chars = ";{}(\n" if self.lang == GO else ";{}(\n,"
|
|
1140
|
+
cut = max(back.rfind(c) for c in cut_chars)
|
|
1141
|
+
lhs = back[cut + 1 :]
|
|
1142
|
+
if lhs.rstrip().endswith((".", ")")):
|
|
1143
|
+
return [], []
|
|
1144
|
+
if self.lang == PHP:
|
|
1145
|
+
names = _PHP_VAR_RE.findall(lhs)
|
|
1146
|
+
if "->" in lhs or "::" in lhs:
|
|
1147
|
+
return [], []
|
|
1148
|
+
return names[-1:], []
|
|
1149
|
+
ids = [x for x in _IDENT_RE[self.lang].findall(lhs) if x not in _KEYWORDS]
|
|
1150
|
+
if not ids:
|
|
1151
|
+
return [], []
|
|
1152
|
+
# member store `obj.prop = v` → not a local
|
|
1153
|
+
last = lhs.rstrip()
|
|
1154
|
+
if re.search(r"\.\s*[A-Za-z_$][\w$]*$", last):
|
|
1155
|
+
return [], []
|
|
1156
|
+
if self.lang == GO:
|
|
1157
|
+
return ids, []
|
|
1158
|
+
return ids[-1:], []
|
|
1159
|
+
|
|
1160
|
+
def _function_state(self, k: int, hdr: tuple[int, int, int]) -> tuple[list, dict]:
|
|
1161
|
+
if k in self._fn_state:
|
|
1162
|
+
return self._fn_state[k]
|
|
1163
|
+
base = self._param_env(hdr)
|
|
1164
|
+
env = {n: list(v) for n, v in base.items()}
|
|
1165
|
+
body_lo, body_hi = self._body(k)
|
|
1166
|
+
events: list[tuple[int, str, int, int, int]] = [] # (offset, type, a, b, extra)
|
|
1167
|
+
for mt in _ASSIGN_OP[self.lang].finditer(self.m, body_lo, body_hi):
|
|
1168
|
+
events.append((mt.start(), "assign", mt.start(), mt.end(), 0))
|
|
1169
|
+
if self.lang == GO:
|
|
1170
|
+
for mt in _GO_OUTPARAM_RE.finditer(self.m, body_lo, body_hi):
|
|
1171
|
+
events.append((mt.start(), "out", mt.start(1), mt.end(1), 0))
|
|
1172
|
+
events.sort()
|
|
1173
|
+
history: list[tuple[int, str, list[Atom]]] = []
|
|
1174
|
+
for off, typ, a, b, _ in events:
|
|
1175
|
+
if typ == "assign":
|
|
1176
|
+
op_text = self.m[a:b]
|
|
1177
|
+
names, merged = self._lhs_names(a, body_lo)
|
|
1178
|
+
if not names and not merged:
|
|
1179
|
+
continue
|
|
1180
|
+
rhs_end = self._stmt_end(b, body_hi)
|
|
1181
|
+
atoms = self._eval(b, rhs_end, env)
|
|
1182
|
+
line = line_number(self.raw, a)
|
|
1183
|
+
stmt_start = self.raw.rfind("\n", 0, a) + 1
|
|
1184
|
+
note = self.raw[stmt_start:rhs_end].strip().split("\n", 1)[0]
|
|
1185
|
+
aug = op_text in ("+=", ".=", "??=")
|
|
1186
|
+
for n in names:
|
|
1187
|
+
new = [x.with_step(line, "propagation", note) for x in atoms]
|
|
1188
|
+
if aug:
|
|
1189
|
+
new = _dedupe(env.get(n, []) + new)
|
|
1190
|
+
env[n] = new
|
|
1191
|
+
history.append((a, n, new))
|
|
1192
|
+
for n in merged:
|
|
1193
|
+
new = _dedupe(env.get(n, []) + [x.with_step(line, "propagation", note) for x in atoms])
|
|
1194
|
+
env[n] = new
|
|
1195
|
+
history.append((a, n, new))
|
|
1196
|
+
else: # Go out-param: Decode(&x) / BindJSON(&x) / Unmarshal(data, &x)
|
|
1197
|
+
name = self.m[a:b]
|
|
1198
|
+
s = max(body_lo, self.m.rfind("\n", 0, off) + 1)
|
|
1199
|
+
e = self._stmt_end(s, body_hi)
|
|
1200
|
+
atoms = [x for x in self._eval(s, e, env) if x.label != name]
|
|
1201
|
+
if atoms:
|
|
1202
|
+
line = line_number(self.raw, off)
|
|
1203
|
+
note = self.raw[s:e].strip().split("\n", 1)[0]
|
|
1204
|
+
new = _dedupe(env.get(name, []) + [x.with_step(line, "propagation", note) for x in atoms])
|
|
1205
|
+
env[name] = new
|
|
1206
|
+
history.append((off, name, new))
|
|
1207
|
+
state = (history, base)
|
|
1208
|
+
self._fn_state[k] = state
|
|
1209
|
+
return state
|
|
1210
|
+
|
|
1211
|
+
def _env_at(self, k: int, hdr: tuple[int, int, int], off: int) -> dict[str, list[Atom]]:
|
|
1212
|
+
history, base = self._function_state(k, hdr)
|
|
1213
|
+
env = {n: list(v) for n, v in base.items()}
|
|
1214
|
+
for hoff, name, atoms in history:
|
|
1215
|
+
if hoff < off:
|
|
1216
|
+
env[name] = atoms
|
|
1217
|
+
return env
|
|
1218
|
+
|
|
1219
|
+
def _leading_literal(self, s: int, e: int) -> str | None:
|
|
1220
|
+
text = self.raw[s:e].strip()
|
|
1221
|
+
fm = _FMT_SPRINTF_RE.match(text)
|
|
1222
|
+
if fm:
|
|
1223
|
+
text = text[fm.end() :]
|
|
1224
|
+
if not text or text[0] not in "\"'`":
|
|
1225
|
+
return None
|
|
1226
|
+
q = text[0]
|
|
1227
|
+
body = text[1:]
|
|
1228
|
+
stop = len(body)
|
|
1229
|
+
for marker in (q, "${") if q == "`" else (q, "$") if self.lang == PHP and q == '"' else (q,):
|
|
1230
|
+
idx = body.find(marker)
|
|
1231
|
+
if idx != -1:
|
|
1232
|
+
stop = min(stop, idx)
|
|
1233
|
+
return body[:stop]
|
|
1234
|
+
|
|
1235
|
+
def _guards(self, kind: str, k: int, hdr: tuple[int, int, int], sink: int, env: dict, atoms: list[Atom]) -> list[Atom]:
|
|
1236
|
+
live = {a.ident for a in atoms if a.sanitizer_for(kind) is None}
|
|
1237
|
+
if not live:
|
|
1238
|
+
return atoms
|
|
1239
|
+
body_lo = self._body(k)[0]
|
|
1240
|
+
for mt in _IF_RE.finditer(self.m, body_lo, sink):
|
|
1241
|
+
p = mt.end()
|
|
1242
|
+
while p < sink and self.m[p] in " \t\r\n":
|
|
1243
|
+
p += 1
|
|
1244
|
+
if self.lang == GO:
|
|
1245
|
+
q = p
|
|
1246
|
+
depth = 0
|
|
1247
|
+
while q < sink:
|
|
1248
|
+
c = self.m[q]
|
|
1249
|
+
if c in "([":
|
|
1250
|
+
depth += 1
|
|
1251
|
+
elif c in ")]":
|
|
1252
|
+
depth -= 1
|
|
1253
|
+
elif c == "{" and depth == 0:
|
|
1254
|
+
break
|
|
1255
|
+
q += 1
|
|
1256
|
+
cond = (p, q)
|
|
1257
|
+
after = q
|
|
1258
|
+
else:
|
|
1259
|
+
if p >= sink or self.m[p] != "(":
|
|
1260
|
+
continue
|
|
1261
|
+
close = self._match_paren(p, _MAX_RHS)
|
|
1262
|
+
cond = (p + 1, close)
|
|
1263
|
+
after = close + 1
|
|
1264
|
+
while after < len(self.m) and self.m[after] in " \t\r\n":
|
|
1265
|
+
after += 1
|
|
1266
|
+
if after < len(self.m) and self.m[after] == "{" and after in self.open_index:
|
|
1267
|
+
bk = self.open_index[after]
|
|
1268
|
+
blk = (self.opens[bk], self.closes[bk])
|
|
1269
|
+
else:
|
|
1270
|
+
blk = (after, self._stmt_end(after, len(self.m)))
|
|
1271
|
+
if blk[1] < sink:
|
|
1272
|
+
if not EARLY_EXIT.search(self.m, blk[0], blk[1] + 1):
|
|
1273
|
+
continue
|
|
1274
|
+
encl = self._innermost(mt.start())
|
|
1275
|
+
if encl >= 0 and not (self.opens[encl] < sink < self.closes[encl]):
|
|
1276
|
+
continue
|
|
1277
|
+
elif not (blk[0] < sink < blk[1]):
|
|
1278
|
+
continue
|
|
1279
|
+
text = self.raw[cond[0] : cond[1]]
|
|
1280
|
+
label = next((lbl for lbl, pat in guard_patterns(kind) if pat.search(text)), None)
|
|
1281
|
+
if label is None:
|
|
1282
|
+
continue
|
|
1283
|
+
bound = {a.ident for a in self._eval(cond[0], cond[1], env)} & live
|
|
1284
|
+
if bound:
|
|
1285
|
+
gl = line_number(self.raw, mt.start())
|
|
1286
|
+
atoms = [a.with_sanitizer((kind,), label, gl, "guard") if a.ident in bound else a for a in atoms]
|
|
1287
|
+
return atoms
|
|
1288
|
+
|
|
1289
|
+
def _receiver(self, end: int) -> tuple[int, int]:
|
|
1290
|
+
"""Region of the receiver expression ending just before ``end``
|
|
1291
|
+
(``url`` in ``url.openStream()``, ``new URL(u)`` in ``new URL(u).openStream()``)."""
|
|
1292
|
+
m = self.m
|
|
1293
|
+
e = end
|
|
1294
|
+
while e > 0 and m[e - 1] in " \t\r\n":
|
|
1295
|
+
e -= 1
|
|
1296
|
+
s = e
|
|
1297
|
+
floor = max(0, e - 300)
|
|
1298
|
+
while s > floor:
|
|
1299
|
+
c = m[s - 1]
|
|
1300
|
+
if c == ")" or c == "]":
|
|
1301
|
+
opener = "(" if c == ")" else "["
|
|
1302
|
+
depth = 0
|
|
1303
|
+
q = s - 1
|
|
1304
|
+
while q >= floor:
|
|
1305
|
+
if m[q] == c:
|
|
1306
|
+
depth += 1
|
|
1307
|
+
elif m[q] == opener:
|
|
1308
|
+
depth -= 1
|
|
1309
|
+
if depth == 0:
|
|
1310
|
+
break
|
|
1311
|
+
q -= 1
|
|
1312
|
+
if q < floor:
|
|
1313
|
+
break
|
|
1314
|
+
s = q
|
|
1315
|
+
elif c.isalnum() or c in "_$.":
|
|
1316
|
+
s -= 1
|
|
1317
|
+
elif c == " " and m[max(floor, s - 4) : s - 1].endswith("new"):
|
|
1318
|
+
s -= 1
|
|
1319
|
+
else:
|
|
1320
|
+
break
|
|
1321
|
+
return s, e
|
|
1322
|
+
|
|
1323
|
+
def _body(self, k: int) -> tuple[int, int]:
|
|
1324
|
+
if k < 0: # PHP top-level script scope
|
|
1325
|
+
return 0, len(self.m)
|
|
1326
|
+
return self.opens[k] + 1, self.closes[k]
|
|
1327
|
+
|
|
1328
|
+
def sink_flow(
|
|
1329
|
+
self,
|
|
1330
|
+
kind: str,
|
|
1331
|
+
paren: int,
|
|
1332
|
+
args_spec: tuple[int, ...] | None,
|
|
1333
|
+
snippet: str,
|
|
1334
|
+
match_start: int,
|
|
1335
|
+
match_end: int,
|
|
1336
|
+
) -> SinkFlow | None:
|
|
1337
|
+
if args_spec == REST:
|
|
1338
|
+
paren = match_start
|
|
1339
|
+
found = self._enclosing_fn(paren)
|
|
1340
|
+
if found is None:
|
|
1341
|
+
if self.lang != PHP:
|
|
1342
|
+
return None
|
|
1343
|
+
# A plain PHP script is its own handler: its top level is the scope.
|
|
1344
|
+
found = (-1, (0, -1, -1))
|
|
1345
|
+
k, hdr = found
|
|
1346
|
+
env = self._env_at(k, hdr, paren)
|
|
1347
|
+
if args_spec == REST:
|
|
1348
|
+
regions = [(match_end, self._stmt_end(match_end, len(self.m)))]
|
|
1349
|
+
elif args_spec == RECEIVER:
|
|
1350
|
+
regions = [self._receiver(match_start)]
|
|
1351
|
+
else:
|
|
1352
|
+
regions = self._split_args(paren + 1, self._match_paren(paren, _MAX_ARGS))
|
|
1353
|
+
if args_spec is not None:
|
|
1354
|
+
regions = [regions[i] for i in args_spec if 0 <= i < len(regions)]
|
|
1355
|
+
atoms: list[Atom] = []
|
|
1356
|
+
for s, e in regions:
|
|
1357
|
+
if _fixed_prefix(kind, self._leading_literal(s, e)):
|
|
1358
|
+
continue
|
|
1359
|
+
atoms += self._eval(s, e, env)
|
|
1360
|
+
atoms = self._guards(kind, k, hdr, paren, env, _dedupe(atoms))
|
|
1361
|
+
line = line_number(self.raw, paren)
|
|
1362
|
+
body_lo, body_hi = self._body(k)
|
|
1363
|
+
span = (line_number(self.raw, hdr[0]), line_number(self.raw, body_hi))
|
|
1364
|
+
return _verdict(kind, atoms, line, snippet, span)
|
|
1365
|
+
|
|
1366
|
+
|
|
1367
|
+
# =============================================================================
|
|
1368
|
+
# Public entry point
|
|
1369
|
+
# =============================================================================
|
|
1370
|
+
|
|
1371
|
+
|
|
1372
|
+
_FLOW_LANGS = frozenset({PY, JS, GO, PHP, JAVA, CSHARP})
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
class FileFlow:
|
|
1376
|
+
"""Lazily-built, per-file flow context. Build one per file and reuse it for
|
|
1377
|
+
every sink in that file — the parse / mask / brace index are cached."""
|
|
1378
|
+
|
|
1379
|
+
def __init__(self, content: str, suffix: str) -> None:
|
|
1380
|
+
self.content = content
|
|
1381
|
+
self.lang = lang_for_suffix(suffix)
|
|
1382
|
+
self._impl: _PyFlow | _TextFlow | None = None
|
|
1383
|
+
# Ruby has no brace-delimited functions to scope: token gate only.
|
|
1384
|
+
self._failed = self.lang is None or self.lang not in _FLOW_LANGS
|
|
1385
|
+
|
|
1386
|
+
def _get(self) -> _PyFlow | _TextFlow | None:
|
|
1387
|
+
if self._failed:
|
|
1388
|
+
return None
|
|
1389
|
+
if self._impl is None:
|
|
1390
|
+
try:
|
|
1391
|
+
self._impl = _PyFlow(self.content) if self.lang == PY else _TextFlow(self.content, self.lang or JS)
|
|
1392
|
+
except (SyntaxError, ValueError, RecursionError, MemoryError):
|
|
1393
|
+
self._failed = True
|
|
1394
|
+
return None
|
|
1395
|
+
return self._impl
|
|
1396
|
+
|
|
1397
|
+
def sink_flow(
|
|
1398
|
+
self,
|
|
1399
|
+
kind: str,
|
|
1400
|
+
paren: int,
|
|
1401
|
+
args_spec: tuple[int, ...] | None,
|
|
1402
|
+
snippet: str = "",
|
|
1403
|
+
match_start: int | None = None,
|
|
1404
|
+
match_end: int | None = None,
|
|
1405
|
+
) -> SinkFlow | None:
|
|
1406
|
+
"""Flow verdict for the sink call whose argument list opens at
|
|
1407
|
+
``paren``; None when the enclosing function can't be determined (the
|
|
1408
|
+
caller then keeps its legacy same-call gate)."""
|
|
1409
|
+
impl = self._get()
|
|
1410
|
+
if impl is None:
|
|
1411
|
+
return None
|
|
1412
|
+
start = paren if match_start is None else match_start
|
|
1413
|
+
end = start if match_end is None else match_end
|
|
1414
|
+
if args_spec != REST and (paren < 0 or paren >= len(self.content) or self.content[paren] != "("):
|
|
1415
|
+
return None
|
|
1416
|
+
try:
|
|
1417
|
+
return impl.sink_flow(kind, paren, args_spec, snippet, start, end)
|
|
1418
|
+
except RecursionError:
|
|
1419
|
+
return None
|