stubby-vex 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stubby/__init__.py +3 -0
- stubby/analyzers/__init__.py +0 -0
- stubby/analyzers/base.py +62 -0
- stubby/analyzers/java.py +326 -0
- stubby/analyzers/java_ast.py +258 -0
- stubby/analyzers/javascript.py +137 -0
- stubby/analyzers/js_ast.py +294 -0
- stubby/analyzers/observations.py +26 -0
- stubby/analyzers/py_ast.py +248 -0
- stubby/analyzers/python.py +272 -0
- stubby/bench.py +164 -0
- stubby/cli.py +154 -0
- stubby/fixes.py +27 -0
- stubby/graph.py +57 -0
- stubby/jars.py +142 -0
- stubby/llm/__init__.py +0 -0
- stubby/llm/base.py +23 -0
- stubby/llm/openai.py +55 -0
- stubby/lockfiles/__init__.py +0 -0
- stubby/lockfiles/java.py +157 -0
- stubby/lockfiles/npm.py +116 -0
- stubby/lockfiles/python.py +165 -0
- stubby/models.py +75 -0
- stubby/osv.py +87 -0
- stubby/pyenv.py +137 -0
- stubby/scan.py +334 -0
- stubby/symbols.py +170 -0
- stubby/vex.py +69 -0
- stubby_vex-0.7.0.dist-info/METADATA +88 -0
- stubby_vex-0.7.0.dist-info/RECORD +32 -0
- stubby_vex-0.7.0.dist-info/WHEEL +4 -0
- stubby_vex-0.7.0.dist-info/entry_points.txt +2 -0
stubby/__init__.py
ADDED
|
File without changes
|
stubby/analyzers/base.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Language adapter interface: one implementation per ecosystem."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Protocol
|
|
8
|
+
|
|
9
|
+
from ..graph import DependencyGraph
|
|
10
|
+
from ..models import Dependency, Evidence
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class UsageReport:
|
|
15
|
+
"""What project code does with its dependencies."""
|
|
16
|
+
|
|
17
|
+
by_package: dict[str, list[Evidence]] = field(default_factory=dict)
|
|
18
|
+
# Code the analysis cannot attribute to a package (dynamic require/import, eval).
|
|
19
|
+
dynamic: list[Evidence] = field(default_factory=list)
|
|
20
|
+
# Method calls on receivers the analysis could not tie to a package, by method
|
|
21
|
+
# name (a few examples each). Only names matching a vulnerable symbol matter.
|
|
22
|
+
unattributed: dict[str, list[Evidence]] = field(default_factory=dict)
|
|
23
|
+
# True when some source could not be read or was too large to analyse.
|
|
24
|
+
incomplete: bool = False
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def remember_unattributed(store: dict[str, list[Evidence]], ev: Evidence, cap: int = 3) -> None:
|
|
28
|
+
examples = store.setdefault(ev.symbol or "", [])
|
|
29
|
+
if len(examples) < cap and ev not in examples:
|
|
30
|
+
examples.append(ev)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class LanguageAdapter(Protocol):
|
|
34
|
+
ecosystem: str
|
|
35
|
+
|
|
36
|
+
def detect(self, project_dir: Path) -> bool:
|
|
37
|
+
"""Return True if this adapter applies to the project."""
|
|
38
|
+
...
|
|
39
|
+
|
|
40
|
+
def load_dependencies(self, project_dir: Path) -> tuple[str, str, list[Dependency]]:
|
|
41
|
+
"""Return ``(project_name, project_version, dependencies)``."""
|
|
42
|
+
...
|
|
43
|
+
|
|
44
|
+
def analyze(self, project_dir: Path) -> UsageReport:
|
|
45
|
+
"""Report imports, call sites and dynamic code, grouped by package name."""
|
|
46
|
+
...
|
|
47
|
+
|
|
48
|
+
def dependency_graph(self, project_dir: Path) -> DependencyGraph | None:
|
|
49
|
+
"""Who requires whom, or ``None`` when the lockfile does not say."""
|
|
50
|
+
...
|
|
51
|
+
|
|
52
|
+
def analyze_package(self, project_dir: Path, location: str) -> UsageReport:
|
|
53
|
+
"""Like ``analyze`` for the installed package at ``location`` (relative to the project)."""
|
|
54
|
+
...
|
|
55
|
+
|
|
56
|
+
def is_installed(self, project_dir: Path, location: str) -> bool:
|
|
57
|
+
"""Whether the package at ``location`` is installed, so its code can be inspected."""
|
|
58
|
+
...
|
|
59
|
+
|
|
60
|
+
def warnings(self, project_dir: Path) -> list[str]:
|
|
61
|
+
"""Things the user should know that limit the analysis."""
|
|
62
|
+
...
|
stubby/analyzers/java.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""Java adapter: Maven (``pom.xml`` + optional dependency tree) and Gradle (``gradle.lockfile``).
|
|
2
|
+
|
|
3
|
+
Source imports name classes, not artifacts, so classes are mapped to artifacts
|
|
4
|
+
through the jars Maven or Gradle downloaded (``~/.m2``, the Gradle cache, a
|
|
5
|
+
``.m2/repository`` in the project or ``--repository``). The same jars give the
|
|
6
|
+
bytecode of dependents, read from the constant pool of their ``.class`` files.
|
|
7
|
+
Without jars only a groupId-prefix guess exists, any class that cannot be
|
|
8
|
+
attributed is reported as unresolved, and nothing can be ruled out.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import itertools
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
from collections import defaultdict
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from ..graph import DependencyGraph
|
|
21
|
+
from ..jars import (
|
|
22
|
+
JDK_PREFIXES,
|
|
23
|
+
REFLECTION,
|
|
24
|
+
JarInfo,
|
|
25
|
+
candidate_repositories,
|
|
26
|
+
dotted,
|
|
27
|
+
locate_jar,
|
|
28
|
+
read_class,
|
|
29
|
+
read_jar_classes,
|
|
30
|
+
read_jar_references,
|
|
31
|
+
)
|
|
32
|
+
from ..lockfiles.java import ECOSYSTEM, TREE_FILES, parse_gradle_lockfile, parse_maven_tree, parse_pom
|
|
33
|
+
from ..models import Dependency, Evidence
|
|
34
|
+
from .base import UsageReport, remember_unattributed
|
|
35
|
+
from .java_ast import analyze_source
|
|
36
|
+
|
|
37
|
+
SKIP_DIRS = {"target", "build", ".gradle", ".git", "out", "node_modules", ".idea", ".mvn", "bin"}
|
|
38
|
+
MAX_FILE_BYTES = 2_000_000
|
|
39
|
+
_PACKAGE = re.compile(r"^\s*package\s+([\w.]+)\s*;", re.MULTILINE)
|
|
40
|
+
_IMPORT_FALLBACK = re.compile(r"^\s*import\s+(?:static\s+)?([\w.]+?)(?:\.\*)?\s*;", re.MULTILINE)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class _State:
|
|
45
|
+
name: str
|
|
46
|
+
version: str
|
|
47
|
+
deps: list[Dependency]
|
|
48
|
+
graph: DependencyGraph | None
|
|
49
|
+
repositories: list[Path]
|
|
50
|
+
warnings: list[str] = field(default_factory=list)
|
|
51
|
+
_jars: dict[str, JarInfo | None] = field(default_factory=dict)
|
|
52
|
+
_by_class: dict[str, set[str]] | None = None
|
|
53
|
+
_by_package: dict[str, set[str]] | None = None
|
|
54
|
+
|
|
55
|
+
def jar(self, key: str) -> JarInfo | None:
|
|
56
|
+
"""The jar of dependency ``key`` (``group:artifact``), with its class list."""
|
|
57
|
+
if key not in self._jars:
|
|
58
|
+
dep = next((d for d in self.deps if d.name == key), None)
|
|
59
|
+
path = None
|
|
60
|
+
if dep is not None:
|
|
61
|
+
group, artifact = dep.name.split(":", 1)
|
|
62
|
+
path = locate_jar(self.repositories, group, artifact, dep.version)
|
|
63
|
+
self._jars[key] = read_jar_classes(path) if path else None
|
|
64
|
+
return self._jars[key]
|
|
65
|
+
|
|
66
|
+
def _build_index(self) -> None:
|
|
67
|
+
self._by_class, self._by_package = defaultdict(set), defaultdict(set)
|
|
68
|
+
for dep in self.deps:
|
|
69
|
+
info = self.jar(dep.name)
|
|
70
|
+
if info is None:
|
|
71
|
+
continue
|
|
72
|
+
for cls in info.classes:
|
|
73
|
+
self._by_class[cls].add(dep.name)
|
|
74
|
+
for pkg in info.packages:
|
|
75
|
+
self._by_package[pkg].add(dep.name)
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def by_class(self) -> dict[str, set[str]]:
|
|
79
|
+
if self._by_class is None:
|
|
80
|
+
self._build_index()
|
|
81
|
+
return self._by_class # type: ignore[return-value]
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def by_package(self) -> dict[str, set[str]]:
|
|
85
|
+
if self._by_package is None:
|
|
86
|
+
self._build_index()
|
|
87
|
+
return self._by_package # type: ignore[return-value]
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def any_jar(self) -> bool:
|
|
91
|
+
return any(self.jar(d.name) is not None for d in self.deps)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class JavaAdapter:
|
|
95
|
+
ecosystem = ECOSYSTEM
|
|
96
|
+
|
|
97
|
+
def __init__(self, repository: Path | None = None) -> None:
|
|
98
|
+
self.repository = repository
|
|
99
|
+
self._cache: dict[Path, _State] = {}
|
|
100
|
+
|
|
101
|
+
def configured(self, site_packages: Path | None = None, repository: Path | None = None) -> "JavaAdapter":
|
|
102
|
+
return JavaAdapter(repository) if repository else self
|
|
103
|
+
|
|
104
|
+
def detect(self, project_dir: Path) -> bool:
|
|
105
|
+
return (project_dir / "pom.xml").is_file() or (project_dir / "gradle.lockfile").is_file()
|
|
106
|
+
|
|
107
|
+
# -- dependencies ------------------------------------------------------
|
|
108
|
+
|
|
109
|
+
def _state(self, project_dir: Path) -> _State:
|
|
110
|
+
key = project_dir.resolve()
|
|
111
|
+
if key not in self._cache:
|
|
112
|
+
self._cache[key] = self._build_state(key)
|
|
113
|
+
return self._cache[key]
|
|
114
|
+
|
|
115
|
+
def _build_state(self, project_dir: Path) -> _State:
|
|
116
|
+
warnings: list[str] = []
|
|
117
|
+
name, version = project_dir.name, "0.0.0"
|
|
118
|
+
pom_path = project_dir / "pom.xml"
|
|
119
|
+
pom = parse_pom(pom_path) if pom_path.is_file() else None
|
|
120
|
+
if pom and pom.artifact:
|
|
121
|
+
name = f"{pom.group}:{pom.artifact}" if pom.group else pom.artifact
|
|
122
|
+
version = pom.version or version
|
|
123
|
+
|
|
124
|
+
graph = None
|
|
125
|
+
tree = next((project_dir / t for t in TREE_FILES if (project_dir / t).is_file()), None)
|
|
126
|
+
gradle = project_dir / "gradle.lockfile"
|
|
127
|
+
if tree is not None:
|
|
128
|
+
parsed = parse_maven_tree(tree)
|
|
129
|
+
deps, graph = parsed.deps, parsed.graph
|
|
130
|
+
if pom is None and parsed.root:
|
|
131
|
+
name, version = f"{parsed.root[0]}:{parsed.root[1]}", parsed.root[2]
|
|
132
|
+
elif pom is not None:
|
|
133
|
+
deps = [Dependency(ECOSYSTEM, f"{g}:{a}", v, direct=True) for g, a, v, _ in pom.deps]
|
|
134
|
+
warnings += pom.warnings
|
|
135
|
+
warnings.append(
|
|
136
|
+
"pom.xml lists direct dependencies only; run "
|
|
137
|
+
"`mvn dependency:tree -DoutputFile=dependency-tree.txt` to include transitive ones"
|
|
138
|
+
)
|
|
139
|
+
else:
|
|
140
|
+
build = "".join(
|
|
141
|
+
p.read_text(encoding="utf-8", errors="replace")
|
|
142
|
+
for p in (project_dir / "build.gradle", project_dir / "build.gradle.kts")
|
|
143
|
+
if p.is_file()
|
|
144
|
+
)
|
|
145
|
+
deps = parse_gradle_lockfile(gradle, build)
|
|
146
|
+
|
|
147
|
+
repositories = candidate_repositories(project_dir, self.repository)
|
|
148
|
+
state = _State(name, version, deps, graph, repositories, warnings)
|
|
149
|
+
if deps and not state.any_jar:
|
|
150
|
+
warnings.append(
|
|
151
|
+
"no downloaded jars found (~/.m2, the Gradle cache, .m2/repository in the project or "
|
|
152
|
+
"--repository): classes cannot be mapped to artifacts and dependents cannot be "
|
|
153
|
+
"inspected, so nothing can be ruled out"
|
|
154
|
+
)
|
|
155
|
+
if any(True for _ in _iter_files(project_dir, ".kt")):
|
|
156
|
+
warnings.append("Kotlin sources are not analysed")
|
|
157
|
+
return state
|
|
158
|
+
|
|
159
|
+
def load_dependencies(self, project_dir: Path) -> tuple[str, str, list[Dependency]]:
|
|
160
|
+
state = self._state(project_dir)
|
|
161
|
+
return state.name, state.version, state.deps
|
|
162
|
+
|
|
163
|
+
def dependency_graph(self, project_dir: Path) -> DependencyGraph | None:
|
|
164
|
+
return self._state(project_dir).graph
|
|
165
|
+
|
|
166
|
+
def warnings(self, project_dir: Path) -> list[str]:
|
|
167
|
+
return list(self._state(project_dir).warnings)
|
|
168
|
+
|
|
169
|
+
def is_installed(self, project_dir: Path, location: str) -> bool:
|
|
170
|
+
return self._state(project_dir).jar(location) is not None
|
|
171
|
+
|
|
172
|
+
# -- analysis ----------------------------------------------------------
|
|
173
|
+
|
|
174
|
+
def analyze(self, project_dir: Path) -> UsageReport:
|
|
175
|
+
project_dir = project_dir.resolve()
|
|
176
|
+
state = self._state(project_dir)
|
|
177
|
+
by_package: dict[str, set[Evidence]] = defaultdict(set)
|
|
178
|
+
dynamic: set[Evidence] = set()
|
|
179
|
+
unattributed: dict[str, list[Evidence]] = {}
|
|
180
|
+
incomplete = False
|
|
181
|
+
|
|
182
|
+
sources: list[tuple[Path, bytes]] = []
|
|
183
|
+
local_packages: set[str] = set()
|
|
184
|
+
for path in _iter_files(project_dir, ".java"):
|
|
185
|
+
try:
|
|
186
|
+
if path.stat().st_size > MAX_FILE_BYTES:
|
|
187
|
+
incomplete = True
|
|
188
|
+
continue
|
|
189
|
+
data = path.read_bytes()
|
|
190
|
+
except OSError:
|
|
191
|
+
incomplete = True
|
|
192
|
+
continue
|
|
193
|
+
sources.append((path, data))
|
|
194
|
+
local_packages.update(_PACKAGE.findall(data.decode("utf-8", errors="replace")))
|
|
195
|
+
|
|
196
|
+
for path, data in sources:
|
|
197
|
+
rel = path.relative_to(project_dir).as_posix()
|
|
198
|
+
text = data.decode("utf-8", errors="replace")
|
|
199
|
+
lines = text.splitlines()
|
|
200
|
+
|
|
201
|
+
def snippet(lineno: int) -> str:
|
|
202
|
+
return lines[lineno - 1].strip()[:200] if 0 < lineno <= len(lines) else ""
|
|
203
|
+
|
|
204
|
+
analysis = analyze_source(data)
|
|
205
|
+
observations = [(o.package, o.line, o.kind, o.symbol, o.module) for o in analysis.observations]
|
|
206
|
+
if analysis.has_syntax_errors:
|
|
207
|
+
seen = {(p, line) for p, line, *_ in observations}
|
|
208
|
+
for match in _IMPORT_FALLBACK.finditer(text):
|
|
209
|
+
line = text.count("\n", 0, match.start(1)) + 1
|
|
210
|
+
if (match.group(1), line) not in seen:
|
|
211
|
+
observations.append((match.group(1), line, "import", None, match.group(1)))
|
|
212
|
+
|
|
213
|
+
for fqn, line, kind, symbol, module in observations:
|
|
214
|
+
ev = Evidence(rel, line, snippet(line), kind, symbol, module)
|
|
215
|
+
if kind == "maybe":
|
|
216
|
+
remember_unattributed(unattributed, ev)
|
|
217
|
+
elif fqn is None:
|
|
218
|
+
dynamic.add(ev)
|
|
219
|
+
else:
|
|
220
|
+
artifacts = _resolve(fqn, state, local_packages)
|
|
221
|
+
if artifacts is None:
|
|
222
|
+
dynamic.add(Evidence(rel, line, snippet(line), "dynamic", f"unresolved class {fqn}"))
|
|
223
|
+
for artifact in artifacts or []:
|
|
224
|
+
by_package[artifact].add(ev)
|
|
225
|
+
|
|
226
|
+
# A dependency without a jar cannot be told apart from its neighbours by class name.
|
|
227
|
+
incomplete = incomplete or any(state.jar(d.name) is None for d in state.deps)
|
|
228
|
+
return UsageReport(
|
|
229
|
+
by_package={a: _ordered(e) for a, e in by_package.items()},
|
|
230
|
+
dynamic=_ordered(dynamic),
|
|
231
|
+
unattributed=unattributed,
|
|
232
|
+
incomplete=incomplete,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
def analyze_package(self, project_dir: Path, location: str) -> UsageReport:
|
|
236
|
+
state = self._state(project_dir.resolve())
|
|
237
|
+
info = state.jar(location)
|
|
238
|
+
if info is None:
|
|
239
|
+
return UsageReport(incomplete=True)
|
|
240
|
+
classes, skipped = read_jar_references(info.path)
|
|
241
|
+
own = {c.name for c in classes}
|
|
242
|
+
own_packages = info.packages
|
|
243
|
+
by_package: dict[str, set[Evidence]] = defaultdict(set)
|
|
244
|
+
dynamic: set[Evidence] = set()
|
|
245
|
+
|
|
246
|
+
def attribute(owner: str) -> list[str] | None:
|
|
247
|
+
"""Artifacts for an internal class name; ``[]`` for the jar itself and the JDK."""
|
|
248
|
+
if owner.startswith("[") or owner in own:
|
|
249
|
+
return []
|
|
250
|
+
name = dotted(owner)
|
|
251
|
+
if name.startswith(JDK_PREFIXES) or name.rsplit(".", 1)[0] in own_packages:
|
|
252
|
+
return []
|
|
253
|
+
return _resolve(name, state, set())
|
|
254
|
+
|
|
255
|
+
for cls in classes:
|
|
256
|
+
where = f"{info.path.name}!/{cls.name}.class"
|
|
257
|
+
for owner, method in cls.methods:
|
|
258
|
+
if (owner, method) in REFLECTION:
|
|
259
|
+
dynamic.add(Evidence(where, 0, f"{owner}.{method}", "dynamic", method))
|
|
260
|
+
artifacts = attribute(owner)
|
|
261
|
+
if artifacts is None:
|
|
262
|
+
dynamic.add(Evidence(where, 0, owner, "dynamic", f"unresolved class {dotted(owner)}"))
|
|
263
|
+
continue
|
|
264
|
+
simple = dotted(owner).rsplit(".", 1)[-1]
|
|
265
|
+
symbol = simple if method == "<init>" else method
|
|
266
|
+
for artifact in artifacts:
|
|
267
|
+
by_package[artifact].add(Evidence(where, 0, f"{owner}.{method}", "call", symbol, dotted(owner)))
|
|
268
|
+
for owner in cls.classes:
|
|
269
|
+
for artifact in attribute(owner) or []:
|
|
270
|
+
simple = dotted(owner).rsplit(".", 1)[-1]
|
|
271
|
+
by_package[artifact].add(Evidence(where, 0, owner, "ref", simple, dotted(owner)))
|
|
272
|
+
|
|
273
|
+
return UsageReport(
|
|
274
|
+
by_package={a: _ordered(e) for a, e in by_package.items()},
|
|
275
|
+
dynamic=_ordered(dynamic),
|
|
276
|
+
incomplete=skipped,
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _class_candidates(fqn: str):
|
|
281
|
+
"""``a.b.Outer.Inner`` may be the class ``a.b.Outer$Inner`` in a jar."""
|
|
282
|
+
yield fqn
|
|
283
|
+
parts = fqn.split(".")
|
|
284
|
+
for n in range(1, min(3, len(parts) - 1) + 1):
|
|
285
|
+
yield ".".join(parts[:-n]) + "$" + "$".join(parts[-n:])
|
|
286
|
+
yield ".".join(parts[:-n])
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _resolve(fqn: str, state: _State, local_packages: set[str]) -> list[str] | None:
|
|
290
|
+
"""Artifacts providing ``fqn`` (a class or, for wildcard imports, a package).
|
|
291
|
+
|
|
292
|
+
``[]`` means ignore it (JDK or project code); ``None`` means it cannot be attributed.
|
|
293
|
+
"""
|
|
294
|
+
if fqn.startswith(JDK_PREFIXES):
|
|
295
|
+
return []
|
|
296
|
+
by_class, by_package = state.by_class, state.by_package
|
|
297
|
+
for candidate in _class_candidates(fqn):
|
|
298
|
+
if candidate in by_class: # a jar wins over a same-named project package: over-reporting is safe
|
|
299
|
+
return sorted(by_class[candidate])
|
|
300
|
+
parts = fqn.split(".")
|
|
301
|
+
for i in range(len(parts), 0, -1):
|
|
302
|
+
pkg = ".".join(parts[:i])
|
|
303
|
+
if pkg in by_package:
|
|
304
|
+
return sorted(by_package[pkg])
|
|
305
|
+
package = ".".join(itertools.takewhile(lambda part: not part[:1].isupper(), parts))
|
|
306
|
+
if package in local_packages:
|
|
307
|
+
return []
|
|
308
|
+
# No jar knows it: guess from the groupId, the longest matching one.
|
|
309
|
+
matches: dict[int, list[str]] = defaultdict(list)
|
|
310
|
+
for dep in state.deps:
|
|
311
|
+
group = dep.name.split(":", 1)[0]
|
|
312
|
+
if fqn == group or fqn.startswith(group + "."):
|
|
313
|
+
matches[len(group)].append(dep.name)
|
|
314
|
+
return sorted(matches[max(matches)]) if matches else None
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _ordered(evidence) -> list[Evidence]:
|
|
318
|
+
return sorted(evidence, key=lambda e: (e.file, e.line, e.kind, e.symbol or ""))
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _iter_files(root: Path, suffix: str):
|
|
322
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
323
|
+
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
|
|
324
|
+
for name in filenames:
|
|
325
|
+
if name.endswith(suffix):
|
|
326
|
+
yield Path(dirpath) / name
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""tree-sitter based analysis of a single Java source file.
|
|
2
|
+
|
|
3
|
+
Java has static types, so a call such as ``yaml.load(x)`` is tied to the class of
|
|
4
|
+
``yaml`` through its declaration (local, field, parameter, ``new``). Receivers that
|
|
5
|
+
cannot be typed this way are reported by method name only (kind ``maybe``).
|
|
6
|
+
Observations use the fully qualified class name as ``package``; the adapter maps
|
|
7
|
+
it to the artifact that contains it. Scoping is ignored on purpose, which can only
|
|
8
|
+
over-report.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
|
|
15
|
+
import tree_sitter_java
|
|
16
|
+
from tree_sitter import Language, Node, Parser
|
|
17
|
+
|
|
18
|
+
from .observations import FileAnalysis, Observation
|
|
19
|
+
|
|
20
|
+
_PARSER = Parser(Language(tree_sitter_java.language()))
|
|
21
|
+
_FQN = re.compile(r"\b((?:[a-z_][\w]*\.){2,}[A-Z][\w$]*)")
|
|
22
|
+
_LINE_SKIP = re.compile(r"^\s*(?:import|package)\s")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def analyze_source(source: bytes) -> FileAnalysis:
|
|
26
|
+
root = _PARSER.parse(source).root_node
|
|
27
|
+
analysis = FileAnalysis(has_syntax_errors=root.has_error)
|
|
28
|
+
nodes = list(_walk(root))
|
|
29
|
+
classes: dict[str, str] = {} # simple name -> fully qualified (imports)
|
|
30
|
+
statics: dict[str, str] = {} # statically imported member -> class
|
|
31
|
+
|
|
32
|
+
for node in nodes:
|
|
33
|
+
if node.type == "import_declaration":
|
|
34
|
+
_handle_import(node, analysis, classes, statics)
|
|
35
|
+
|
|
36
|
+
var_types: dict[str, set[str]] = {}
|
|
37
|
+
for node in nodes:
|
|
38
|
+
_collect_declaration(node, var_types)
|
|
39
|
+
|
|
40
|
+
for node in nodes:
|
|
41
|
+
if node.type == "type_identifier" and not _in_import(node):
|
|
42
|
+
_handle_type(node, analysis, classes)
|
|
43
|
+
elif node.type == "identifier" and not _in_import(node):
|
|
44
|
+
_handle_identifier(node, analysis, classes)
|
|
45
|
+
elif node.type == "method_invocation":
|
|
46
|
+
_handle_invocation(node, analysis, classes, statics, var_types)
|
|
47
|
+
|
|
48
|
+
_fqn_fallback(source.decode("utf-8", errors="replace"), analysis)
|
|
49
|
+
return analysis
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _walk(root: Node):
|
|
53
|
+
stack = [root]
|
|
54
|
+
while stack:
|
|
55
|
+
node = stack.pop()
|
|
56
|
+
yield node
|
|
57
|
+
stack.extend(reversed(node.children))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _text(node: Node) -> str:
|
|
61
|
+
return node.text.decode("utf-8", errors="replace")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _line(node: Node) -> int:
|
|
65
|
+
return node.start_point[0] + 1
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _in_import(node: Node) -> bool:
|
|
69
|
+
parent = node.parent
|
|
70
|
+
while parent is not None:
|
|
71
|
+
if parent.type in ("import_declaration", "package_declaration"):
|
|
72
|
+
return True
|
|
73
|
+
if parent.type in ("class_body", "block"):
|
|
74
|
+
return False
|
|
75
|
+
parent = parent.parent
|
|
76
|
+
return False
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _handle_import(node: Node, analysis: FileAnalysis, classes: dict, statics: dict) -> None:
|
|
80
|
+
ident = next((c for c in node.named_children if c.type in ("scoped_identifier", "identifier")), None)
|
|
81
|
+
if ident is None:
|
|
82
|
+
return
|
|
83
|
+
full = _text(ident)
|
|
84
|
+
is_static = any(c.type == "static" for c in node.children)
|
|
85
|
+
wildcard = any(c.type == "asterisk" for c in node.children)
|
|
86
|
+
line = _line(node)
|
|
87
|
+
if wildcard:
|
|
88
|
+
analysis.observations.append(Observation(full, line, "escape", "*", full + ".*"))
|
|
89
|
+
elif is_static:
|
|
90
|
+
owner, _, member = full.rpartition(".")
|
|
91
|
+
statics[member] = owner
|
|
92
|
+
analysis.observations.append(Observation(owner, line, "import", member, owner))
|
|
93
|
+
else:
|
|
94
|
+
simple = full.rsplit(".", 1)[-1]
|
|
95
|
+
classes[simple] = full
|
|
96
|
+
analysis.observations.append(Observation(full, line, "import", simple, full))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _type_names(node: Node | None) -> set[str]:
|
|
100
|
+
"""Simple (or qualified) class names a declared type refers to, ignoring generics."""
|
|
101
|
+
if node is None:
|
|
102
|
+
return set()
|
|
103
|
+
if node.type == "type_identifier":
|
|
104
|
+
return {_text(node)}
|
|
105
|
+
if node.type == "scoped_type_identifier":
|
|
106
|
+
return {_text(node)}
|
|
107
|
+
if node.type == "generic_type":
|
|
108
|
+
first = next((c for c in node.named_children if c.type in ("type_identifier", "scoped_type_identifier")), None)
|
|
109
|
+
return _type_names(first)
|
|
110
|
+
if node.type == "array_type":
|
|
111
|
+
return _type_names(node.child_by_field_name("element"))
|
|
112
|
+
if node.type == "catch_type":
|
|
113
|
+
return {n for c in node.named_children for n in _type_names(c)}
|
|
114
|
+
return set()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _collect_declaration(node: Node, var_types: dict[str, set[str]]) -> None:
|
|
118
|
+
kind = node.type
|
|
119
|
+
if kind in ("local_variable_declaration", "field_declaration"):
|
|
120
|
+
type_node = node.child_by_field_name("type")
|
|
121
|
+
for decl in node.named_children:
|
|
122
|
+
if decl.type != "variable_declarator":
|
|
123
|
+
continue
|
|
124
|
+
names = _type_names(type_node)
|
|
125
|
+
value = decl.child_by_field_name("value")
|
|
126
|
+
if names == {"var"} and value is not None and value.type == "object_creation_expression":
|
|
127
|
+
names = _type_names(value.child_by_field_name("type"))
|
|
128
|
+
var_types.setdefault(_text(decl.child_by_field_name("name")), set()).update(names)
|
|
129
|
+
elif kind in ("formal_parameter", "enhanced_for_statement", "resource"):
|
|
130
|
+
name = node.child_by_field_name("name")
|
|
131
|
+
if name is not None:
|
|
132
|
+
var_types.setdefault(_text(name), set()).update(_type_names(node.child_by_field_name("type")))
|
|
133
|
+
elif kind == "catch_formal_parameter":
|
|
134
|
+
name = node.child_by_field_name("name")
|
|
135
|
+
catch_type = next((c for c in node.named_children if c.type == "catch_type"), None)
|
|
136
|
+
if name is not None:
|
|
137
|
+
var_types.setdefault(_text(name), set()).update(_type_names(catch_type))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _handle_type(node: Node, analysis: FileAnalysis, classes: dict) -> None:
|
|
141
|
+
name = _text(node)
|
|
142
|
+
if name not in classes:
|
|
143
|
+
return
|
|
144
|
+
parent = node.parent
|
|
145
|
+
is_new = parent is not None and parent.type == "object_creation_expression" and (
|
|
146
|
+
parent.child_by_field_name("type") == node
|
|
147
|
+
)
|
|
148
|
+
analysis.observations.append(
|
|
149
|
+
Observation(classes[name], _line(node), "call" if is_new else "ref", name, classes[name])
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _handle_identifier(node: Node, analysis: FileAnalysis, classes: dict) -> None:
|
|
154
|
+
name = _text(node)
|
|
155
|
+
if name not in classes:
|
|
156
|
+
return
|
|
157
|
+
parent = node.parent
|
|
158
|
+
if parent is None:
|
|
159
|
+
return
|
|
160
|
+
uses_class = (
|
|
161
|
+
(parent.type in ("method_invocation", "field_access") and parent.child_by_field_name("object") == node)
|
|
162
|
+
or (parent.type in ("annotation", "marker_annotation") and parent.child_by_field_name("name") == node)
|
|
163
|
+
or parent.type == "method_reference"
|
|
164
|
+
)
|
|
165
|
+
if not uses_class:
|
|
166
|
+
return
|
|
167
|
+
analysis.observations.append(Observation(classes[name], _line(node), "ref", name, classes[name]))
|
|
168
|
+
if parent.type == "field_access": # constant or nested class: Level.INFO
|
|
169
|
+
member = parent.child_by_field_name("field")
|
|
170
|
+
if member is not None:
|
|
171
|
+
analysis.observations.append(Observation(classes[name], _line(node), "ref", _text(member), classes[name]))
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _resolve(name: str, classes: dict) -> str | None:
|
|
175
|
+
if name in classes:
|
|
176
|
+
return classes[name]
|
|
177
|
+
return name if "." in name else None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _handle_invocation(node: Node, analysis: FileAnalysis, classes: dict, statics: dict, var_types: dict) -> None:
|
|
181
|
+
name_node = node.child_by_field_name("name")
|
|
182
|
+
if name_node is None:
|
|
183
|
+
return
|
|
184
|
+
method = _text(name_node)
|
|
185
|
+
obj = node.child_by_field_name("object")
|
|
186
|
+
line = _line(node)
|
|
187
|
+
|
|
188
|
+
def call(fqn: str) -> None:
|
|
189
|
+
analysis.observations.append(Observation(fqn, line, "call", method, fqn))
|
|
190
|
+
|
|
191
|
+
def maybe() -> None:
|
|
192
|
+
analysis.observations.append(Observation(None, line, "maybe", method))
|
|
193
|
+
|
|
194
|
+
_handle_dynamic(node, method, obj, analysis, line)
|
|
195
|
+
|
|
196
|
+
if obj is None:
|
|
197
|
+
if method in statics:
|
|
198
|
+
call(statics[method])
|
|
199
|
+
return
|
|
200
|
+
if obj.type == "this":
|
|
201
|
+
return
|
|
202
|
+
if obj.type == "identifier":
|
|
203
|
+
text = _text(obj)
|
|
204
|
+
if text in classes:
|
|
205
|
+
call(classes[text])
|
|
206
|
+
elif text in var_types:
|
|
207
|
+
for tname in var_types[text]:
|
|
208
|
+
fqn = _resolve(tname, classes)
|
|
209
|
+
if fqn:
|
|
210
|
+
call(fqn)
|
|
211
|
+
if "var" in var_types[text]: # inferred from something other than `new`
|
|
212
|
+
maybe()
|
|
213
|
+
else:
|
|
214
|
+
maybe()
|
|
215
|
+
return
|
|
216
|
+
if obj.type == "field_access":
|
|
217
|
+
target = obj.child_by_field_name("object")
|
|
218
|
+
field = obj.child_by_field_name("field")
|
|
219
|
+
if target is not None and target.type == "this" and field is not None and _text(field) in var_types:
|
|
220
|
+
for tname in var_types[_text(field)]:
|
|
221
|
+
fqn = _resolve(tname, classes)
|
|
222
|
+
if fqn:
|
|
223
|
+
call(fqn)
|
|
224
|
+
return
|
|
225
|
+
maybe()
|
|
226
|
+
return
|
|
227
|
+
if obj.type == "object_creation_expression":
|
|
228
|
+
for tname in _type_names(obj.child_by_field_name("type")):
|
|
229
|
+
fqn = _resolve(tname, classes)
|
|
230
|
+
if fqn:
|
|
231
|
+
call(fqn)
|
|
232
|
+
return
|
|
233
|
+
maybe()
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _handle_dynamic(node: Node, method: str, obj: Node | None, analysis: FileAnalysis, line: int) -> None:
|
|
237
|
+
receiver = _text(obj) if obj is not None and obj.type == "identifier" else ""
|
|
238
|
+
args = node.child_by_field_name("arguments")
|
|
239
|
+
first = args.named_children[0] if args is not None and args.named_children else None
|
|
240
|
+
if method == "forName" and receiver == "Class":
|
|
241
|
+
if first is not None and first.type == "string_literal":
|
|
242
|
+
analysis.observations.append(Observation(_text(first)[1:-1], line, "escape", "*", _text(first)[1:-1]))
|
|
243
|
+
else:
|
|
244
|
+
analysis.observations.append(Observation(None, line, "dynamic", "Class.forName"))
|
|
245
|
+
elif method == "loadClass" or (method == "load" and receiver == "ServiceLoader"):
|
|
246
|
+
analysis.observations.append(Observation(None, line, "dynamic", method))
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _fqn_fallback(text: str, analysis: FileAnalysis) -> None:
|
|
250
|
+
"""Fully qualified names written inline (code, reflection strings) count as class use."""
|
|
251
|
+
seen = {(o.package, o.line) for o in analysis.observations}
|
|
252
|
+
for lineno, line in enumerate(text.splitlines(), start=1):
|
|
253
|
+
if _LINE_SKIP.match(line):
|
|
254
|
+
continue
|
|
255
|
+
for match in _FQN.finditer(line):
|
|
256
|
+
fqn = match.group(1)
|
|
257
|
+
if (fqn, lineno) not in seen:
|
|
258
|
+
analysis.observations.append(Observation(fqn, lineno, "import", fqn.rsplit(".", 1)[-1], fqn))
|