codeanalyzer-python 1.3.0__py3-none-any.whl → 1.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +7 -13
- codeanalyzer/artifacts/dependencies.py +3 -3
- codeanalyzer/artifacts/discovery.py +8 -34
- codeanalyzer/core.py +3 -4
- codeanalyzer/dataflow/builder.py +10 -2
- codeanalyzer/dataflow/identity.py +3 -3
- codeanalyzer/dataflow/sdg.py +17 -2
- codeanalyzer/dataflow/summaries.py +27 -4
- codeanalyzer/entrypoints/matching.py +36 -7
- codeanalyzer/entrypoints/pipeline.py +52 -4
- codeanalyzer/entrypoints/rules.py +12 -1
- codeanalyzer/entrypoints/rules.yml +34 -0
- codeanalyzer/neo4j/bolt.py +101 -45
- codeanalyzer/neo4j/cypher.py +10 -4
- codeanalyzer/neo4j/emit.py +3 -2
- codeanalyzer/neo4j/project.py +97 -42
- codeanalyzer/neo4j/rows.py +36 -1
- codeanalyzer/neo4j/schema.py +10 -8
- codeanalyzer/options/options.py +2 -3
- codeanalyzer/schema/ids.py +17 -0
- codeanalyzer/schema/l1_body.py +2 -0
- codeanalyzer/schema/py_schema.py +8 -3
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/METADATA +33 -18
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/RECORD +28 -28
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/__main__.py
CHANGED
|
@@ -175,7 +175,10 @@ def main(
|
|
|
175
175
|
bool,
|
|
176
176
|
typer.Option(
|
|
177
177
|
"--eager/--lazy",
|
|
178
|
-
help="Enable eager or lazy analysis. Defaults to lazy."
|
|
178
|
+
help="Enable eager or lazy analysis. Defaults to lazy. Also gates every "
|
|
179
|
+
"destructive step of a '--emit neo4j' Bolt push: a lazy push only adds and "
|
|
180
|
+
"updates, an eager one also removes declarations and edges the source no "
|
|
181
|
+
"longer has.",
|
|
179
182
|
),
|
|
180
183
|
] = False,
|
|
181
184
|
skip_tests: Annotated[
|
|
@@ -239,19 +242,11 @@ def main(
|
|
|
239
242
|
typer.Option(
|
|
240
243
|
"--artifact-text/--no-artifact-text",
|
|
241
244
|
help="Capture verbatim `source` text on discovered artifacts. "
|
|
242
|
-
"--no-artifact-text empties
|
|
245
|
+
"`source` is the whole file; --no-artifact-text empties it "
|
|
246
|
+
"everywhere (inventory unchanged). sha256/size_bytes always "
|
|
247
|
+
"reflect the full file.",
|
|
243
248
|
),
|
|
244
249
|
] = True,
|
|
245
|
-
artifact_text_max_bytes: Annotated[
|
|
246
|
-
int,
|
|
247
|
-
typer.Option(
|
|
248
|
-
"--artifact-text-max-bytes",
|
|
249
|
-
help="Per-file byte cap on captured artifact `source`; a decodable "
|
|
250
|
-
"file over the cap is truncated (text_truncated=True). "
|
|
251
|
-
"sha256/size_bytes always reflect the full file.",
|
|
252
|
-
min=1,
|
|
253
|
-
),
|
|
254
|
-
] = 262144,
|
|
255
250
|
):
|
|
256
251
|
# Determinism: pin the interpreter hash seed before any analysis (no-op
|
|
257
252
|
# when PYTHONHASHSEED is already set; --version exits before this).
|
|
@@ -336,7 +331,6 @@ def main(
|
|
|
336
331
|
verbosity=verbosity,
|
|
337
332
|
entrypoint_rules=tuple(entrypoint_rules or ()),
|
|
338
333
|
artifact_text=artifact_text,
|
|
339
|
-
artifact_text_max_bytes=artifact_text_max_bytes,
|
|
340
334
|
)
|
|
341
335
|
|
|
342
336
|
_set_log_level(options.verbosity)
|
|
@@ -81,9 +81,9 @@ def _resolve_ref(manifest_path: str, ref: str) -> Optional[str]:
|
|
|
81
81
|
|
|
82
82
|
def _full_text(project_dir: Path, path: str, art: PyArtifact) -> str:
|
|
83
83
|
"""Manifest/lock extraction must never depend on the stored ``source`` --
|
|
84
|
-
that's
|
|
85
|
-
|
|
86
|
-
|
|
84
|
+
that's emptied by ``capture_text=False`` (a payload-size control on the
|
|
85
|
+
JSON/Neo4j payload, not an extraction control). Read the real file fresh
|
|
86
|
+
instead; fall back to ``art.source``
|
|
87
87
|
only if it is gone (e.g. a synthetic artifact in a unit test, or the file
|
|
88
88
|
vanished mid-run).
|
|
89
89
|
|
|
@@ -76,27 +76,11 @@ def _classify(rel_posix: str) -> Tuple[str, List[str]] | None:
|
|
|
76
76
|
return None
|
|
77
77
|
|
|
78
78
|
|
|
79
|
-
def _capture_source(
|
|
80
|
-
raw: bytes, text: str, capture_text: bool, text_max_bytes: int
|
|
81
|
-
) -> Tuple[str, bool]:
|
|
82
|
-
"""Decide ``(source, text_truncated)`` for a decodable file.
|
|
83
|
-
|
|
84
|
-
Slices ``raw`` (not ``text``) for the cap, so it is a true byte cap even
|
|
85
|
-
when it lands inside a multi-byte character -- ``errors="ignore"`` drops
|
|
86
|
-
the dangling partial char at the cut, so this never raises."""
|
|
87
|
-
if not capture_text:
|
|
88
|
-
return "", False
|
|
89
|
-
if len(raw) <= text_max_bytes:
|
|
90
|
-
return text, False
|
|
91
|
-
return raw[:text_max_bytes].decode("utf-8", errors="ignore"), True
|
|
92
|
-
|
|
93
|
-
|
|
94
79
|
def discover_artifacts(
|
|
95
80
|
project_dir: Path,
|
|
96
81
|
app_name: str,
|
|
97
82
|
*,
|
|
98
83
|
capture_text: bool = True,
|
|
99
|
-
text_max_bytes: int = 262144,
|
|
100
84
|
) -> Dict[str, PyArtifact]:
|
|
101
85
|
"""Walk the project and return every file as an artifact, sorted by path.
|
|
102
86
|
|
|
@@ -109,15 +93,11 @@ def discover_artifacts(
|
|
|
109
93
|
deliberate exception -- it IS rule-matched (a dependency-manifest), so it
|
|
110
94
|
is captured like any other manifest despite the `.py` suffix.
|
|
111
95
|
|
|
112
|
-
``
|
|
113
|
-
|
|
114
|
-
``source``
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
parses, not bulk/incidental content, so the byte cap does not apply to
|
|
118
|
-
it (``capture_text=False`` still empties it like everything else).
|
|
119
|
-
``sha256``/``size_bytes`` always reflect the full file regardless of
|
|
120
|
-
either knob."""
|
|
96
|
+
``source`` is the WHOLE file or nothing -- never a prefix (#172). A
|
|
97
|
+
decodable file is captured in full; ``capture_text=False`` empties
|
|
98
|
+
``source`` everywhere (inventory otherwise identical), and an undecodable
|
|
99
|
+
file gets ``""`` as ``binary``. ``sha256``/``size_bytes`` always reflect
|
|
100
|
+
the full file regardless."""
|
|
121
101
|
out: Dict[str, PyArtifact] = {}
|
|
122
102
|
for path in sorted(project_dir.rglob("*")):
|
|
123
103
|
if not path.is_file():
|
|
@@ -148,20 +128,14 @@ def discover_artifacts(
|
|
|
148
128
|
if decodable and "." not in name and text.startswith("#!"):
|
|
149
129
|
roles = ["script"]
|
|
150
130
|
if decodable:
|
|
151
|
-
|
|
152
|
-
# dependency_view parses it) -- the byte cap targets bulk/incidental
|
|
153
|
-
# assets, never the files extraction depends on, so manifests are
|
|
154
|
-
# exempt from it. capture_text=False still empties source (handled
|
|
155
|
-
# inside _capture_source); only the byte CAP is bypassed here.
|
|
156
|
-
cap = len(raw) if "dependency-manifest" in roles else text_max_bytes
|
|
157
|
-
source, text_truncated = _capture_source(raw, text, capture_text, cap)
|
|
131
|
+
source = text if capture_text else ""
|
|
158
132
|
else:
|
|
159
|
-
fmt, source
|
|
133
|
+
fmt, source = "binary", ""
|
|
160
134
|
|
|
161
135
|
out[rel_posix] = PyArtifact(
|
|
162
136
|
id=artifact_id(app_name, rel_posix), path=rel_posix, format=fmt,
|
|
163
137
|
roles=list(roles), size_bytes=len(raw),
|
|
164
138
|
sha256=hashlib.sha256(raw).hexdigest(),
|
|
165
|
-
source=source,
|
|
139
|
+
source=source,
|
|
166
140
|
)
|
|
167
141
|
return out
|
codeanalyzer/core.py
CHANGED
|
@@ -40,9 +40,9 @@ from codeanalyzer.provenance import analyzer_info, repository_info
|
|
|
40
40
|
def _artifact_full_text(project_dir: Path, path: str, art) -> str:
|
|
41
41
|
"""Mirrors ``artifacts.dependencies._full_text`` verbatim (not imported
|
|
42
42
|
-- that name is module-private to ``dependencies.py``): config-key
|
|
43
|
-
extraction (#152) must never depend on the stored ``source`` --
|
|
44
|
-
by ``
|
|
45
|
-
|
|
43
|
+
extraction (#152) must never depend on the stored ``source`` -- emptied
|
|
44
|
+
by ``capture_text=False`` (a payload-size control, not an extraction
|
|
45
|
+
control). Read the real file fresh instead;
|
|
46
46
|
fall back to ``art.source`` only if it's gone (e.g. a synthetic artifact
|
|
47
47
|
in a unit test, or the file vanished mid-run). Keep the two in sync if
|
|
48
48
|
this logic changes."""
|
|
@@ -673,7 +673,6 @@ class Codeanalyzer:
|
|
|
673
673
|
app.artifacts = discover_artifacts(
|
|
674
674
|
self.project_dir, app_name,
|
|
675
675
|
capture_text=self.options.artifact_text,
|
|
676
|
-
text_max_bytes=self.options.artifact_text_max_bytes,
|
|
677
676
|
)
|
|
678
677
|
app.dependencies, app.unresolved_imports = build_dependency_view(
|
|
679
678
|
app.artifacts,
|
codeanalyzer/dataflow/builder.py
CHANGED
|
@@ -43,6 +43,7 @@ from codeanalyzer.dataflow.alias import TypeBasedAliasOracle
|
|
|
43
43
|
from codeanalyzer.dataflow.pdg import build_pdg
|
|
44
44
|
from codeanalyzer.dataflow.sdg import ProgramGraphsIR, assemble_sdg
|
|
45
45
|
from codeanalyzer.dataflow.summaries import CallSite, FunctionInfo, compute_summaries
|
|
46
|
+
from codeanalyzer.schema.ids import stamp_body_ids
|
|
46
47
|
from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule
|
|
47
48
|
from codeanalyzer.utils import logger
|
|
48
49
|
|
|
@@ -333,6 +334,7 @@ def emit_l3_body(
|
|
|
333
334
|
for e in pdg.edges
|
|
334
335
|
if e.type == "DDG"
|
|
335
336
|
]
|
|
337
|
+
stamp_body_ids(pycallable)
|
|
336
338
|
|
|
337
339
|
|
|
338
340
|
def build_program_graphs(
|
|
@@ -442,8 +444,13 @@ def build_program_graphs(
|
|
|
442
444
|
for t in cs.targets:
|
|
443
445
|
call_edges.append((sig, t))
|
|
444
446
|
|
|
445
|
-
|
|
446
|
-
|
|
447
|
+
# The converged (facts, ddg) per function are threaded straight into the
|
|
448
|
+
# assembler rather than re-derived there (#155).
|
|
449
|
+
solutions: Dict[str, object] = {}
|
|
450
|
+
summaries = compute_summaries(
|
|
451
|
+
infos, sorted(set(call_edges)), solutions=solutions
|
|
452
|
+
)
|
|
453
|
+
return assemble_sdg(infos, summaries, k, solutions=solutions)
|
|
447
454
|
|
|
448
455
|
|
|
449
456
|
def emit_l4(
|
|
@@ -533,6 +540,7 @@ def emit_l4(
|
|
|
533
540
|
pycallable.body[im.local(pn.id)] = BodyNode(
|
|
534
541
|
kind=pn.kind, of=pn.var, parent=parent
|
|
535
542
|
)
|
|
543
|
+
stamp_body_ids(pycallable)
|
|
536
544
|
|
|
537
545
|
# (b/c/d) SDG edges → summary / param_in / param_out; CALL dropped.
|
|
538
546
|
for e in ir.sdg_edges:
|
|
@@ -16,6 +16,8 @@ from __future__ import annotations
|
|
|
16
16
|
from collections import defaultdict
|
|
17
17
|
from typing import Dict, Iterable, Optional, Tuple
|
|
18
18
|
|
|
19
|
+
from codeanalyzer.schema.ids import global_ordinal
|
|
20
|
+
|
|
19
21
|
|
|
20
22
|
class IdentityMap:
|
|
21
23
|
def __init__(self, callable_id: str, id_to_local: Dict[int, str]):
|
|
@@ -83,9 +85,7 @@ class IdentityMap:
|
|
|
83
85
|
|
|
84
86
|
def global_id(self, node_id: int) -> str:
|
|
85
87
|
"""Fully addressable id: ``"<callable-id>@<local>"``."""
|
|
86
|
-
|
|
87
|
-
# local statements are "line:col"; bookends already carry the leading "@"
|
|
88
|
-
return f"{self._callable_id}{loc}" if loc.startswith("@") else f"{self._callable_id}@{loc}"
|
|
88
|
+
return global_ordinal(self._callable_id, self._map[node_id])
|
|
89
89
|
|
|
90
90
|
def node_ids(self) -> Iterable[int]:
|
|
91
91
|
return self._map.keys()
|
codeanalyzer/dataflow/sdg.py
CHANGED
|
@@ -386,8 +386,18 @@ def assemble_sdg(
|
|
|
386
386
|
infos: Dict[str, FunctionInfo],
|
|
387
387
|
summaries: Dict[str, FunctionSummary],
|
|
388
388
|
k: int,
|
|
389
|
+
*,
|
|
390
|
+
solutions: Optional[Dict[str, Tuple[Dict[int, object], List[object]]]] = None,
|
|
389
391
|
) -> ProgramGraphsIR:
|
|
390
|
-
"""Stitch every function's PDG into the whole-program SDG.
|
|
392
|
+
"""Stitch every function's PDG into the whole-program SDG.
|
|
393
|
+
|
|
394
|
+
*solutions* optionally carries the converged ``(facts, ddg)`` that
|
|
395
|
+
:func:`~codeanalyzer.dataflow.summaries.compute_summaries` already
|
|
396
|
+
derived, sparing a second identical solve per function (#155). Omit it and
|
|
397
|
+
every function is re-solved, which is the historical behaviour and the
|
|
398
|
+
right posture whenever *summaries* did not come from an immediately
|
|
399
|
+
preceding run over these same *infos*.
|
|
400
|
+
"""
|
|
391
401
|
ir = ProgramGraphsIR(k_limit=k)
|
|
392
402
|
|
|
393
403
|
# Pass 1: solve each function against the final summaries and lay out its
|
|
@@ -396,7 +406,12 @@ def assemble_sdg(
|
|
|
396
406
|
formal_ids: Dict[str, Dict[str, int]] = {}
|
|
397
407
|
for sig in sorted(infos):
|
|
398
408
|
info = infos[sig]
|
|
399
|
-
|
|
409
|
+
cached = solutions.get(sig) if solutions is not None else None
|
|
410
|
+
if cached is None:
|
|
411
|
+
summary, facts, ddg = solve_function(info, summaries)
|
|
412
|
+
else:
|
|
413
|
+
facts, ddg = cached
|
|
414
|
+
summary = summaries[sig]
|
|
400
415
|
asm = _FunctionAssembler(info, summary, facts, ddg)
|
|
401
416
|
asm.build_formals()
|
|
402
417
|
assemblers[sig] = asm
|
|
@@ -199,19 +199,42 @@ def solve_function(
|
|
|
199
199
|
def compute_summaries(
|
|
200
200
|
infos: Dict[str, FunctionInfo],
|
|
201
201
|
call_edges: List[Tuple[str, str]],
|
|
202
|
+
*,
|
|
203
|
+
solutions: Optional[Dict[str, Tuple[Dict[int, object], List[DDGEdge]]]] = None,
|
|
202
204
|
) -> Dict[str, FunctionSummary]:
|
|
203
205
|
"""Bottom-up composition over the SCC condensation DAG, monotone fixpoint
|
|
204
|
-
within each SCC.
|
|
206
|
+
within each SCC.
|
|
207
|
+
|
|
208
|
+
A **singleton SCC with no self-edge** is solved exactly once: the
|
|
209
|
+
condensation is processed bottom-up, so every callee summary it reads is
|
|
210
|
+
already final and a second pass could only recompute the same answer to
|
|
211
|
+
observe that nothing changed. Genuinely recursive SCCs (several members,
|
|
212
|
+
or one member calling itself) still iterate to fixpoint.
|
|
213
|
+
|
|
214
|
+
When *solutions* is supplied it receives each signature's converged
|
|
215
|
+
``(facts, ddg)`` — the by-products of the final solve, which
|
|
216
|
+
:func:`~codeanalyzer.dataflow.sdg.assemble_sdg` would otherwise recompute
|
|
217
|
+
from scratch. They are the same values that a fresh solve against the
|
|
218
|
+
final summaries produces, because a converged pass is by definition one
|
|
219
|
+
in which no member's summary changed (#155).
|
|
220
|
+
"""
|
|
205
221
|
order = strongly_connected_components(sorted(infos), call_edges)
|
|
222
|
+
self_calls = {src for src, dst in call_edges if src == dst}
|
|
206
223
|
summaries: Dict[str, FunctionSummary] = {}
|
|
207
224
|
for scc in order:
|
|
208
225
|
members = [s for s in scc if s in infos]
|
|
209
|
-
|
|
210
|
-
|
|
226
|
+
if not members:
|
|
227
|
+
continue
|
|
228
|
+
recursive = len(members) > 1 or members[0] in self_calls
|
|
229
|
+
while True:
|
|
211
230
|
changed = False
|
|
212
231
|
for sig in members:
|
|
213
|
-
new,
|
|
232
|
+
new, facts, ddg = solve_function(infos[sig], summaries)
|
|
233
|
+
if solutions is not None:
|
|
234
|
+
solutions[sig] = (facts, ddg)
|
|
214
235
|
if summaries.get(sig) != new:
|
|
215
236
|
summaries[sig] = new
|
|
216
237
|
changed = True
|
|
238
|
+
if not (recursive and changed):
|
|
239
|
+
break
|
|
217
240
|
return summaries
|
|
@@ -64,7 +64,8 @@ def _compile(pattern: str) -> str:
|
|
|
64
64
|
if ch == "{":
|
|
65
65
|
j = pattern.index("}", i)
|
|
66
66
|
alts = pattern[i + 1 : j].split(",")
|
|
67
|
-
|
|
67
|
+
# `*` keeps its meaning inside an alternative, so `{route,*.route}` works.
|
|
68
|
+
out.append("(?:" + "|".join(_compile(a.strip()) for a in alts) + ")")
|
|
68
69
|
i = j + 1
|
|
69
70
|
elif ch == "*":
|
|
70
71
|
out.append(r"[^.\s]*")
|
|
@@ -93,15 +94,17 @@ def _route_of(dec, spec: Optional[Dict[str, Any]]) -> Optional[str]:
|
|
|
93
94
|
if idx >= len(args):
|
|
94
95
|
return None
|
|
95
96
|
value = _literal(args[idx])
|
|
97
|
+
if isinstance(value, (list, tuple)):
|
|
98
|
+
value = next((v for v in value if isinstance(v, str)), None)
|
|
96
99
|
return value if isinstance(value, str) else None
|
|
97
100
|
|
|
98
101
|
|
|
99
|
-
def _methods_of(dec, spec: Optional[Dict[str, Any]]) -> List[str]:
|
|
102
|
+
def _methods_of(dec, spec: Optional[Dict[str, Any]], qualified: Optional[str] = None) -> List[str]:
|
|
100
103
|
if not spec:
|
|
101
104
|
return []
|
|
102
105
|
source = spec.get("from")
|
|
103
106
|
if source == "match_suffix":
|
|
104
|
-
verb = (dec.qualified_name or "").rsplit(".", 1)[-1]
|
|
107
|
+
verb = (qualified or dec.qualified_name or "").rsplit(".", 1)[-1]
|
|
105
108
|
return [verb.upper()]
|
|
106
109
|
if source == "keyword":
|
|
107
110
|
raw = (dec.keyword_arguments or {}).get(spec.get("name", ""))
|
|
@@ -113,12 +116,27 @@ def _methods_of(dec, spec: Optional[Dict[str, Any]]) -> List[str]:
|
|
|
113
116
|
|
|
114
117
|
|
|
115
118
|
def entrypoints_from_decorators(
|
|
116
|
-
node,
|
|
119
|
+
node,
|
|
120
|
+
framework: str,
|
|
121
|
+
rules: Iterable["DecoratorRule"],
|
|
122
|
+
resolve: Optional[Callable[[str], str]] = None,
|
|
123
|
+
on_written: bool = False,
|
|
117
124
|
) -> List[PyEntrypoint]:
|
|
125
|
+
"""``resolve`` is the module's import-table resolver (#177): when Jedi could
|
|
126
|
+
not resolve a decorator (the framework is not importable in the analysis
|
|
127
|
+
environment -- every ``--no-venv`` run), ``@http.route`` still resolves to
|
|
128
|
+
``odoo.http.route`` from ``from odoo import http`` alone, the same way base
|
|
129
|
+
classes already do. Jedi's definition path wins when it exists.
|
|
130
|
+
|
|
131
|
+
``on_written`` is the heuristic tier: rules match the decorator's spelling
|
|
132
|
+
as WRITTEN (``http.route``, ``router.post``), no resolution at all, so a
|
|
133
|
+
shape that reads as an HTTP entrypoint is recorded whether or not any
|
|
134
|
+
framework rule knows the library behind it."""
|
|
118
135
|
out: List[PyEntrypoint] = []
|
|
119
136
|
for dec in getattr(node, "decorators", []) or []:
|
|
137
|
+
qualified = dec.name if on_written else decorator_qualified_name(dec, resolve)
|
|
120
138
|
for rule in rules:
|
|
121
|
-
if not match_pattern(rule.match,
|
|
139
|
+
if not match_pattern(rule.match, qualified):
|
|
122
140
|
continue
|
|
123
141
|
out.append(
|
|
124
142
|
PyEntrypoint(
|
|
@@ -126,14 +144,25 @@ def entrypoints_from_decorators(
|
|
|
126
144
|
confidence=rule.confidence,
|
|
127
145
|
rule=rule.id,
|
|
128
146
|
ruleset=rule.origin,
|
|
129
|
-
evidence=
|
|
147
|
+
evidence=qualified,
|
|
130
148
|
route=_route_of(dec, rule.route),
|
|
131
|
-
http_methods=_methods_of(dec, rule.methods),
|
|
149
|
+
http_methods=_methods_of(dec, rule.methods, qualified),
|
|
132
150
|
)
|
|
133
151
|
)
|
|
134
152
|
return out
|
|
135
153
|
|
|
136
154
|
|
|
155
|
+
def decorator_qualified_name(dec, resolve: Optional[Callable[[str], str]]) -> Optional[str]:
|
|
156
|
+
"""Jedi's resolution, else the import-table resolution of the written
|
|
157
|
+
spelling, else ``None`` (a spelling the import table cannot map either)."""
|
|
158
|
+
if dec.qualified_name:
|
|
159
|
+
return dec.qualified_name
|
|
160
|
+
if resolve is None:
|
|
161
|
+
return None
|
|
162
|
+
resolved = resolve(dec.name)
|
|
163
|
+
return resolved if resolved != dec.name else None
|
|
164
|
+
|
|
165
|
+
|
|
137
166
|
def entrypoints_from_bases(
|
|
138
167
|
cls,
|
|
139
168
|
framework: str,
|
|
@@ -6,11 +6,14 @@ loses flags, never the analysis.
|
|
|
6
6
|
"""
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
|
+
import builtins
|
|
9
10
|
from pathlib import Path
|
|
10
|
-
from typing import Dict, Iterable, Iterator
|
|
11
|
+
from typing import Dict, Iterable, Iterator, Set
|
|
11
12
|
|
|
12
13
|
from codeanalyzer.entrypoints.detect import detected_frameworks
|
|
13
|
-
from codeanalyzer.entrypoints.matching import
|
|
14
|
+
from codeanalyzer.entrypoints.matching import (
|
|
15
|
+
decorator_qualified_name, entrypoints_from_bases, entrypoints_from_decorators,
|
|
16
|
+
)
|
|
14
17
|
from codeanalyzer.entrypoints.rules import RuleSet, load_rules
|
|
15
18
|
from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule
|
|
16
19
|
from codeanalyzer.utils import logger
|
|
@@ -56,18 +59,34 @@ def _run_stages(app: PyApplication, project_dir: Path, rules: RuleSet) -> None:
|
|
|
56
59
|
"""
|
|
57
60
|
for node in _walk(app):
|
|
58
61
|
node.entrypoints = []
|
|
62
|
+
app.entrypoint_report.unresolved = {}
|
|
59
63
|
|
|
60
64
|
app.entrypoint_report.rulesets = list(rules.rulesets)
|
|
61
65
|
frameworks = detected_frameworks(app, project_dir, rules)
|
|
62
66
|
app.entrypoint_report.frameworks_detected = sorted(frameworks)
|
|
63
67
|
|
|
64
68
|
names = sorted(frameworks)
|
|
69
|
+
unresolved = app.entrypoint_report.unresolved
|
|
65
70
|
for mod in app.symbol_table.values():
|
|
66
71
|
resolve = _base_resolver(mod)
|
|
72
|
+
known = _known_heads(mod)
|
|
67
73
|
for node in _walk_module(mod):
|
|
74
|
+
# #177: what neither Jedi nor the import table could name. This is
|
|
75
|
+
# the counter that makes silence visible; it was never written before.
|
|
76
|
+
# A builtin, a declared class, or a name whose head is imported is
|
|
77
|
+
# nameable and is not counted (`object`, `Exception`, `typing.Generic[T]`).
|
|
78
|
+
for dec in getattr(node, "decorators", None) or []:
|
|
79
|
+
if decorator_qualified_name(dec, resolve) is None and _unnameable(dec.name, known):
|
|
80
|
+
unresolved[dec.name] = unresolved.get(dec.name, 0) + 1
|
|
81
|
+
if isinstance(node, PyClass):
|
|
82
|
+
for base in node.base_classes or []:
|
|
83
|
+
if _unnameable(base, known):
|
|
84
|
+
unresolved[base] = unresolved.get(base, 0) + 1
|
|
68
85
|
for name in names:
|
|
69
86
|
fw = rules.frameworks[name]
|
|
70
|
-
node.entrypoints.extend(
|
|
87
|
+
node.entrypoints.extend(
|
|
88
|
+
entrypoints_from_decorators(node, name, fw.decorators, resolve)
|
|
89
|
+
)
|
|
71
90
|
if isinstance(node, PyClass) and fw.bases:
|
|
72
91
|
class_eps, method_eps = entrypoints_from_bases(
|
|
73
92
|
node, name, fw.bases, resolve
|
|
@@ -77,6 +96,34 @@ def _run_stages(app: PyApplication, project_dir: Path, rules: RuleSet) -> None:
|
|
|
77
96
|
target = (node.callables or {}).get(method_name)
|
|
78
97
|
if target is not None:
|
|
79
98
|
target.entrypoints.extend(eps)
|
|
99
|
+
# Heuristic tier: the written spelling, no framework needed. Runs
|
|
100
|
+
# last so a node a framework rule already claimed keeps one record.
|
|
101
|
+
if not node.entrypoints and rules.heuristics:
|
|
102
|
+
node.entrypoints.extend(
|
|
103
|
+
entrypoints_from_decorators(
|
|
104
|
+
node, "heuristic", rules.heuristics, resolve, on_written=True
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
_BUILTIN_NAMES: Set[str] = set(dir(builtins))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _known_heads(mod: PyModule) -> Set[str]:
|
|
113
|
+
"""Names that can head a nameable spelling in this module: its declared classes
|
|
114
|
+
and every imported name or alias."""
|
|
115
|
+
heads = {cl.name for cl in (mod.types or {}).values()}
|
|
116
|
+
for imp in mod.imports or []:
|
|
117
|
+
heads.add(imp.alias or imp.name)
|
|
118
|
+
return heads
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _unnameable(written: str, known: Set[str]) -> bool:
|
|
122
|
+
"""Whether a written base/decorator spelling maps to nothing this module can
|
|
123
|
+
name: not a builtin, not a declared class, and its head is not imported.
|
|
124
|
+
Subscripts (`Generic[T]`, `dict[K, V]`) are stripped before the check."""
|
|
125
|
+
head = written.split("[", 1)[0].split(".", 1)[0].strip()
|
|
126
|
+
return bool(head) and head not in _BUILTIN_NAMES and head not in known
|
|
80
127
|
|
|
81
128
|
|
|
82
129
|
def _base_resolver(mod: PyModule):
|
|
@@ -90,7 +137,8 @@ def _base_resolver(mod: PyModule):
|
|
|
90
137
|
aliases: Dict[str, str] = {}
|
|
91
138
|
for imp in mod.imports or []:
|
|
92
139
|
original = imp.alias or imp.name
|
|
93
|
-
|
|
140
|
+
module = imp.resolved_module or imp.module
|
|
141
|
+
aliases[imp.name] = module if imp.module == original else f"{module}.{original}"
|
|
94
142
|
|
|
95
143
|
def resolve(written: str) -> str:
|
|
96
144
|
head, _, rest = written.partition(".")
|
|
@@ -22,7 +22,7 @@ _CONFIDENCE = {"declared", "certain", "heuristic"}
|
|
|
22
22
|
# blocks (Units 4-5) not implemented yet; they are deliberately absent here
|
|
23
23
|
# rather than accepted-and-ignored, so a user file using them fails loudly
|
|
24
24
|
# instead of loading clean and doing nothing.
|
|
25
|
-
_TOP_LEVEL_KEYS = {"version", "frameworks", "disable"}
|
|
25
|
+
_TOP_LEVEL_KEYS = {"version", "frameworks", "heuristics", "disable"}
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
class RulesError(Exception):
|
|
@@ -60,6 +60,10 @@ class Framework:
|
|
|
60
60
|
@dataclass
|
|
61
61
|
class RuleSet:
|
|
62
62
|
frameworks: Dict[str, Framework] = field(default_factory=dict)
|
|
63
|
+
# Framework-independent decorator rules matched on the WRITTEN spelling,
|
|
64
|
+
# confidence `heuristic` by default. They run on every node regardless of
|
|
65
|
+
# `frameworks_detected` and never double a record a framework rule made.
|
|
66
|
+
heuristics: List[DecoratorRule] = field(default_factory=list)
|
|
63
67
|
rulesets: List[str] = field(default_factory=list)
|
|
64
68
|
|
|
65
69
|
|
|
@@ -103,9 +107,16 @@ def _merge(out: RuleSet, data: Dict[str, Any], origin: str) -> None:
|
|
|
103
107
|
for raw in body.get("bases") or []:
|
|
104
108
|
fw.bases.append(_base_rule(raw, origin))
|
|
105
109
|
|
|
110
|
+
heuristics = data.get("heuristics") or {}
|
|
111
|
+
if not isinstance(heuristics, dict):
|
|
112
|
+
raise RulesError(f"{origin}: `heuristics` must be a mapping")
|
|
113
|
+
for raw in heuristics.get("decorators") or []:
|
|
114
|
+
out.heuristics.append(_decorator_rule({"confidence": "heuristic", **raw}, origin))
|
|
115
|
+
|
|
106
116
|
for fw in out.frameworks.values():
|
|
107
117
|
fw.decorators = [r for r in fw.decorators if r.id not in disabled]
|
|
108
118
|
fw.bases = [r for r in fw.bases if r.id not in disabled]
|
|
119
|
+
out.heuristics = [r for r in out.heuristics if r.id not in disabled]
|
|
109
120
|
|
|
110
121
|
|
|
111
122
|
def _disable_list(data: Dict[str, Any], origin: str) -> List[str]:
|
|
@@ -22,6 +22,24 @@ frameworks:
|
|
|
22
22
|
transitive: true
|
|
23
23
|
dispatch: [get, post, put, delete, patch]
|
|
24
24
|
|
|
25
|
+
odoo:
|
|
26
|
+
detect: [odoo]
|
|
27
|
+
decorators:
|
|
28
|
+
# `route` is a plain function in odoo/http.py, so Jedi's definition path
|
|
29
|
+
# and the import-table fallback (`from odoo import http` + `@http.route`,
|
|
30
|
+
# the shape every --no-venv run sees) both spell it `odoo.http.route`.
|
|
31
|
+
# The first positional may be one route or a list of them. Odoo serves
|
|
32
|
+
# GET and POST on a route unless `methods=` narrows it (json-typed routes
|
|
33
|
+
# are POST), so the default is both, not GET.
|
|
34
|
+
- id: odoo.route
|
|
35
|
+
match: "odoo.http.route"
|
|
36
|
+
route: {from: positional, index: 0}
|
|
37
|
+
methods: {from: keyword, name: methods, default: [GET, POST]}
|
|
38
|
+
bases:
|
|
39
|
+
- id: odoo.controller
|
|
40
|
+
match: "odoo.http.Controller"
|
|
41
|
+
transitive: true
|
|
42
|
+
|
|
25
43
|
fastapi:
|
|
26
44
|
detect: [fastapi]
|
|
27
45
|
decorators:
|
|
@@ -86,3 +104,19 @@ frameworks:
|
|
|
86
104
|
match: "django.views.generic.*"
|
|
87
105
|
transitive: true
|
|
88
106
|
dispatch: [get, post, put, patch, delete, head, options]
|
|
107
|
+
|
|
108
|
+
# Framework-independent tier. Matched on the decorator's WRITTEN spelling, never
|
|
109
|
+
# on a resolved name, so a shape that reads as an HTTP entrypoint is flagged even
|
|
110
|
+
# when the library behind it has no `frameworks:` block above (or is not
|
|
111
|
+
# importable). Confidence `heuristic`; a consumer wanting only certain hits
|
|
112
|
+
# filters on it. A node a framework rule already matched gets no heuristic record.
|
|
113
|
+
heuristics:
|
|
114
|
+
decorators:
|
|
115
|
+
- id: heuristic.http-route
|
|
116
|
+
match: "{route,*.route,*.*.route}"
|
|
117
|
+
route: {from: positional, index: 0}
|
|
118
|
+
methods: {from: keyword, name: methods}
|
|
119
|
+
- id: heuristic.http-verb
|
|
120
|
+
match: "{*,*.*}.{get,post,put,patch,delete,head,options,websocket}"
|
|
121
|
+
route: {from: positional, index: 0}
|
|
122
|
+
methods: {from: match_suffix}
|