codeanalyzer-python 1.4.0__py3-none-any.whl → 1.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/dataflow/builder.py +3 -0
- codeanalyzer/dataflow/identity.py +3 -3
- codeanalyzer/entrypoints/matching.py +36 -7
- codeanalyzer/entrypoints/pipeline.py +52 -4
- codeanalyzer/entrypoints/rules.py +12 -1
- codeanalyzer/entrypoints/rules.yml +34 -0
- codeanalyzer/neo4j/bolt.py +83 -56
- codeanalyzer/neo4j/cypher.py +10 -4
- codeanalyzer/neo4j/project.py +97 -41
- codeanalyzer/neo4j/rows.py +36 -1
- codeanalyzer/neo4j/schema.py +9 -24
- codeanalyzer/schema/ids.py +17 -0
- codeanalyzer/schema/l1_body.py +2 -0
- codeanalyzer/schema/py_schema.py +6 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/METADATA +13 -4
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/RECORD +20 -20
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/dataflow/builder.py
CHANGED
|
@@ -43,6 +43,7 @@ from codeanalyzer.dataflow.alias import TypeBasedAliasOracle
|
|
|
43
43
|
from codeanalyzer.dataflow.pdg import build_pdg
|
|
44
44
|
from codeanalyzer.dataflow.sdg import ProgramGraphsIR, assemble_sdg
|
|
45
45
|
from codeanalyzer.dataflow.summaries import CallSite, FunctionInfo, compute_summaries
|
|
46
|
+
from codeanalyzer.schema.ids import stamp_body_ids
|
|
46
47
|
from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule
|
|
47
48
|
from codeanalyzer.utils import logger
|
|
48
49
|
|
|
@@ -333,6 +334,7 @@ def emit_l3_body(
|
|
|
333
334
|
for e in pdg.edges
|
|
334
335
|
if e.type == "DDG"
|
|
335
336
|
]
|
|
337
|
+
stamp_body_ids(pycallable)
|
|
336
338
|
|
|
337
339
|
|
|
338
340
|
def build_program_graphs(
|
|
@@ -538,6 +540,7 @@ def emit_l4(
|
|
|
538
540
|
pycallable.body[im.local(pn.id)] = BodyNode(
|
|
539
541
|
kind=pn.kind, of=pn.var, parent=parent
|
|
540
542
|
)
|
|
543
|
+
stamp_body_ids(pycallable)
|
|
541
544
|
|
|
542
545
|
# (b/c/d) SDG edges → summary / param_in / param_out; CALL dropped.
|
|
543
546
|
for e in ir.sdg_edges:
|
|
@@ -16,6 +16,8 @@ from __future__ import annotations
|
|
|
16
16
|
from collections import defaultdict
|
|
17
17
|
from typing import Dict, Iterable, Optional, Tuple
|
|
18
18
|
|
|
19
|
+
from codeanalyzer.schema.ids import global_ordinal
|
|
20
|
+
|
|
19
21
|
|
|
20
22
|
class IdentityMap:
|
|
21
23
|
def __init__(self, callable_id: str, id_to_local: Dict[int, str]):
|
|
@@ -83,9 +85,7 @@ class IdentityMap:
|
|
|
83
85
|
|
|
84
86
|
def global_id(self, node_id: int) -> str:
|
|
85
87
|
"""Fully addressable id: ``"<callable-id>@<local>"``."""
|
|
86
|
-
|
|
87
|
-
# local statements are "line:col"; bookends already carry the leading "@"
|
|
88
|
-
return f"{self._callable_id}{loc}" if loc.startswith("@") else f"{self._callable_id}@{loc}"
|
|
88
|
+
return global_ordinal(self._callable_id, self._map[node_id])
|
|
89
89
|
|
|
90
90
|
def node_ids(self) -> Iterable[int]:
|
|
91
91
|
return self._map.keys()
|
|
@@ -64,7 +64,8 @@ def _compile(pattern: str) -> str:
|
|
|
64
64
|
if ch == "{":
|
|
65
65
|
j = pattern.index("}", i)
|
|
66
66
|
alts = pattern[i + 1 : j].split(",")
|
|
67
|
-
|
|
67
|
+
# `*` keeps its meaning inside an alternative, so `{route,*.route}` works.
|
|
68
|
+
out.append("(?:" + "|".join(_compile(a.strip()) for a in alts) + ")")
|
|
68
69
|
i = j + 1
|
|
69
70
|
elif ch == "*":
|
|
70
71
|
out.append(r"[^.\s]*")
|
|
@@ -93,15 +94,17 @@ def _route_of(dec, spec: Optional[Dict[str, Any]]) -> Optional[str]:
|
|
|
93
94
|
if idx >= len(args):
|
|
94
95
|
return None
|
|
95
96
|
value = _literal(args[idx])
|
|
97
|
+
if isinstance(value, (list, tuple)):
|
|
98
|
+
value = next((v for v in value if isinstance(v, str)), None)
|
|
96
99
|
return value if isinstance(value, str) else None
|
|
97
100
|
|
|
98
101
|
|
|
99
|
-
def _methods_of(dec, spec: Optional[Dict[str, Any]]) -> List[str]:
|
|
102
|
+
def _methods_of(dec, spec: Optional[Dict[str, Any]], qualified: Optional[str] = None) -> List[str]:
|
|
100
103
|
if not spec:
|
|
101
104
|
return []
|
|
102
105
|
source = spec.get("from")
|
|
103
106
|
if source == "match_suffix":
|
|
104
|
-
verb = (dec.qualified_name or "").rsplit(".", 1)[-1]
|
|
107
|
+
verb = (qualified or dec.qualified_name or "").rsplit(".", 1)[-1]
|
|
105
108
|
return [verb.upper()]
|
|
106
109
|
if source == "keyword":
|
|
107
110
|
raw = (dec.keyword_arguments or {}).get(spec.get("name", ""))
|
|
@@ -113,12 +116,27 @@ def _methods_of(dec, spec: Optional[Dict[str, Any]]) -> List[str]:
|
|
|
113
116
|
|
|
114
117
|
|
|
115
118
|
def entrypoints_from_decorators(
|
|
116
|
-
node,
|
|
119
|
+
node,
|
|
120
|
+
framework: str,
|
|
121
|
+
rules: Iterable["DecoratorRule"],
|
|
122
|
+
resolve: Optional[Callable[[str], str]] = None,
|
|
123
|
+
on_written: bool = False,
|
|
117
124
|
) -> List[PyEntrypoint]:
|
|
125
|
+
"""``resolve`` is the module's import-table resolver (#177): when Jedi could
|
|
126
|
+
not resolve a decorator (the framework is not importable in the analysis
|
|
127
|
+
environment -- every ``--no-venv`` run), ``@http.route`` still resolves to
|
|
128
|
+
``odoo.http.route`` from ``from odoo import http`` alone, the same way base
|
|
129
|
+
classes already do. Jedi's definition path wins when it exists.
|
|
130
|
+
|
|
131
|
+
``on_written`` is the heuristic tier: rules match the decorator's spelling
|
|
132
|
+
as WRITTEN (``http.route``, ``router.post``), no resolution at all, so a
|
|
133
|
+
shape that reads as an HTTP entrypoint is recorded whether or not any
|
|
134
|
+
framework rule knows the library behind it."""
|
|
118
135
|
out: List[PyEntrypoint] = []
|
|
119
136
|
for dec in getattr(node, "decorators", []) or []:
|
|
137
|
+
qualified = dec.name if on_written else decorator_qualified_name(dec, resolve)
|
|
120
138
|
for rule in rules:
|
|
121
|
-
if not match_pattern(rule.match,
|
|
139
|
+
if not match_pattern(rule.match, qualified):
|
|
122
140
|
continue
|
|
123
141
|
out.append(
|
|
124
142
|
PyEntrypoint(
|
|
@@ -126,14 +144,25 @@ def entrypoints_from_decorators(
|
|
|
126
144
|
confidence=rule.confidence,
|
|
127
145
|
rule=rule.id,
|
|
128
146
|
ruleset=rule.origin,
|
|
129
|
-
evidence=
|
|
147
|
+
evidence=qualified,
|
|
130
148
|
route=_route_of(dec, rule.route),
|
|
131
|
-
http_methods=_methods_of(dec, rule.methods),
|
|
149
|
+
http_methods=_methods_of(dec, rule.methods, qualified),
|
|
132
150
|
)
|
|
133
151
|
)
|
|
134
152
|
return out
|
|
135
153
|
|
|
136
154
|
|
|
155
|
+
def decorator_qualified_name(dec, resolve: Optional[Callable[[str], str]]) -> Optional[str]:
|
|
156
|
+
"""Jedi's resolution, else the import-table resolution of the written
|
|
157
|
+
spelling, else ``None`` (a spelling the import table cannot map either)."""
|
|
158
|
+
if dec.qualified_name:
|
|
159
|
+
return dec.qualified_name
|
|
160
|
+
if resolve is None:
|
|
161
|
+
return None
|
|
162
|
+
resolved = resolve(dec.name)
|
|
163
|
+
return resolved if resolved != dec.name else None
|
|
164
|
+
|
|
165
|
+
|
|
137
166
|
def entrypoints_from_bases(
|
|
138
167
|
cls,
|
|
139
168
|
framework: str,
|
|
@@ -6,11 +6,14 @@ loses flags, never the analysis.
|
|
|
6
6
|
"""
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
|
+
import builtins
|
|
9
10
|
from pathlib import Path
|
|
10
|
-
from typing import Dict, Iterable, Iterator
|
|
11
|
+
from typing import Dict, Iterable, Iterator, Set
|
|
11
12
|
|
|
12
13
|
from codeanalyzer.entrypoints.detect import detected_frameworks
|
|
13
|
-
from codeanalyzer.entrypoints.matching import
|
|
14
|
+
from codeanalyzer.entrypoints.matching import (
|
|
15
|
+
decorator_qualified_name, entrypoints_from_bases, entrypoints_from_decorators,
|
|
16
|
+
)
|
|
14
17
|
from codeanalyzer.entrypoints.rules import RuleSet, load_rules
|
|
15
18
|
from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule
|
|
16
19
|
from codeanalyzer.utils import logger
|
|
@@ -56,18 +59,34 @@ def _run_stages(app: PyApplication, project_dir: Path, rules: RuleSet) -> None:
|
|
|
56
59
|
"""
|
|
57
60
|
for node in _walk(app):
|
|
58
61
|
node.entrypoints = []
|
|
62
|
+
app.entrypoint_report.unresolved = {}
|
|
59
63
|
|
|
60
64
|
app.entrypoint_report.rulesets = list(rules.rulesets)
|
|
61
65
|
frameworks = detected_frameworks(app, project_dir, rules)
|
|
62
66
|
app.entrypoint_report.frameworks_detected = sorted(frameworks)
|
|
63
67
|
|
|
64
68
|
names = sorted(frameworks)
|
|
69
|
+
unresolved = app.entrypoint_report.unresolved
|
|
65
70
|
for mod in app.symbol_table.values():
|
|
66
71
|
resolve = _base_resolver(mod)
|
|
72
|
+
known = _known_heads(mod)
|
|
67
73
|
for node in _walk_module(mod):
|
|
74
|
+
# #177: what neither Jedi nor the import table could name. This is
|
|
75
|
+
# the counter that makes silence visible; it was never written before.
|
|
76
|
+
# A builtin, a declared class, or a name whose head is imported is
|
|
77
|
+
# nameable and is not counted (`object`, `Exception`, `typing.Generic[T]`).
|
|
78
|
+
for dec in getattr(node, "decorators", None) or []:
|
|
79
|
+
if decorator_qualified_name(dec, resolve) is None and _unnameable(dec.name, known):
|
|
80
|
+
unresolved[dec.name] = unresolved.get(dec.name, 0) + 1
|
|
81
|
+
if isinstance(node, PyClass):
|
|
82
|
+
for base in node.base_classes or []:
|
|
83
|
+
if _unnameable(base, known):
|
|
84
|
+
unresolved[base] = unresolved.get(base, 0) + 1
|
|
68
85
|
for name in names:
|
|
69
86
|
fw = rules.frameworks[name]
|
|
70
|
-
node.entrypoints.extend(
|
|
87
|
+
node.entrypoints.extend(
|
|
88
|
+
entrypoints_from_decorators(node, name, fw.decorators, resolve)
|
|
89
|
+
)
|
|
71
90
|
if isinstance(node, PyClass) and fw.bases:
|
|
72
91
|
class_eps, method_eps = entrypoints_from_bases(
|
|
73
92
|
node, name, fw.bases, resolve
|
|
@@ -77,6 +96,34 @@ def _run_stages(app: PyApplication, project_dir: Path, rules: RuleSet) -> None:
|
|
|
77
96
|
target = (node.callables or {}).get(method_name)
|
|
78
97
|
if target is not None:
|
|
79
98
|
target.entrypoints.extend(eps)
|
|
99
|
+
# Heuristic tier: the written spelling, no framework needed. Runs
|
|
100
|
+
# last so a node a framework rule already claimed keeps one record.
|
|
101
|
+
if not node.entrypoints and rules.heuristics:
|
|
102
|
+
node.entrypoints.extend(
|
|
103
|
+
entrypoints_from_decorators(
|
|
104
|
+
node, "heuristic", rules.heuristics, resolve, on_written=True
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
_BUILTIN_NAMES: Set[str] = set(dir(builtins))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _known_heads(mod: PyModule) -> Set[str]:
|
|
113
|
+
"""Names that can head a nameable spelling in this module: its declared classes
|
|
114
|
+
and every imported name or alias."""
|
|
115
|
+
heads = {cl.name for cl in (mod.types or {}).values()}
|
|
116
|
+
for imp in mod.imports or []:
|
|
117
|
+
heads.add(imp.alias or imp.name)
|
|
118
|
+
return heads
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _unnameable(written: str, known: Set[str]) -> bool:
|
|
122
|
+
"""Whether a written base/decorator spelling maps to nothing this module can
|
|
123
|
+
name: not a builtin, not a declared class, and its head is not imported.
|
|
124
|
+
Subscripts (`Generic[T]`, `dict[K, V]`) are stripped before the check."""
|
|
125
|
+
head = written.split("[", 1)[0].split(".", 1)[0].strip()
|
|
126
|
+
return bool(head) and head not in _BUILTIN_NAMES and head not in known
|
|
80
127
|
|
|
81
128
|
|
|
82
129
|
def _base_resolver(mod: PyModule):
|
|
@@ -90,7 +137,8 @@ def _base_resolver(mod: PyModule):
|
|
|
90
137
|
aliases: Dict[str, str] = {}
|
|
91
138
|
for imp in mod.imports or []:
|
|
92
139
|
original = imp.alias or imp.name
|
|
93
|
-
|
|
140
|
+
module = imp.resolved_module or imp.module
|
|
141
|
+
aliases[imp.name] = module if imp.module == original else f"{module}.{original}"
|
|
94
142
|
|
|
95
143
|
def resolve(written: str) -> str:
|
|
96
144
|
head, _, rest = written.partition(".")
|
|
@@ -22,7 +22,7 @@ _CONFIDENCE = {"declared", "certain", "heuristic"}
|
|
|
22
22
|
# blocks (Units 4-5) not implemented yet; they are deliberately absent here
|
|
23
23
|
# rather than accepted-and-ignored, so a user file using them fails loudly
|
|
24
24
|
# instead of loading clean and doing nothing.
|
|
25
|
-
_TOP_LEVEL_KEYS = {"version", "frameworks", "disable"}
|
|
25
|
+
_TOP_LEVEL_KEYS = {"version", "frameworks", "heuristics", "disable"}
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
class RulesError(Exception):
|
|
@@ -60,6 +60,10 @@ class Framework:
|
|
|
60
60
|
@dataclass
|
|
61
61
|
class RuleSet:
|
|
62
62
|
frameworks: Dict[str, Framework] = field(default_factory=dict)
|
|
63
|
+
# Framework-independent decorator rules matched on the WRITTEN spelling,
|
|
64
|
+
# confidence `heuristic` by default. They run on every node regardless of
|
|
65
|
+
# `frameworks_detected` and never double a record a framework rule made.
|
|
66
|
+
heuristics: List[DecoratorRule] = field(default_factory=list)
|
|
63
67
|
rulesets: List[str] = field(default_factory=list)
|
|
64
68
|
|
|
65
69
|
|
|
@@ -103,9 +107,16 @@ def _merge(out: RuleSet, data: Dict[str, Any], origin: str) -> None:
|
|
|
103
107
|
for raw in body.get("bases") or []:
|
|
104
108
|
fw.bases.append(_base_rule(raw, origin))
|
|
105
109
|
|
|
110
|
+
heuristics = data.get("heuristics") or {}
|
|
111
|
+
if not isinstance(heuristics, dict):
|
|
112
|
+
raise RulesError(f"{origin}: `heuristics` must be a mapping")
|
|
113
|
+
for raw in heuristics.get("decorators") or []:
|
|
114
|
+
out.heuristics.append(_decorator_rule({"confidence": "heuristic", **raw}, origin))
|
|
115
|
+
|
|
106
116
|
for fw in out.frameworks.values():
|
|
107
117
|
fw.decorators = [r for r in fw.decorators if r.id not in disabled]
|
|
108
118
|
fw.bases = [r for r in fw.bases if r.id not in disabled]
|
|
119
|
+
out.heuristics = [r for r in out.heuristics if r.id not in disabled]
|
|
109
120
|
|
|
110
121
|
|
|
111
122
|
def _disable_list(data: Dict[str, Any], origin: str) -> List[str]:
|
|
@@ -22,6 +22,24 @@ frameworks:
|
|
|
22
22
|
transitive: true
|
|
23
23
|
dispatch: [get, post, put, delete, patch]
|
|
24
24
|
|
|
25
|
+
odoo:
|
|
26
|
+
detect: [odoo]
|
|
27
|
+
decorators:
|
|
28
|
+
# `route` is a plain function in odoo/http.py, so Jedi's definition path
|
|
29
|
+
# and the import-table fallback (`from odoo import http` + `@http.route`,
|
|
30
|
+
# the shape every --no-venv run sees) both spell it `odoo.http.route`.
|
|
31
|
+
# The first positional may be one route or a list of them. Odoo serves
|
|
32
|
+
# GET and POST on a route unless `methods=` narrows it (json-typed routes
|
|
33
|
+
# are POST), so the default is both, not GET.
|
|
34
|
+
- id: odoo.route
|
|
35
|
+
match: "odoo.http.route"
|
|
36
|
+
route: {from: positional, index: 0}
|
|
37
|
+
methods: {from: keyword, name: methods, default: [GET, POST]}
|
|
38
|
+
bases:
|
|
39
|
+
- id: odoo.controller
|
|
40
|
+
match: "odoo.http.Controller"
|
|
41
|
+
transitive: true
|
|
42
|
+
|
|
25
43
|
fastapi:
|
|
26
44
|
detect: [fastapi]
|
|
27
45
|
decorators:
|
|
@@ -86,3 +104,19 @@ frameworks:
|
|
|
86
104
|
match: "django.views.generic.*"
|
|
87
105
|
transitive: true
|
|
88
106
|
dispatch: [get, post, put, patch, delete, head, options]
|
|
107
|
+
|
|
108
|
+
# Framework-independent tier. Matched on the decorator's WRITTEN spelling, never
|
|
109
|
+
# on a resolved name, so a shape that reads as an HTTP entrypoint is flagged even
|
|
110
|
+
# when the library behind it has no `frameworks:` block above (or is not
|
|
111
|
+
# importable). Confidence `heuristic`; a consumer wanting only certain hits
|
|
112
|
+
# filters on it. A node a framework rule already matched gets no heuristic record.
|
|
113
|
+
heuristics:
|
|
114
|
+
decorators:
|
|
115
|
+
- id: heuristic.http-route
|
|
116
|
+
match: "{route,*.route,*.*.route}"
|
|
117
|
+
route: {from: positional, index: 0}
|
|
118
|
+
methods: {from: keyword, name: methods}
|
|
119
|
+
- id: heuristic.http-verb
|
|
120
|
+
match: "{*,*.*}.{get,post,put,patch,delete,head,options,websocket}"
|
|
121
|
+
route: {from: positional, index: 0}
|
|
122
|
+
methods: {from: match_suffix}
|
codeanalyzer/neo4j/bolt.py
CHANGED
|
@@ -37,13 +37,16 @@ opt-in under the same flag that already forces a clean analysis rebuild.
|
|
|
37
37
|
|
|
38
38
|
Nodes are MERGE-upserted, never blindly deleted, so a declaration another
|
|
39
39
|
(unchanged) module still references survives and its incoming edges stay valid.
|
|
40
|
-
``:PyExternal`` / ``:PyPackage`` / ``:PyDecorator``
|
|
40
|
+
``:PyExternal`` / ``:PyPackage`` / ``:PyDecorator`` have no owning module and are
|
|
41
41
|
MERGE-only.
|
|
42
42
|
|
|
43
|
-
Every
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
43
|
+
**Every destructive statement is scoped on the ``can://`` id prefix** (#173). The id is a
|
|
44
|
+
path — ``can://python/<app>/<file>/...`` — so ``id = <module-id> OR id STARTS WITH
|
|
45
|
+
<module-id> + '/'`` is containment, and it is one language, one application and one
|
|
46
|
+
module at once. That is what neither a label anchor nor the retired ``_module`` property
|
|
47
|
+
could give: two python applications sharing ``src/foo.py`` carry identical labels and an
|
|
48
|
+
identical file key, and only the id tells them apart. ``:PyCanNode`` anchors the predicate
|
|
49
|
+
so it seeks an index instead of scanning the store; it carries no safety claim.
|
|
47
50
|
|
|
48
51
|
The ``neo4j`` driver is imported lazily so it stays an optional dependency and
|
|
49
52
|
off the default (json) output path entirely.
|
|
@@ -53,16 +56,34 @@ from __future__ import annotations
|
|
|
53
56
|
from dataclasses import dataclass
|
|
54
57
|
from typing import Dict, List, Optional
|
|
55
58
|
|
|
56
|
-
from codeanalyzer.neo4j.rows import
|
|
57
|
-
|
|
59
|
+
from codeanalyzer.neo4j.rows import (
|
|
60
|
+
CAN_NODE, EdgeRow, GraphRows, NodeRow, application_prefix, chunk, descendant_prefix,
|
|
61
|
+
)
|
|
62
|
+
from codeanalyzer.neo4j.schema import CONSTRAINTS, INDEXES
|
|
58
63
|
from codeanalyzer.utils import logger
|
|
59
64
|
|
|
60
|
-
DESCENDANTS = (
|
|
61
|
-
"[:PY_DECLARES|PY_HAS_METHOD|PY_HAS_ATTRIBUTE|PY_DECLARES_VAR"
|
|
62
|
-
"|PY_HAS_CALLSITE|PY_HAS_BODY_NODE*1..]"
|
|
63
|
-
)
|
|
64
65
|
BATCH = 1000
|
|
65
66
|
|
|
67
|
+
# The per-module purge (#173): the module by equality, its subtree by prefix. Anchored
|
|
68
|
+
# on :PyCanNode only so the predicate can seek (see ``rows.CAN_NODE``).
|
|
69
|
+
PURGE_MODULE_EDGES = (
|
|
70
|
+
f"MATCH (x:{CAN_NODE}) WHERE x.id = $mid OR x.id STARTS WITH $pre "
|
|
71
|
+
"MATCH (x)-[r]->() DELETE r"
|
|
72
|
+
)
|
|
73
|
+
PURGE_VANISHED_NODES = (
|
|
74
|
+
f"MATCH (x:{CAN_NODE}) WHERE (x.id = $mid OR x.id STARTS WITH $pre) "
|
|
75
|
+
"AND NOT x.id IN $keys DETACH DELETE x"
|
|
76
|
+
)
|
|
77
|
+
# The orphan prune: modules inside this application's prefix that the run no longer
|
|
78
|
+
# emits, and everything under each. Batched — deleting a large application in one
|
|
79
|
+
# transaction exhausts dbms.memory.transaction.total.max (typescript#116).
|
|
80
|
+
PRUNE_VANISHED_MODULES = (
|
|
81
|
+
f"MATCH (m:PyModule:{CAN_NODE}) WHERE m.id STARTS WITH $app AND NOT m.id IN $present "
|
|
82
|
+
f"CALL {{ WITH m MATCH (x:{CAN_NODE}) WHERE x.id = m.id OR x.id STARTS WITH m.id + '/' "
|
|
83
|
+
"DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS "
|
|
84
|
+
"RETURN count(*) AS pruned"
|
|
85
|
+
)
|
|
86
|
+
|
|
66
87
|
|
|
67
88
|
@dataclass
|
|
68
89
|
class BoltConfig:
|
|
@@ -93,35 +114,44 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool, eager: bool =
|
|
|
93
114
|
for stmt in [*CONSTRAINTS, *INDEXES]:
|
|
94
115
|
s.run(stmt)
|
|
95
116
|
|
|
96
|
-
# The application anchor
|
|
97
|
-
#
|
|
117
|
+
# The application anchor. Every destructive statement below is scoped to
|
|
118
|
+
# ``can://python/<app>/``; an empty application id is refused up front rather
|
|
119
|
+
# than becoming ``STARTS WITH ''`` (every node in the database).
|
|
98
120
|
app_name = next(
|
|
99
121
|
(n.value for n in rows.nodes if n.labels and n.labels[0] == "PyApplication"),
|
|
100
122
|
None,
|
|
101
123
|
)
|
|
124
|
+
app_prefix = application_prefix(app_name)
|
|
102
125
|
|
|
103
|
-
# Partition nodes by owning module
|
|
126
|
+
# Partition nodes by owning module (an in-memory field, never emitted, #173);
|
|
127
|
+
# shared nodes have none.
|
|
104
128
|
by_module: Dict[str, List[NodeRow]] = {}
|
|
105
129
|
shared: List[NodeRow] = []
|
|
106
130
|
module_of: Dict[str, str] = {} # node value → owning module
|
|
107
131
|
for n in rows.nodes:
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
module_of[n.value] = m
|
|
132
|
+
if n.module is not None:
|
|
133
|
+
by_module.setdefault(n.module, []).append(n)
|
|
134
|
+
module_of[n.value] = n.module
|
|
112
135
|
else:
|
|
113
136
|
shared.append(n)
|
|
114
137
|
|
|
115
|
-
# 2. diff content_hash.
|
|
138
|
+
# 2. diff content_hash, keyed by module id inside this application's prefix.
|
|
139
|
+
# Keyed by file key it was application-blind: a second application whose
|
|
140
|
+
# module shares the path and the hash looked "unchanged" and was never written.
|
|
116
141
|
db_hash: Dict[str, Optional[str]] = {}
|
|
117
142
|
with session() as s:
|
|
118
|
-
res = s.run(
|
|
143
|
+
res = s.run(
|
|
144
|
+
f"MATCH (m:PyModule:{CAN_NODE}) WHERE m.id STARTS WITH $app "
|
|
145
|
+
"RETURN m.id AS k, m.content_hash AS h",
|
|
146
|
+
app=app_prefix,
|
|
147
|
+
)
|
|
119
148
|
for rec in res:
|
|
120
149
|
db_hash[rec["k"]] = rec["h"]
|
|
121
150
|
changed = set()
|
|
122
151
|
for m, nodes in by_module.items():
|
|
123
|
-
|
|
124
|
-
|
|
152
|
+
mid = _module_id_of(nodes)
|
|
153
|
+
row_hash = _hash_of(nodes)
|
|
154
|
+
if mid not in db_hash or row_hash is None or row_hash != db_hash.get(mid):
|
|
125
155
|
changed.add(m)
|
|
126
156
|
logger.info(
|
|
127
157
|
f"neo4j(bolt): {len(by_module)} modules ({len(changed)} changed), "
|
|
@@ -139,23 +169,19 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool, eager: bool =
|
|
|
139
169
|
if not eager:
|
|
140
170
|
_upsert_nodes(session, neo4j, nodes)
|
|
141
171
|
continue
|
|
172
|
+
# The module id comes from the module's own row, never by splitting a
|
|
173
|
+
# declaration's id: a file key may itself contain '/'.
|
|
174
|
+
module_id = _module_id_of(nodes)
|
|
175
|
+
if module_id is None or not module_id.startswith(app_prefix):
|
|
176
|
+
raise ValueError(
|
|
177
|
+
f"neo4j: module {m!r} has no can:// id under {app_prefix!r}; "
|
|
178
|
+
"refusing to purge"
|
|
179
|
+
)
|
|
142
180
|
with session() as s:
|
|
143
|
-
def _purge(tx,
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
tx.run(
|
|
148
|
-
f"MATCH (x:{MODULE_OWNED_PATTERN}) WHERE x._module = $m "
|
|
149
|
-
"MATCH (x)-[r]->() DELETE r",
|
|
150
|
-
m=module,
|
|
151
|
-
)
|
|
152
|
-
tx.run(
|
|
153
|
-
f"MATCH (x:{MODULE_OWNED_PATTERN}) WHERE x._module = $m "
|
|
154
|
-
"AND NOT coalesce(x.signature, x.id, x.file_key) IN $keys "
|
|
155
|
-
"DETACH DELETE x",
|
|
156
|
-
m=module,
|
|
157
|
-
keys=node_keys,
|
|
158
|
-
)
|
|
181
|
+
def _purge(tx, mid=module_id, node_keys=keys):
|
|
182
|
+
params = {"mid": mid, "pre": descendant_prefix(mid)}
|
|
183
|
+
tx.run(PURGE_MODULE_EDGES, **params)
|
|
184
|
+
tx.run(PURGE_VANISHED_NODES, keys=node_keys, **params)
|
|
159
185
|
|
|
160
186
|
s.execute_write(_purge)
|
|
161
187
|
_upsert_nodes(session, neo4j, nodes)
|
|
@@ -169,19 +195,13 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool, eager: bool =
|
|
|
169
195
|
_upsert_edges(session, neo4j, edges)
|
|
170
196
|
|
|
171
197
|
# 6. orphan prune — only safe on a full run (a targeted run can't tell deleted from untargeted).
|
|
172
|
-
#
|
|
173
|
-
#
|
|
174
|
-
|
|
175
|
-
|
|
198
|
+
# Scoped to ``can://python/<app>/`` so a full run for application B never deletes
|
|
199
|
+
# application A's modules from a shared database — even when both are python and
|
|
200
|
+
# share a module path.
|
|
201
|
+
if full_run and eager:
|
|
202
|
+
present = [mid for mid in (_module_id_of(ns) for ns in by_module.values()) if mid]
|
|
176
203
|
with session() as s:
|
|
177
|
-
res = s.run(
|
|
178
|
-
"MATCH (:PyApplication {name: $app})-[:PY_HAS_MODULE]->(m:PyModule) "
|
|
179
|
-
"WHERE NOT m.file_key IN $present "
|
|
180
|
-
f"OPTIONAL MATCH (m)-{DESCENDANTS}->(x) DETACH DELETE x, m "
|
|
181
|
-
"RETURN count(m) AS pruned",
|
|
182
|
-
app=app_name,
|
|
183
|
-
present=present,
|
|
184
|
-
)
|
|
204
|
+
res = s.run(PRUNE_VANISHED_MODULES, app=app_prefix, present=present)
|
|
185
205
|
pruned = res.single()
|
|
186
206
|
pruned_count = pruned["pruned"] if pruned else 0
|
|
187
207
|
logger.info(f"neo4j(bolt): pruned {pruned_count} vanished module(s)")
|
|
@@ -263,12 +283,19 @@ def _upsert_edges(session, neo4j, edges: List[EdgeRow]) -> None:
|
|
|
263
283
|
# ----------------------------------------------------------------------------------------------
|
|
264
284
|
|
|
265
285
|
|
|
266
|
-
def
|
|
267
|
-
for n in nodes
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
286
|
+
def _module_row(nodes: List[NodeRow]) -> Optional[NodeRow]:
|
|
287
|
+
return next((n for n in nodes if n.labels[0] == "PyModule"), None)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _module_id_of(nodes: List[NodeRow]) -> Optional[str]:
|
|
291
|
+
row = _module_row(nodes)
|
|
292
|
+
return row.value if row is not None else None
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _hash_of(nodes: List[NodeRow]) -> Optional[str]:
|
|
296
|
+
row = _module_row(nodes)
|
|
297
|
+
h = row.props.get("content_hash") if row is not None else None
|
|
298
|
+
return h if isinstance(h, str) else None
|
|
272
299
|
|
|
273
300
|
|
|
274
301
|
def _to_params(props, neo4j) -> dict:
|
codeanalyzer/neo4j/cypher.py
CHANGED
|
@@ -28,9 +28,11 @@ from __future__ import annotations
|
|
|
28
28
|
from typing import Dict, List
|
|
29
29
|
|
|
30
30
|
from codeanalyzer.neo4j.rows import (
|
|
31
|
+
CAN_NODE,
|
|
31
32
|
EdgeRow,
|
|
32
33
|
GraphRows,
|
|
33
34
|
NodeRow,
|
|
35
|
+
application_prefix,
|
|
34
36
|
chunk,
|
|
35
37
|
cypher_map,
|
|
36
38
|
cypher_value,
|
|
@@ -66,13 +68,17 @@ def render_cypher(rows: GraphRows, app_name: str) -> str:
|
|
|
66
68
|
|
|
67
69
|
|
|
68
70
|
def _wipe(app_name: str) -> str:
|
|
71
|
+
"""Everything under ``can://python/<app>/`` plus the application anchor (#173).
|
|
72
|
+
Scoped by id prefix, so it is one language and one application by construction —
|
|
73
|
+
a second python app sharing a module path, a sibling analyzer's graph, and the
|
|
74
|
+
cross-language :Artifact / :Package nodes are all outside it."""
|
|
75
|
+
prefix = cypher_value(application_prefix(app_name))
|
|
69
76
|
name = cypher_value(app_name)
|
|
70
77
|
return "\n".join(
|
|
71
78
|
[
|
|
72
|
-
f"MATCH (
|
|
73
|
-
"
|
|
74
|
-
"
|
|
75
|
-
"DETACH DELETE x, m, a;",
|
|
79
|
+
f"MATCH (x:{CAN_NODE}) WHERE x.id STARTS WITH {prefix}",
|
|
80
|
+
"CALL { WITH x DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS;",
|
|
81
|
+
f"MATCH (a:PyApplication {{name: {name}}}) DETACH DELETE a;",
|
|
76
82
|
]
|
|
77
83
|
)
|
|
78
84
|
|
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -27,14 +27,15 @@ Modelling decisions (mirror of the TypeScript backend):
|
|
|
27
27
|
- call-graph endpoints absent from the symbol table become ``:PyExternal`` ghost
|
|
28
28
|
nodes, so RPC / third-party / framework edges are preserved (matching the
|
|
29
29
|
analyzer's own ghost-node behaviour).
|
|
30
|
-
- every project-owned node
|
|
30
|
+
- every project-owned node names its owning module (``_module`` in the props it
|
|
31
|
+
hands RowBuilder, lifted to ``NodeRow.module`` and never emitted, #173), so
|
|
31
32
|
the incremental writer can delete exactly what a re-analyzed module emitted.
|
|
32
33
|
"""
|
|
33
34
|
from __future__ import annotations
|
|
34
35
|
|
|
35
36
|
import json
|
|
36
37
|
from pathlib import Path
|
|
37
|
-
from typing import Any, List, Optional
|
|
38
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
38
39
|
|
|
39
40
|
from codeanalyzer.neo4j.schema import SCHEMA_VERSION
|
|
40
41
|
from codeanalyzer.neo4j.rows import GraphRows, NodeRef, Props, RowBuilder, prune
|
|
@@ -47,7 +48,8 @@ from codeanalyzer.schema import (
|
|
|
47
48
|
PyModule,
|
|
48
49
|
PyVariableDeclaration,
|
|
49
50
|
)
|
|
50
|
-
from codeanalyzer.schema
|
|
51
|
+
from codeanalyzer.schema import model_dump
|
|
52
|
+
from codeanalyzer.schema.ids import application_id, global_ordinal, purl_pypi
|
|
51
53
|
from codeanalyzer.schema.py_schema import PyDecorator
|
|
52
54
|
|
|
53
55
|
|
|
@@ -70,6 +72,13 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
70
72
|
"repo_uri": app.repository.uri if app.repository else None,
|
|
71
73
|
"source_revision": app.repository.revision if app.repository else None,
|
|
72
74
|
"repo_dirty": app.repository.dirty if app.repository else None,
|
|
75
|
+
# #177: the entrypoint pass under-approximates by design, so a
|
|
76
|
+
# graph consumer must be able to tell "no entrypoints" from "the
|
|
77
|
+
# pass found nothing". Always present, even when empty.
|
|
78
|
+
"entrypoint_frameworks": list(app.entrypoint_report.frameworks_detected),
|
|
79
|
+
"entrypoint_report_json": json.dumps(
|
|
80
|
+
model_dump(app.entrypoint_report, mode="json"), sort_keys=True
|
|
81
|
+
),
|
|
73
82
|
}
|
|
74
83
|
),
|
|
75
84
|
)
|
|
@@ -87,19 +96,21 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
87
96
|
for file_key, mod in app.symbol_table.items():
|
|
88
97
|
mod_ref = b.node(["PyModule"], "id", mod.id, _module_props(mod, file_key))
|
|
89
98
|
b.edge("PY_HAS_MODULE", app_ref, mod_ref)
|
|
90
|
-
_project_module_body(b, file_key, mod_ref, mod, externals, sig_to_id, module_id_by_key
|
|
99
|
+
_project_module_body(b, file_key, mod_ref, mod, externals, sig_to_id, module_id_by_key,
|
|
100
|
+
application_id(app_name))
|
|
91
101
|
|
|
92
102
|
# The aggregated :PY_CALLS twin.
|
|
103
|
+
app_can_id = application_id(app_name)
|
|
93
104
|
for e in app.call_graph:
|
|
94
|
-
src = _call_endpoint(b, e.src, externals, sig_to_id)
|
|
95
|
-
tgt = _call_endpoint(b, e.dst, externals, sig_to_id)
|
|
105
|
+
src = _call_endpoint(b, e.src, externals, sig_to_id, app_can_id)
|
|
106
|
+
tgt = _call_endpoint(b, e.dst, externals, sig_to_id, app_can_id)
|
|
96
107
|
b.edge(
|
|
97
108
|
"PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.prov or []))
|
|
98
109
|
)
|
|
99
110
|
|
|
100
111
|
# Level-3 CPG overlay: each callable's v2 body/cfg/cdg/ddg. Idempotent under
|
|
101
112
|
# MERGE — a no-op when no callable carries L3 fields (levels 1/2).
|
|
102
|
-
_project_program_graphs(b, app, externals, sig_to_id)
|
|
113
|
+
_project_program_graphs(b, app, externals, sig_to_id, app_can_id)
|
|
103
114
|
|
|
104
115
|
# Neutral artifact/dependency subgraph (Task 6). L1 data — always present,
|
|
105
116
|
# full-depth-always regardless of -a.
|
|
@@ -109,7 +120,7 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
109
120
|
# _project_program_graphs above) into the config-key subgraph
|
|
110
121
|
# (ConfigKey from _project_artifacts above), plus first-class unresolved
|
|
111
122
|
# reads.
|
|
112
|
-
_project_config_uses(b, app, app_ref, externals, sig_to_id)
|
|
123
|
+
_project_config_uses(b, app, app_ref, externals, sig_to_id, app_can_id)
|
|
113
124
|
|
|
114
125
|
return b.finish()
|
|
115
126
|
|
|
@@ -128,11 +139,7 @@ def _global_ordinal(callable_id: str, local_key: str) -> str:
|
|
|
128
139
|
This MUST agree with :meth:`IdentityMap.global_id` for the same node, so the
|
|
129
140
|
JSON ``body``/``cfg`` projection and this Neo4j projection land on one node
|
|
130
141
|
identity (two-projection agreement)."""
|
|
131
|
-
return (
|
|
132
|
-
f"{callable_id}{local_key}"
|
|
133
|
-
if local_key.startswith("@")
|
|
134
|
-
else f"{callable_id}@{local_key}"
|
|
135
|
-
)
|
|
142
|
+
return global_ordinal(callable_id, local_key)
|
|
136
143
|
|
|
137
144
|
|
|
138
145
|
def _body_ref(callable_id: str, local_key: str) -> NodeRef:
|
|
@@ -140,7 +147,7 @@ def _body_ref(callable_id: str, local_key: str) -> NodeRef:
|
|
|
140
147
|
|
|
141
148
|
|
|
142
149
|
def _project_program_graphs(
|
|
143
|
-
b: RowBuilder, app: PyApplication, externals: dict, sig_to_id: dict
|
|
150
|
+
b: RowBuilder, app: PyApplication, externals: dict, sig_to_id: dict, app_can_id: str,
|
|
144
151
|
) -> None:
|
|
145
152
|
"""Level-3 CPG overlay, projected off each callable's v2 ``body``/``cfg``/
|
|
146
153
|
``cdg``/``ddg`` (populated by ``emit_l3_body`` at ``-a 3``; empty otherwise).
|
|
@@ -209,7 +216,7 @@ def _project_program_graphs(
|
|
|
209
216
|
b.edge(
|
|
210
217
|
"PY_RESOLVES_TO",
|
|
211
218
|
ref,
|
|
212
|
-
_call_endpoint(b, node.callee, externals, sig_to_id),
|
|
219
|
+
_call_endpoint(b, node.callee, externals, sig_to_id, app_can_id),
|
|
213
220
|
)
|
|
214
221
|
for e in c.cfg or []:
|
|
215
222
|
# kind-discriminated: a conditional's true/false pair between one
|
|
@@ -399,6 +406,7 @@ def _project_artifacts(b: RowBuilder, app: PyApplication, app_name: str, app_ref
|
|
|
399
406
|
|
|
400
407
|
def _project_config_uses(
|
|
401
408
|
b: RowBuilder, app: PyApplication, app_ref: NodeRef, externals: dict, sig_to_id: dict,
|
|
409
|
+
app_can_id: str,
|
|
402
410
|
) -> None:
|
|
403
411
|
"""config_use (#162): PY_USES_CONFIG (`app.config_uses`) and
|
|
404
412
|
PY_READS_CONFIG_UNRESOLVED (`app.config_reads_unresolved`).
|
|
@@ -428,7 +436,7 @@ def _project_config_uses(
|
|
|
428
436
|
prune({"prov": list(e.prov) if e.prov else None}),
|
|
429
437
|
)
|
|
430
438
|
for r in app.config_reads_unresolved:
|
|
431
|
-
ghost_ref = _call_endpoint(b, r.callee, externals, sig_to_id)
|
|
439
|
+
ghost_ref = _call_endpoint(b, r.callee, externals, sig_to_id, app_can_id)
|
|
432
440
|
b.edge(
|
|
433
441
|
"PY_READS_CONFIG_UNRESOLVED",
|
|
434
442
|
app_ref,
|
|
@@ -453,8 +461,54 @@ def _symbol_ref(signature: str, externals: dict, sig_to_id: dict) -> NodeRef:
|
|
|
453
461
|
return NodeRef("PySymbol", "signature", signature)
|
|
454
462
|
|
|
455
463
|
|
|
464
|
+
def _base_ref_resolver(
|
|
465
|
+
b: RowBuilder, mod: PyModule, externals: dict, sig_to_id: dict, app_can_id: str,
|
|
466
|
+
) -> Callable[[str], NodeRef]:
|
|
467
|
+
"""Per-module: the written base spelling → the NodeRef PY_EXTENDS lands on (#178).
|
|
468
|
+
|
|
469
|
+
``base_classes`` stores the spelling as written (``Base``, ``views.View``),
|
|
470
|
+
while ``sig_to_id`` is keyed by signature (``pkg.mod.Base``), so the two never
|
|
471
|
+
met and every PY_EXTENDS row was dropped as dangling. Resolution order: a class
|
|
472
|
+
declared in this module (bare name or ``Outer.Inner`` path) → its can:// id; a
|
|
473
|
+
name the module's import table maps (same resolver the entrypoint pass uses)
|
|
474
|
+
that is a declared class elsewhere → its can:// id; otherwise an ``@external``
|
|
475
|
+
ghost with the id shape ``_home_external_symbols`` uses, so a call to the same
|
|
476
|
+
symbol MERGEs onto the same node."""
|
|
477
|
+
from codeanalyzer.entrypoints.pipeline import _base_resolver
|
|
478
|
+
|
|
479
|
+
local: Dict[str, str] = {}
|
|
480
|
+
|
|
481
|
+
def index(cl: PyClass, path: str) -> None:
|
|
482
|
+
local.setdefault(cl.name, cl.signature)
|
|
483
|
+
local[path] = cl.signature
|
|
484
|
+
for ic in (cl.types or {}).values():
|
|
485
|
+
index(ic, f"{path}.{ic.name}")
|
|
486
|
+
|
|
487
|
+
for cl in (mod.types or {}).values():
|
|
488
|
+
index(cl, cl.name)
|
|
489
|
+
resolve = _base_resolver(mod)
|
|
490
|
+
|
|
491
|
+
def base_ref(written: str) -> NodeRef:
|
|
492
|
+
sig = local.get(written) or resolve(written)
|
|
493
|
+
can_id = sig_to_id.get(sig)
|
|
494
|
+
if can_id is not None:
|
|
495
|
+
return _sym(can_id)
|
|
496
|
+
return _external_ghost(b, app_can_id, sig)
|
|
497
|
+
|
|
498
|
+
return base_ref
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _external_ghost(b: RowBuilder, app_can_id: str, signature: str) -> NodeRef:
|
|
502
|
+
"""A :PyExternal ghost for a dotted signature nobody homed, with the id shape
|
|
503
|
+
``_home_external_symbols`` uses — ``<app>/@external/<module>/<name>`` — so it
|
|
504
|
+
sits inside the application prefix (#173) and MERGEs with a homed twin."""
|
|
505
|
+
module, name = signature.rsplit(".", 1) if "." in signature else (None, signature)
|
|
506
|
+
ext_id = f"{app_can_id}/@external/{module}/{name}" if module else f"{app_can_id}/@external/{name}"
|
|
507
|
+
return b.node(["PySymbol", "PyExternal"], "id", ext_id, prune({"name": name, "module": module}))
|
|
508
|
+
|
|
509
|
+
|
|
456
510
|
def _call_endpoint(
|
|
457
|
-
b: RowBuilder, signature: str, externals: dict, sig_to_id: dict
|
|
511
|
+
b: RowBuilder, signature: str, externals: dict, sig_to_id: dict, app_can_id: str,
|
|
458
512
|
) -> NodeRef:
|
|
459
513
|
"""A call-graph endpoint: a declared callable already emitted (keyed by its
|
|
460
514
|
canonical ``can://`` id, resolved through ``sig_to_id``), or an external symbol
|
|
@@ -484,13 +538,7 @@ def _call_endpoint(
|
|
|
484
538
|
ext.id or signature,
|
|
485
539
|
prune({"name": ext.name, "module": ext.module}),
|
|
486
540
|
)
|
|
487
|
-
|
|
488
|
-
return b.node(
|
|
489
|
-
["PySymbol", "PyExternal"],
|
|
490
|
-
"id",
|
|
491
|
-
signature,
|
|
492
|
-
prune({"name": name}),
|
|
493
|
-
)
|
|
541
|
+
return _external_ghost(b, app_can_id, signature)
|
|
494
542
|
|
|
495
543
|
|
|
496
544
|
# ----------------------------------------------------------------------------------------------
|
|
@@ -500,16 +548,17 @@ def _call_endpoint(
|
|
|
500
548
|
|
|
501
549
|
def _project_module_body(
|
|
502
550
|
b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: PyModule,
|
|
503
|
-
externals: dict, sig_to_id: dict, module_id_by_key: dict,
|
|
551
|
+
externals: dict, sig_to_id: dict, module_id_by_key: dict, app_can_id: str,
|
|
504
552
|
) -> None:
|
|
553
|
+
base_ref = _base_ref_resolver(b, mod, externals, sig_to_id, app_can_id)
|
|
505
554
|
for fn in (mod.functions or {}).values():
|
|
506
555
|
_project_callable(b, file_key, mod_ref, "PY_DECLARES", fn, externals, sig_to_id,
|
|
507
|
-
mod.source)
|
|
556
|
+
mod.source, base_ref)
|
|
508
557
|
for cl in (mod.types or {}).values():
|
|
509
558
|
_project_class(b, file_key, mod_ref, "PY_DECLARES", cl, externals, sig_to_id,
|
|
510
|
-
mod.source)
|
|
559
|
+
mod.source, base_ref)
|
|
511
560
|
for v in mod.variables or []:
|
|
512
|
-
_project_variable(b, file_key, mod_ref,
|
|
561
|
+
_project_variable(b, file_key, mod_ref, v)
|
|
513
562
|
_project_imports(b, mod_ref, mod, module_id_by_key)
|
|
514
563
|
|
|
515
564
|
|
|
@@ -575,7 +624,7 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule,
|
|
|
575
624
|
|
|
576
625
|
def _project_class(
|
|
577
626
|
b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass,
|
|
578
|
-
externals: dict, sig_to_id: dict, source: str,
|
|
627
|
+
externals: dict, sig_to_id: dict, source: str, base_ref: Callable[[str], NodeRef],
|
|
579
628
|
) -> None:
|
|
580
629
|
ref = b.node(
|
|
581
630
|
["PySymbol", "PyClass"], "id", cl.id, _class_props(cl, file_key, source)
|
|
@@ -587,20 +636,21 @@ def _project_class(
|
|
|
587
636
|
|
|
588
637
|
for base in cl.base_classes or []:
|
|
589
638
|
if base:
|
|
590
|
-
b.edge_to_symbol("PY_EXTENDS", ref,
|
|
639
|
+
b.edge_to_symbol("PY_EXTENDS", ref, base_ref(base))
|
|
591
640
|
|
|
592
641
|
for m in (cl.callables or {}).values():
|
|
593
642
|
_project_callable(b, file_key, ref, "PY_HAS_METHOD", m, externals, sig_to_id,
|
|
594
|
-
source)
|
|
643
|
+
source, base_ref)
|
|
595
644
|
for a in (cl.attributes or {}).values():
|
|
596
|
-
_project_attribute(b, file_key, ref,
|
|
645
|
+
_project_attribute(b, file_key, ref, a)
|
|
597
646
|
for ic in (cl.types or {}).values():
|
|
598
|
-
_project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id, source
|
|
647
|
+
_project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id, source,
|
|
648
|
+
base_ref)
|
|
599
649
|
|
|
600
650
|
|
|
601
651
|
def _project_callable(
|
|
602
652
|
b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable,
|
|
603
|
-
externals: dict, sig_to_id: dict, source: str,
|
|
653
|
+
externals: dict, sig_to_id: dict, source: str, base_ref: Callable[[str], NodeRef],
|
|
604
654
|
) -> None:
|
|
605
655
|
ref = b.node(
|
|
606
656
|
["PySymbol", "PyCallable"],
|
|
@@ -614,18 +664,22 @@ def _project_callable(
|
|
|
614
664
|
_project_decorator(b, ref, d)
|
|
615
665
|
|
|
616
666
|
for v in c.local_variables or []:
|
|
617
|
-
_project_variable(b, file_key, ref,
|
|
667
|
+
_project_variable(b, file_key, ref, v)
|
|
618
668
|
for ic in (c.callables or {}).values():
|
|
619
669
|
_project_callable(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id,
|
|
620
|
-
source)
|
|
670
|
+
source, base_ref)
|
|
621
671
|
for cl in (c.types or {}).values():
|
|
622
|
-
_project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id, source
|
|
672
|
+
_project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id, source,
|
|
673
|
+
base_ref)
|
|
623
674
|
|
|
624
675
|
|
|
625
676
|
def _project_attribute(
|
|
626
|
-
b: RowBuilder, file_key: str, owner: NodeRef,
|
|
677
|
+
b: RowBuilder, file_key: str, owner: NodeRef, a: PyClassAttribute
|
|
627
678
|
) -> None:
|
|
628
|
-
|
|
679
|
+
# ``<class can:// id>/<name>`` (#173): minted from the owner's id so it carries
|
|
680
|
+
# the application segment. The signature-minted ``service.Service.name`` it
|
|
681
|
+
# replaced was identical across applications, so two apps MERGEd onto one node.
|
|
682
|
+
attr_id = f"{owner.value}/{a.name}"
|
|
629
683
|
ref = b.node(["PyAttribute"], "id", attr_id, _attribute_props(a, attr_id, file_key))
|
|
630
684
|
b.edge("PY_HAS_ATTRIBUTE", owner, ref)
|
|
631
685
|
|
|
@@ -634,10 +688,12 @@ def _project_variable(
|
|
|
634
688
|
b: RowBuilder,
|
|
635
689
|
file_key: str,
|
|
636
690
|
owner: NodeRef,
|
|
637
|
-
owner_id: str,
|
|
638
691
|
v: PyVariableDeclaration,
|
|
639
692
|
) -> None:
|
|
640
|
-
|
|
693
|
+
# ``<owner can:// id>/<name>@<line>`` (#173) — the owner is the module or the
|
|
694
|
+
# callable, so a module-level variable sits under ``<module-id>/`` like every
|
|
695
|
+
# other declaration and the module's prefix purge reaches it.
|
|
696
|
+
var_id = f"{owner.value}/{v.name}@{v.start_line}"
|
|
641
697
|
ref = b.node(["PyVariable"], "id", var_id, _variable_props(v, var_id, file_key))
|
|
642
698
|
b.edge("PY_DECLARES_VAR", owner, ref)
|
|
643
699
|
|
codeanalyzer/neo4j/rows.py
CHANGED
|
@@ -50,6 +50,35 @@ class NodeRow:
|
|
|
50
50
|
key_prop: str
|
|
51
51
|
value: str
|
|
52
52
|
props: Props
|
|
53
|
+
# The owning module's file key, for the incremental writer's per-module diff.
|
|
54
|
+
# In memory only (#173): it used to be emitted as ``_module`` and every
|
|
55
|
+
# destructive statement matched on it, which is application-blind. Scope now
|
|
56
|
+
# comes from the ``can://`` id prefix; this field only groups rows.
|
|
57
|
+
module: Optional[str] = None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# The marker label on every node keyed by a ``can://python/`` id (#173). It is an
|
|
61
|
+
# INDEX ANCHOR, nothing more: Neo4j property indexes are label-scoped, so the
|
|
62
|
+
# prefix predicate ``id STARTS WITH $p`` needs a label to seek on. Safety comes
|
|
63
|
+
# from the prefix, which carries language, application and module.
|
|
64
|
+
CAN_NODE = "PyCanNode"
|
|
65
|
+
_PY_CAN_PREFIX = "can://python/"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def descendant_prefix(can_id: str) -> str:
|
|
69
|
+
"""The prefix that matches a node's descendants and nothing else. The separator
|
|
70
|
+
is the point: ``can://python/app/src/foo.py`` is also a prefix of
|
|
71
|
+
``can://python/app/src/foo.pyX``, so descendants match on ``id + '/'`` and the
|
|
72
|
+
node itself by equality."""
|
|
73
|
+
return f"{can_id}/"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def application_prefix(app_name: Optional[str]) -> str:
|
|
77
|
+
"""``can://python/<app>/`` — the scope of every destructive statement. Refuses an
|
|
78
|
+
empty application: ``STARTS WITH ''`` would match every node in the database."""
|
|
79
|
+
if not app_name:
|
|
80
|
+
raise ValueError("neo4j: refusing a destructive statement without an application id")
|
|
81
|
+
return descendant_prefix(f"{_PY_CAN_PREFIX}{app_name}")
|
|
53
82
|
|
|
54
83
|
|
|
55
84
|
@dataclass
|
|
@@ -100,14 +129,20 @@ class RowBuilder:
|
|
|
100
129
|
(last write wins) and unions labels — the in-memory analog of
|
|
101
130
|
``MERGE (n:Label {key}) SET n += props``."""
|
|
102
131
|
node_id = f"{labels[0]} {value}"
|
|
132
|
+
props = dict(props)
|
|
133
|
+
module = props.pop("_module", None) # lifted off the graph (#173)
|
|
134
|
+
if key_prop == "id" and value.startswith(_PY_CAN_PREFIX) and CAN_NODE not in labels:
|
|
135
|
+
labels = [*labels, CAN_NODE]
|
|
103
136
|
existing = self._nodes.get(node_id)
|
|
104
137
|
if existing is not None:
|
|
105
138
|
existing.props.update(props)
|
|
139
|
+
if module is not None:
|
|
140
|
+
existing.module = module
|
|
106
141
|
for label in labels:
|
|
107
142
|
if label not in existing.labels:
|
|
108
143
|
existing.labels.append(label)
|
|
109
144
|
else:
|
|
110
|
-
self._nodes[node_id] = NodeRow(list(labels), key_prop, value,
|
|
145
|
+
self._nodes[node_id] = NodeRow(list(labels), key_prop, value, props, module)
|
|
111
146
|
self._keys.add((labels[0], value))
|
|
112
147
|
return NodeRef(labels[0], key_prop, value)
|
|
113
148
|
|
codeanalyzer/neo4j/schema.py
CHANGED
|
@@ -57,7 +57,9 @@ class RelType:
|
|
|
57
57
|
|
|
58
58
|
|
|
59
59
|
# Labels layered onto a node in addition to its primary/specific label.
|
|
60
|
-
|
|
60
|
+
# ``PyCanNode`` (#173) rides every node keyed by a ``can://python/`` id — an index
|
|
61
|
+
# anchor for the prefix-scoped destructive statements (see ``rows.CAN_NODE``).
|
|
62
|
+
MARKER_LABELS: List[str] = ["PyCanNode"]
|
|
61
63
|
|
|
62
64
|
_SPAN = {"start_line": "integer", "end_line": "integer"}
|
|
63
65
|
|
|
@@ -75,6 +77,8 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
75
77
|
"repo_uri": "string",
|
|
76
78
|
"source_revision": "string",
|
|
77
79
|
"repo_dirty": "boolean",
|
|
80
|
+
"entrypoint_frameworks": "string[]",
|
|
81
|
+
"entrypoint_report_json": "string",
|
|
78
82
|
},
|
|
79
83
|
),
|
|
80
84
|
NodeLabel(
|
|
@@ -88,7 +92,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
88
92
|
"content_hash": "string",
|
|
89
93
|
"last_modified": "float",
|
|
90
94
|
"file_size": "integer",
|
|
91
|
-
"_module": "string",
|
|
92
95
|
},
|
|
93
96
|
),
|
|
94
97
|
NodeLabel(
|
|
@@ -104,7 +107,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
104
107
|
"decorators": "string[]",
|
|
105
108
|
"docstring": "string",
|
|
106
109
|
**_SPAN,
|
|
107
|
-
"_module": "string",
|
|
108
110
|
"is_entrypoint": "boolean",
|
|
109
111
|
"entrypoint_frameworks": "string[]",
|
|
110
112
|
},
|
|
@@ -128,7 +130,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
128
130
|
"modifiers": "string[]",
|
|
129
131
|
"parameters_json": "string",
|
|
130
132
|
"accessed_symbols_json": "string",
|
|
131
|
-
"_module": "string",
|
|
132
133
|
"is_entrypoint": "boolean",
|
|
133
134
|
"entrypoint_frameworks": "string[]",
|
|
134
135
|
},
|
|
@@ -157,7 +158,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
157
158
|
"initializer": "string",
|
|
158
159
|
"docstring": "string",
|
|
159
160
|
**_SPAN,
|
|
160
|
-
"_module": "string",
|
|
161
161
|
},
|
|
162
162
|
),
|
|
163
163
|
NodeLabel(
|
|
@@ -171,7 +171,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
171
171
|
"initializer": "string",
|
|
172
172
|
"scope": "string",
|
|
173
173
|
**_SPAN,
|
|
174
|
-
"_module": "string",
|
|
175
174
|
},
|
|
176
175
|
),
|
|
177
176
|
# Level-3 CPG overlay (present only at -a 3). The dataflow vocabulary is
|
|
@@ -198,7 +197,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
198
197
|
"is_constructor_call": "boolean",
|
|
199
198
|
"arguments_json": "string",
|
|
200
199
|
**_SPAN,
|
|
201
|
-
"_module": "string",
|
|
202
200
|
},
|
|
203
201
|
),
|
|
204
202
|
# Neutral artifact/dependency subgraph (spec 2026-08-27, Task 6). No `Py`
|
|
@@ -332,27 +330,14 @@ def uniqueness_constraints() -> list[str]:
|
|
|
332
330
|
|
|
333
331
|
CONSTRAINTS: List[str] = uniqueness_constraints()
|
|
334
332
|
|
|
335
|
-
# The labels this analyzer owns per module -- the ones carrying the internal ``_module``
|
|
336
|
-
# provenance property. Derived from NODE_LABELS so a new module-scoped label is covered
|
|
337
|
-
# without a second list to maintain. `_module` is NOT python-private: codeanalyzer-java
|
|
338
|
-
# and codeanalyzer-typescript set the same property on their nodes, so every statement
|
|
339
|
-
# matching on it must be anchored to these labels or it matches a sibling analyzer's graph
|
|
340
|
-
# in a shared database (#171).
|
|
341
|
-
MODULE_OWNED_LABELS: List[str] = [n.label for n in NODE_LABELS if "_module" in n.properties]
|
|
342
|
-
|
|
343
|
-
# The label disjunction to anchor such a statement with: ``MATCH (x:PyModule|PyClass|...)``.
|
|
344
|
-
MODULE_OWNED_PATTERN: str = "|".join(MODULE_OWNED_LABELS)
|
|
345
|
-
|
|
346
333
|
INDEXES: List[str] = [
|
|
347
334
|
"CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
|
|
348
335
|
"CREATE INDEX py_class_name IF NOT EXISTS FOR (c:PyClass) ON (c.name)",
|
|
349
336
|
"CREATE FULLTEXT INDEX py_code_fts IF NOT EXISTS FOR (c:PyCallable) ON EACH [c.code, c.docstring]",
|
|
350
|
-
|
|
351
|
-
#
|
|
352
|
-
#
|
|
353
|
-
|
|
354
|
-
f"CREATE INDEX {label.lower()}_module IF NOT EXISTS FOR (x:{label}) ON (x._module)"
|
|
355
|
-
for label in MODULE_OWNED_LABELS
|
|
337
|
+
# #173: every destructive statement is ``MATCH (x:PyCanNode) WHERE x.id STARTS WITH $p``.
|
|
338
|
+
# A range index on the marker makes that a prefix seek; without it, a store scan per
|
|
339
|
+
# changed module. STARTS WITH is index-backed; CONTAINS / ENDS WITH are not.
|
|
340
|
+
"CREATE INDEX py_can_node_id IF NOT EXISTS FOR (n:PyCanNode) ON (n.id)",
|
|
356
341
|
]
|
|
357
342
|
|
|
358
343
|
|
codeanalyzer/schema/ids.py
CHANGED
|
@@ -23,6 +23,23 @@ def ordinal_id(callable_id: str, tag: str) -> str:
|
|
|
23
23
|
return f"{callable_id}@{tag}"
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
def global_ordinal(callable_id: str, local_key: str) -> str:
|
|
27
|
+
"""The GLOBAL ordinal id of a body node from its LOCAL key: synthetic keys
|
|
28
|
+
(`@entry`, `@formal_in:0`) already carry the `@`; positional keys (`15:2`,
|
|
29
|
+
`15:2/actual_in:0`) get one. This is the :PyBodyNode merge key and, since
|
|
30
|
+
#176, `BodyNode.id` — the one implementation both projections share."""
|
|
31
|
+
return f"{callable_id}{local_key}" if local_key.startswith("@") else f"{callable_id}@{local_key}"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def stamp_body_ids(callable) -> None:
|
|
35
|
+
"""Stamp `id` on every body node and parameter of one callable (#176).
|
|
36
|
+
Idempotent; each body emitter calls it after writing its nodes."""
|
|
37
|
+
for key, node in callable.body.items():
|
|
38
|
+
node.id = global_ordinal(callable.id, key)
|
|
39
|
+
for i, p in enumerate(callable.parameters or []):
|
|
40
|
+
p.id = ordinal_id(callable.id, f"formal_in:{i}")
|
|
41
|
+
|
|
42
|
+
|
|
26
43
|
def artifact_id(app_name: str, rel_path: str) -> str:
|
|
27
44
|
"""Language-neutral artifact id: ``can://artifact/<app>/<rel-path>``.
|
|
28
45
|
|
codeanalyzer/schema/l1_body.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""L1 body population: materialize `call` nodes from existing call sites.
|
|
2
2
|
`callee` is left None here — the sanctioned null→id refinement happens at L2."""
|
|
3
3
|
from __future__ import annotations
|
|
4
|
+
from codeanalyzer.schema.ids import stamp_body_ids
|
|
4
5
|
from codeanalyzer.schema.py_schema import PyApplication, PyClass, PyCallable, BodyNode, Span, byte_offsets
|
|
5
6
|
|
|
6
7
|
def _do_callable(source: str, c: PyCallable) -> None:
|
|
@@ -20,6 +21,7 @@ def _do_callable(source: str, c: PyCallable) -> None:
|
|
|
20
21
|
is_constructor_call=cs.is_constructor_call,
|
|
21
22
|
arguments=list(cs.arguments or []),
|
|
22
23
|
)
|
|
24
|
+
stamp_body_ids(c)
|
|
23
25
|
for ic in (c.callables or {}).values():
|
|
24
26
|
_do_callable(source, ic)
|
|
25
27
|
for icl in (c.types or {}).values():
|
codeanalyzer/schema/py_schema.py
CHANGED
|
@@ -128,6 +128,9 @@ class BodyNode(BaseModel):
|
|
|
128
128
|
"""A node in a callable's `body`: an AST region (statement/call/branch/…) or
|
|
129
129
|
a synthetic analysis vertex (entry/exit/formal_in/out/actual_in/out)."""
|
|
130
130
|
kind: str
|
|
131
|
+
# #176: the GLOBAL ordinal id — `<callable-id>@<local>` — the same value the
|
|
132
|
+
# Neo4j projection merges :PyBodyNode on. Stamped by `ids.stamp_body_ids`.
|
|
133
|
+
id: str = ""
|
|
131
134
|
span: Optional[Span] = None
|
|
132
135
|
callee: Optional[str] = None # only on `call` nodes; the sanctioned null→id slot
|
|
133
136
|
of: Optional[str] = None # param vertices: the variable/return they carry
|
|
@@ -287,6 +290,9 @@ class PyCallableParameter(BaseModel):
|
|
|
287
290
|
"""Represents a parameter of a Python callable (function/method)."""
|
|
288
291
|
|
|
289
292
|
name: str
|
|
293
|
+
# #176: `<callable-id>@formal_in:<i>` for position i — the L4 formal_in vertex
|
|
294
|
+
# that carries this parameter. A forward reference below level 4.
|
|
295
|
+
id: str = ""
|
|
290
296
|
type: Optional[str] = None
|
|
291
297
|
default_value: Optional[str] = None
|
|
292
298
|
decorators: List[PyDecorator] = []
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: codeanalyzer-python
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.1
|
|
4
4
|
Summary: Static analysis for Python — canonical schema v2 (symbol table, call graph, and native CFG/PDG/SDG dataflow) as analysis.json or a Neo4j property graph.
|
|
5
5
|
Author-email: Rahul Krishna <i.m.ralk@gmail.com>
|
|
6
6
|
License-File: LICENSE
|
|
@@ -459,7 +459,7 @@ levels are cumulative and additive — `analysis.json(-a 1) ⊆ … ⊆ analysis
|
|
|
459
459
|
| **1** | `-a 1` (default) | Symbol table, Jedi call graph, and `call` nodes in each callable's `body` | `body` calls (`callee: null`) |
|
|
460
460
|
| **2** | `-a 2` | Defuse-linker call-graph enrichment; each call's `callee` backfilled to a `can://` id | `call_graph`, `body` callees |
|
|
461
461
|
| **3** | `-a 3` | Native **intraprocedural** CFG/CDG/DDG (syntactic, name-equality, `prov: ["ssa"]`) | `cfg`, `cdg`, `ddg`, `@entry`/`@exit` on each callable |
|
|
462
|
-
| **4** | `-a 4` | **Interprocedural** SDG: synthetic param vertices, alias-aware DDG (`prov: ["points-to"]`) | `param_in`, `param_out`, `summary`, semantic `ddg` |
|
|
462
|
+
| **4** | `-a 4` | **Interprocedural** SDG: synthetic param vertices, alias-aware DDG (`prov: ["points-to"]`), port-wiring DDG between statements and param vertices (`prov: ["reaching-defs"]`) | `param_in`, `param_out`, `summary`, semantic `ddg` |
|
|
463
463
|
|
|
464
464
|
`-a 1`/`-a 2` timings and output are unaffected by the heavier levels — nothing at level 3+ runs
|
|
465
465
|
unless requested. Flag gating: `--graphs sdg` requires `-a 4`; `--graphs cfg,dfg,pdg` and
|
|
@@ -481,7 +481,11 @@ symbol-table signature by construction
|
|
|
481
481
|
- **Points-to oracle (level 4):** the **Scalpel** may-alias oracle — `ScalpelAliasOracle`
|
|
482
482
|
(`codeanalyzer/dataflow/scalpel_oracle.py`) — consumes Scalpel's SSA copy/const facts to answer
|
|
483
483
|
`may_alias(path_a, path_b)`, adding the alias-aware DDG edges (`prov: ["points-to"]`) and the
|
|
484
|
-
interprocedural summaries.
|
|
484
|
+
interprocedural summaries. Level 4 also wires the statement-level DDG to the param
|
|
485
|
+
vertices (def → `actual_in`, `actual_out` → call site, `formal_in` → use, def → `formal_out`)
|
|
486
|
+
with `prov: ["reaching-defs"]`; without those the SDG would be two disconnected graphs. So
|
|
487
|
+
`prov` takes three values: `ssa` (syntactic, L3), `reaching-defs` (port wiring, L4) and
|
|
488
|
+
`points-to` (alias-derived, L4). Scalpel is **vendored** — a `typed_ast`-free slice built into the
|
|
485
489
|
package under `codeanalyzer/dataflow/scalpel/` — so it is the **default** level-4 oracle with no
|
|
486
490
|
external dependency to install; the analyzer falls back to the built-in `TypeBasedAliasOracle`
|
|
487
491
|
(Jedi-inferred types; unknown types conservatively alias) only when Scalpel can't resolve a
|
|
@@ -546,7 +550,8 @@ A **callable** (function or method) carries its own CPG, keyed by node id:
|
|
|
546
550
|
"body": { // node id → node
|
|
547
551
|
"@entry": { "kind": "entry" },
|
|
548
552
|
"6:4": { "kind": "statement", "span": { … } },
|
|
549
|
-
"6:8": { "kind": "call", "span": { … },
|
|
553
|
+
"6:8": { "id": "can://…/main()@6:8", "kind": "call", "span": { … },
|
|
554
|
+
"callee": "can://…/helper(x)" }, // callee null until L2
|
|
550
555
|
"@formal_in:0": { "kind": "formal_in", "of": "a" }, // L4 param vertices
|
|
551
556
|
"6:4/actual_in:0": { "kind": "actual_in", "of": "a", "parent": "6:4" },
|
|
552
557
|
"@exit": { "kind": "exit" }
|
|
@@ -686,6 +691,10 @@ RETURN DISTINCT c.id
|
|
|
686
691
|
MATCH (m:PyCallable {is_entrypoint: true})
|
|
687
692
|
RETURN m.id, m.entrypoint_frameworks
|
|
688
693
|
|
|
694
|
+
// did the entrypoint pass find anything? (no entrypoints vs. nothing detected)
|
|
695
|
+
MATCH (a:PyApplication)
|
|
696
|
+
RETURN a.entrypoint_frameworks, a.entrypoint_report_json
|
|
697
|
+
|
|
689
698
|
// data dependences into one statement (level 3+)
|
|
690
699
|
MATCH (s:PyBodyNode {id: $stmt})<-[d:PY_DDG]-(src:PyBodyNode)
|
|
691
700
|
RETURN src.id, d.var, d.prov
|
|
@@ -13,11 +13,11 @@ codeanalyzer/artifacts/parsers.py,sha256=xOjC53tT0Mv0k7ONGwzl5XUvpBLnZlEcGmyrX5D
|
|
|
13
13
|
codeanalyzer/dataflow/__init__.py,sha256=HCNOH24rAGLUMG94N9aC-2SsAWSntk3uys7e39ipU5o,1725
|
|
14
14
|
codeanalyzer/dataflow/access_paths.py,sha256=wC8Q9qD-RZzkoFWMVvu_6uNNmYP8z48OGp9h9v3F1d4,23623
|
|
15
15
|
codeanalyzer/dataflow/alias.py,sha256=ZYKY0DiR3GHv6YJr7C001Lode9rZsk6-gxdomUyIvwg,3928
|
|
16
|
-
codeanalyzer/dataflow/builder.py,sha256=
|
|
16
|
+
codeanalyzer/dataflow/builder.py,sha256=kUjI_oBHn5H8cfcSYxj40OHxiqXX-IbxXqahNLgffyQ,32812
|
|
17
17
|
codeanalyzer/dataflow/cfg.py,sha256=u5YXJjAaDshdgv-9kWWITbBJFP-MwXKq6hbNh0fhDm8,25307
|
|
18
18
|
codeanalyzer/dataflow/defuse.py,sha256=LrX1ToOZzR79NlAGg5T647UAFkpiEqEnT7t0K5RXOiI,4377
|
|
19
19
|
codeanalyzer/dataflow/dominance.py,sha256=X7Ki1QRdVqaseYsJtiUL7m9MbcC2WHEv9b8bczSLFUA,5086
|
|
20
|
-
codeanalyzer/dataflow/identity.py,sha256=
|
|
20
|
+
codeanalyzer/dataflow/identity.py,sha256=6aAz3iPSOpoS7l_moC7XGr2RS_GqJp_FlsWGglMJDjM,3812
|
|
21
21
|
codeanalyzer/dataflow/pdg.py,sha256=1Bm7AnoRqXBryZulII2idPw0feV8eZQ1VqqprQ8PW-g,3731
|
|
22
22
|
codeanalyzer/dataflow/scalpel_oracle.py,sha256=FxRadKrTWar0VKHyQJM40n3cq25sld3Sj4lMdOm5NFk,11494
|
|
23
23
|
codeanalyzer/dataflow/scc.py,sha256=Doa_0-5f3agCu_5UmeEDlCUErg34TtniKIv8Uqd7Jiw,3493
|
|
@@ -38,28 +38,28 @@ codeanalyzer/dataflow/scalpel/core/func_call_visitor.py,sha256=ps0snjTchBXilhor3
|
|
|
38
38
|
codeanalyzer/dataflow/scalpel/core/vars_visitor.py,sha256=gE5fNyJS6jslD6vsMRbFLoy3n1xTwH-b54mBcNYO72M,5660
|
|
39
39
|
codeanalyzer/entrypoints/__init__.py,sha256=VaMd4mSEPLuPlRxxAve_nTJ9ZH62stXZGTPNwU_BQ50,99
|
|
40
40
|
codeanalyzer/entrypoints/detect.py,sha256=fsWRQz1njvB9d3LIJEHxsNFE1GrB7q52KCvozw_UIUQ,4594
|
|
41
|
-
codeanalyzer/entrypoints/matching.py,sha256=
|
|
42
|
-
codeanalyzer/entrypoints/pipeline.py,sha256=
|
|
43
|
-
codeanalyzer/entrypoints/rules.py,sha256=
|
|
44
|
-
codeanalyzer/entrypoints/rules.yml,sha256=
|
|
41
|
+
codeanalyzer/entrypoints/matching.py,sha256=rQeY_w2dyG36DBUjle7GLXxqDKsoMhJYPjN4s11OGtM,7948
|
|
42
|
+
codeanalyzer/entrypoints/pipeline.py,sha256=vVSdttMfSAWour2fsC3LomMtwFLzRqmJHixJS82VY1I,7828
|
|
43
|
+
codeanalyzer/entrypoints/rules.py,sha256=yG772JHmNc2L10jIpxfg1xTh-QouJa6lNU9ubo3EZYg,6003
|
|
44
|
+
codeanalyzer/entrypoints/rules.yml,sha256=sguICRfDDNGjJPfZz-JSwkI_hzbTJaau1liHV_y4rTA,4637
|
|
45
45
|
codeanalyzer/jedi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
46
46
|
codeanalyzer/jedi/jedi.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
47
47
|
codeanalyzer/neo4j/__init__.py,sha256=8Ei6uThTOqmul6GhnKoGOjLXJZbyDwF5Jo-8I8NxnAk,1574
|
|
48
|
-
codeanalyzer/neo4j/bolt.py,sha256=
|
|
49
|
-
codeanalyzer/neo4j/cypher.py,sha256=
|
|
48
|
+
codeanalyzer/neo4j/bolt.py,sha256=kcBhujHuUst6s1eZD-vARDn0iuKUAxSHrN14nrsKGM8,13472
|
|
49
|
+
codeanalyzer/neo4j/cypher.py,sha256=y2AW9OUAbZJ4TVHkFpHChJU3r_K7q81tTZKt0VNve6g,5677
|
|
50
50
|
codeanalyzer/neo4j/emit.py,sha256=OBCO77TDTaNpqk9Di3JIQzBouLpHPdgoXuPSs44iJuU,3587
|
|
51
|
-
codeanalyzer/neo4j/project.py,sha256=
|
|
52
|
-
codeanalyzer/neo4j/rows.py,sha256=
|
|
53
|
-
codeanalyzer/neo4j/schema.py,sha256=
|
|
51
|
+
codeanalyzer/neo4j/project.py,sha256=zoLsr2XGywlZbqnwlYvlhIpgu1jmIvSMTyAgIJZkmeM,38494
|
|
52
|
+
codeanalyzer/neo4j/rows.py,sha256=PMlJEyLVIrKITkkK5TOutIzDgBuKLA1RGAkrQ1HLpZo,9579
|
|
53
|
+
codeanalyzer/neo4j/schema.py,sha256=TyT3-Tu9thfWJTuvRtZ3BAIM4kQk13uUEJ-sgr9YAdE,14716
|
|
54
54
|
codeanalyzer/options/__init__.py,sha256=6NewN0a-1HAEgKcxQjYyVn2WEDLrhax8h3WLgZddeqI,94
|
|
55
55
|
codeanalyzer/options/options.py,sha256=2jDCcs74iSEOhz90LoRGts6PC8yV9IDPVxnLPNmonOA,1609
|
|
56
56
|
codeanalyzer/schema/__init__.py,sha256=hIyz02abUJNPW8yUgguaeKflZjTmOZWbDiKICNBD7yk,4141
|
|
57
57
|
codeanalyzer/schema/assign_ids.py,sha256=pQHluLaO5a6hFREp3vjVRsUeAfTYSJ95RXS9TZvnoys,1590
|
|
58
58
|
codeanalyzer/schema/call_graph_ids.py,sha256=mWAhJDBsW8i-siJiINOHCXEm4qM6F_5se9-HuHWIFqU,568
|
|
59
|
-
codeanalyzer/schema/ids.py,sha256=
|
|
60
|
-
codeanalyzer/schema/l1_body.py,sha256=
|
|
59
|
+
codeanalyzer/schema/ids.py,sha256=gBjpOlx4S_1JULlXoAHGommd5e81D5dwd5JSopYwdo8,2543
|
|
60
|
+
codeanalyzer/schema/l1_body.py,sha256=_sca0mTkMRb-5lasep87faGjDxZJd9jGB0hmBx3uCgk,1757
|
|
61
61
|
codeanalyzer/schema/l2_callees.py,sha256=CTcBeGoO04DstDC6h-1sHMbbIrHDEOMxP01U9VLBcfc,2327
|
|
62
|
-
codeanalyzer/schema/py_schema.py,sha256=
|
|
62
|
+
codeanalyzer/schema/py_schema.py,sha256=lbW7fIRWOdVLH0Ql0yYCYZzaS-iaACBLOuDPmXesjKs,24238
|
|
63
63
|
codeanalyzer/semantic_analysis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
64
64
|
codeanalyzer/semantic_analysis/call_graph.py,sha256=6YEB_wTn5-oQYLrbIhJYE0BsHl4fpWPoy5Hwd9mTnGc,11918
|
|
65
65
|
codeanalyzer/semantic_analysis/defuse_linker.py,sha256=bSn1rCun28OQqSd2NOc9aVBSm47Gt9Iul4r_gwfqO1w,64930
|
|
@@ -70,9 +70,9 @@ codeanalyzer/syntactic_analysis/symbol_table_builder.py,sha256=eTknBDlUuuQd3JEwb
|
|
|
70
70
|
codeanalyzer/utils/__init__.py,sha256=hC6VWdR5rerSqBxzu9KQHTASWqwrrYJv-CMDwrTlzkc,137
|
|
71
71
|
codeanalyzer/utils/logging.py,sha256=Fw0tattPAOMs3o0JMjjXhRVLIF64f-SCcygUXF9jqeg,904
|
|
72
72
|
codeanalyzer/utils/progress_bar.py,sha256=C9JtzVdd10lIxTv-KA6PebqjKWueC_vMGwVzAtHuHIw,2818
|
|
73
|
-
codeanalyzer_python-1.4.
|
|
74
|
-
codeanalyzer_python-1.4.
|
|
75
|
-
codeanalyzer_python-1.4.
|
|
76
|
-
codeanalyzer_python-1.4.
|
|
77
|
-
codeanalyzer_python-1.4.
|
|
78
|
-
codeanalyzer_python-1.4.
|
|
73
|
+
codeanalyzer_python-1.4.1.dist-info/METADATA,sha256=tmpHTz20bMUPMGvpJp66Guu8NtDqo-SIFso_Sk0BoHI,43485
|
|
74
|
+
codeanalyzer_python-1.4.1.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
75
|
+
codeanalyzer_python-1.4.1.dist-info/entry_points.txt,sha256=v4Vux0Nnx7sOntVk_CH7W9RX6SkIkvR1FQYq73oVlCQ,105
|
|
76
|
+
codeanalyzer_python-1.4.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
77
|
+
codeanalyzer_python-1.4.1.dist-info/licenses/NOTICE,sha256=MdVkNYqHJ20on2FmWvgD4WpMsX5mir33xdyPvTXd0A0,1223
|
|
78
|
+
codeanalyzer_python-1.4.1.dist-info/RECORD,,
|
|
File without changes
|
{codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|