codeanalyzer-python 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/core.py +2 -2
- codeanalyzer/dataflow/builder.py +3 -0
- codeanalyzer/dataflow/identity.py +3 -3
- codeanalyzer/entrypoints/matching.py +36 -7
- codeanalyzer/entrypoints/pipeline.py +52 -4
- codeanalyzer/entrypoints/rules.py +12 -1
- codeanalyzer/entrypoints/rules.yml +34 -0
- codeanalyzer/neo4j/bolt.py +90 -57
- codeanalyzer/neo4j/cypher.py +19 -5
- codeanalyzer/neo4j/project.py +105 -44
- codeanalyzer/neo4j/rows.py +47 -1
- codeanalyzer/neo4j/schema.py +11 -25
- codeanalyzer/schema/ids.py +57 -10
- codeanalyzer/schema/l1_body.py +2 -0
- codeanalyzer/schema/py_schema.py +8 -2
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/METADATA +42 -11
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/RECORD +21 -21
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/core.py
CHANGED
|
@@ -19,6 +19,7 @@ from codeanalyzer.schema import (
|
|
|
19
19
|
model_validate_json,
|
|
20
20
|
)
|
|
21
21
|
from codeanalyzer.schema.assign_ids import assign_ids
|
|
22
|
+
from codeanalyzer.schema.ids import external_id
|
|
22
23
|
from codeanalyzer.schema.l1_body import populate_l1_body
|
|
23
24
|
from codeanalyzer.schema.l2_callees import backfill_callees
|
|
24
25
|
from codeanalyzer.schema.call_graph_ids import reidentify_call_graph
|
|
@@ -564,8 +565,7 @@ class Codeanalyzer:
|
|
|
564
565
|
if sig in sig_to_id:
|
|
565
566
|
continue
|
|
566
567
|
module, name = sig.rsplit(".", 1) if "." in sig else (None, sig)
|
|
567
|
-
ext_id =
|
|
568
|
-
f"{app_id}/@external/{name}"
|
|
568
|
+
ext_id = external_id(app_id, module, name)
|
|
569
569
|
sig_to_id[sig] = ext_id
|
|
570
570
|
externals[ext_id] = PyExternalSymbol(
|
|
571
571
|
id=ext_id, name=name, module=module
|
codeanalyzer/dataflow/builder.py
CHANGED
|
@@ -43,6 +43,7 @@ from codeanalyzer.dataflow.alias import TypeBasedAliasOracle
|
|
|
43
43
|
from codeanalyzer.dataflow.pdg import build_pdg
|
|
44
44
|
from codeanalyzer.dataflow.sdg import ProgramGraphsIR, assemble_sdg
|
|
45
45
|
from codeanalyzer.dataflow.summaries import CallSite, FunctionInfo, compute_summaries
|
|
46
|
+
from codeanalyzer.schema.ids import stamp_body_ids
|
|
46
47
|
from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule
|
|
47
48
|
from codeanalyzer.utils import logger
|
|
48
49
|
|
|
@@ -333,6 +334,7 @@ def emit_l3_body(
|
|
|
333
334
|
for e in pdg.edges
|
|
334
335
|
if e.type == "DDG"
|
|
335
336
|
]
|
|
337
|
+
stamp_body_ids(pycallable)
|
|
336
338
|
|
|
337
339
|
|
|
338
340
|
def build_program_graphs(
|
|
@@ -538,6 +540,7 @@ def emit_l4(
|
|
|
538
540
|
pycallable.body[im.local(pn.id)] = BodyNode(
|
|
539
541
|
kind=pn.kind, of=pn.var, parent=parent
|
|
540
542
|
)
|
|
543
|
+
stamp_body_ids(pycallable)
|
|
541
544
|
|
|
542
545
|
# (b/c/d) SDG edges → summary / param_in / param_out; CALL dropped.
|
|
543
546
|
for e in ir.sdg_edges:
|
|
@@ -16,6 +16,8 @@ from __future__ import annotations
|
|
|
16
16
|
from collections import defaultdict
|
|
17
17
|
from typing import Dict, Iterable, Optional, Tuple
|
|
18
18
|
|
|
19
|
+
from codeanalyzer.schema.ids import global_ordinal
|
|
20
|
+
|
|
19
21
|
|
|
20
22
|
class IdentityMap:
|
|
21
23
|
def __init__(self, callable_id: str, id_to_local: Dict[int, str]):
|
|
@@ -83,9 +85,7 @@ class IdentityMap:
|
|
|
83
85
|
|
|
84
86
|
def global_id(self, node_id: int) -> str:
|
|
85
87
|
"""Fully addressable id: ``"<callable-id>@<local>"``."""
|
|
86
|
-
|
|
87
|
-
# local statements are "line:col"; bookends already carry the leading "@"
|
|
88
|
-
return f"{self._callable_id}{loc}" if loc.startswith("@") else f"{self._callable_id}@{loc}"
|
|
88
|
+
return global_ordinal(self._callable_id, self._map[node_id])
|
|
89
89
|
|
|
90
90
|
def node_ids(self) -> Iterable[int]:
|
|
91
91
|
return self._map.keys()
|
|
@@ -64,7 +64,8 @@ def _compile(pattern: str) -> str:
|
|
|
64
64
|
if ch == "{":
|
|
65
65
|
j = pattern.index("}", i)
|
|
66
66
|
alts = pattern[i + 1 : j].split(",")
|
|
67
|
-
|
|
67
|
+
# `*` keeps its meaning inside an alternative, so `{route,*.route}` works.
|
|
68
|
+
out.append("(?:" + "|".join(_compile(a.strip()) for a in alts) + ")")
|
|
68
69
|
i = j + 1
|
|
69
70
|
elif ch == "*":
|
|
70
71
|
out.append(r"[^.\s]*")
|
|
@@ -93,15 +94,17 @@ def _route_of(dec, spec: Optional[Dict[str, Any]]) -> Optional[str]:
|
|
|
93
94
|
if idx >= len(args):
|
|
94
95
|
return None
|
|
95
96
|
value = _literal(args[idx])
|
|
97
|
+
if isinstance(value, (list, tuple)):
|
|
98
|
+
value = next((v for v in value if isinstance(v, str)), None)
|
|
96
99
|
return value if isinstance(value, str) else None
|
|
97
100
|
|
|
98
101
|
|
|
99
|
-
def _methods_of(dec, spec: Optional[Dict[str, Any]]) -> List[str]:
|
|
102
|
+
def _methods_of(dec, spec: Optional[Dict[str, Any]], qualified: Optional[str] = None) -> List[str]:
|
|
100
103
|
if not spec:
|
|
101
104
|
return []
|
|
102
105
|
source = spec.get("from")
|
|
103
106
|
if source == "match_suffix":
|
|
104
|
-
verb = (dec.qualified_name or "").rsplit(".", 1)[-1]
|
|
107
|
+
verb = (qualified or dec.qualified_name or "").rsplit(".", 1)[-1]
|
|
105
108
|
return [verb.upper()]
|
|
106
109
|
if source == "keyword":
|
|
107
110
|
raw = (dec.keyword_arguments or {}).get(spec.get("name", ""))
|
|
@@ -113,12 +116,27 @@ def _methods_of(dec, spec: Optional[Dict[str, Any]]) -> List[str]:
|
|
|
113
116
|
|
|
114
117
|
|
|
115
118
|
def entrypoints_from_decorators(
|
|
116
|
-
node,
|
|
119
|
+
node,
|
|
120
|
+
framework: str,
|
|
121
|
+
rules: Iterable["DecoratorRule"],
|
|
122
|
+
resolve: Optional[Callable[[str], str]] = None,
|
|
123
|
+
on_written: bool = False,
|
|
117
124
|
) -> List[PyEntrypoint]:
|
|
125
|
+
"""``resolve`` is the module's import-table resolver (#177): when Jedi could
|
|
126
|
+
not resolve a decorator (the framework is not importable in the analysis
|
|
127
|
+
environment -- every ``--no-venv`` run), ``@http.route`` still resolves to
|
|
128
|
+
``odoo.http.route`` from ``from odoo import http`` alone, the same way base
|
|
129
|
+
classes already do. Jedi's definition path wins when it exists.
|
|
130
|
+
|
|
131
|
+
``on_written`` is the heuristic tier: rules match the decorator's spelling
|
|
132
|
+
as WRITTEN (``http.route``, ``router.post``), no resolution at all, so a
|
|
133
|
+
shape that reads as an HTTP entrypoint is recorded whether or not any
|
|
134
|
+
framework rule knows the library behind it."""
|
|
118
135
|
out: List[PyEntrypoint] = []
|
|
119
136
|
for dec in getattr(node, "decorators", []) or []:
|
|
137
|
+
qualified = dec.name if on_written else decorator_qualified_name(dec, resolve)
|
|
120
138
|
for rule in rules:
|
|
121
|
-
if not match_pattern(rule.match,
|
|
139
|
+
if not match_pattern(rule.match, qualified):
|
|
122
140
|
continue
|
|
123
141
|
out.append(
|
|
124
142
|
PyEntrypoint(
|
|
@@ -126,14 +144,25 @@ def entrypoints_from_decorators(
|
|
|
126
144
|
confidence=rule.confidence,
|
|
127
145
|
rule=rule.id,
|
|
128
146
|
ruleset=rule.origin,
|
|
129
|
-
evidence=
|
|
147
|
+
evidence=qualified,
|
|
130
148
|
route=_route_of(dec, rule.route),
|
|
131
|
-
http_methods=_methods_of(dec, rule.methods),
|
|
149
|
+
http_methods=_methods_of(dec, rule.methods, qualified),
|
|
132
150
|
)
|
|
133
151
|
)
|
|
134
152
|
return out
|
|
135
153
|
|
|
136
154
|
|
|
155
|
+
def decorator_qualified_name(dec, resolve: Optional[Callable[[str], str]]) -> Optional[str]:
|
|
156
|
+
"""Jedi's resolution, else the import-table resolution of the written
|
|
157
|
+
spelling, else ``None`` (a spelling the import table cannot map either)."""
|
|
158
|
+
if dec.qualified_name:
|
|
159
|
+
return dec.qualified_name
|
|
160
|
+
if resolve is None:
|
|
161
|
+
return None
|
|
162
|
+
resolved = resolve(dec.name)
|
|
163
|
+
return resolved if resolved != dec.name else None
|
|
164
|
+
|
|
165
|
+
|
|
137
166
|
def entrypoints_from_bases(
|
|
138
167
|
cls,
|
|
139
168
|
framework: str,
|
|
@@ -6,11 +6,14 @@ loses flags, never the analysis.
|
|
|
6
6
|
"""
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
|
+
import builtins
|
|
9
10
|
from pathlib import Path
|
|
10
|
-
from typing import Dict, Iterable, Iterator
|
|
11
|
+
from typing import Dict, Iterable, Iterator, Set
|
|
11
12
|
|
|
12
13
|
from codeanalyzer.entrypoints.detect import detected_frameworks
|
|
13
|
-
from codeanalyzer.entrypoints.matching import
|
|
14
|
+
from codeanalyzer.entrypoints.matching import (
|
|
15
|
+
decorator_qualified_name, entrypoints_from_bases, entrypoints_from_decorators,
|
|
16
|
+
)
|
|
14
17
|
from codeanalyzer.entrypoints.rules import RuleSet, load_rules
|
|
15
18
|
from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyClass, PyModule
|
|
16
19
|
from codeanalyzer.utils import logger
|
|
@@ -56,18 +59,34 @@ def _run_stages(app: PyApplication, project_dir: Path, rules: RuleSet) -> None:
|
|
|
56
59
|
"""
|
|
57
60
|
for node in _walk(app):
|
|
58
61
|
node.entrypoints = []
|
|
62
|
+
app.entrypoint_report.unresolved = {}
|
|
59
63
|
|
|
60
64
|
app.entrypoint_report.rulesets = list(rules.rulesets)
|
|
61
65
|
frameworks = detected_frameworks(app, project_dir, rules)
|
|
62
66
|
app.entrypoint_report.frameworks_detected = sorted(frameworks)
|
|
63
67
|
|
|
64
68
|
names = sorted(frameworks)
|
|
69
|
+
unresolved = app.entrypoint_report.unresolved
|
|
65
70
|
for mod in app.symbol_table.values():
|
|
66
71
|
resolve = _base_resolver(mod)
|
|
72
|
+
known = _known_heads(mod)
|
|
67
73
|
for node in _walk_module(mod):
|
|
74
|
+
# #177: what neither Jedi nor the import table could name. This is
|
|
75
|
+
# the counter that makes silence visible; it was never written before.
|
|
76
|
+
# A builtin, a declared class, or a name whose head is imported is
|
|
77
|
+
# nameable and is not counted (`object`, `Exception`, `typing.Generic[T]`).
|
|
78
|
+
for dec in getattr(node, "decorators", None) or []:
|
|
79
|
+
if decorator_qualified_name(dec, resolve) is None and _unnameable(dec.name, known):
|
|
80
|
+
unresolved[dec.name] = unresolved.get(dec.name, 0) + 1
|
|
81
|
+
if isinstance(node, PyClass):
|
|
82
|
+
for base in node.base_classes or []:
|
|
83
|
+
if _unnameable(base, known):
|
|
84
|
+
unresolved[base] = unresolved.get(base, 0) + 1
|
|
68
85
|
for name in names:
|
|
69
86
|
fw = rules.frameworks[name]
|
|
70
|
-
node.entrypoints.extend(
|
|
87
|
+
node.entrypoints.extend(
|
|
88
|
+
entrypoints_from_decorators(node, name, fw.decorators, resolve)
|
|
89
|
+
)
|
|
71
90
|
if isinstance(node, PyClass) and fw.bases:
|
|
72
91
|
class_eps, method_eps = entrypoints_from_bases(
|
|
73
92
|
node, name, fw.bases, resolve
|
|
@@ -77,6 +96,34 @@ def _run_stages(app: PyApplication, project_dir: Path, rules: RuleSet) -> None:
|
|
|
77
96
|
target = (node.callables or {}).get(method_name)
|
|
78
97
|
if target is not None:
|
|
79
98
|
target.entrypoints.extend(eps)
|
|
99
|
+
# Heuristic tier: the written spelling, no framework needed. Runs
|
|
100
|
+
# last so a node a framework rule already claimed keeps one record.
|
|
101
|
+
if not node.entrypoints and rules.heuristics:
|
|
102
|
+
node.entrypoints.extend(
|
|
103
|
+
entrypoints_from_decorators(
|
|
104
|
+
node, "heuristic", rules.heuristics, resolve, on_written=True
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
_BUILTIN_NAMES: Set[str] = set(dir(builtins))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _known_heads(mod: PyModule) -> Set[str]:
|
|
113
|
+
"""Names that can head a nameable spelling in this module: its declared classes
|
|
114
|
+
and every imported name or alias."""
|
|
115
|
+
heads = {cl.name for cl in (mod.types or {}).values()}
|
|
116
|
+
for imp in mod.imports or []:
|
|
117
|
+
heads.add(imp.alias or imp.name)
|
|
118
|
+
return heads
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _unnameable(written: str, known: Set[str]) -> bool:
|
|
122
|
+
"""Whether a written base/decorator spelling maps to nothing this module can
|
|
123
|
+
name: not a builtin, not a declared class, and its head is not imported.
|
|
124
|
+
Subscripts (`Generic[T]`, `dict[K, V]`) are stripped before the check."""
|
|
125
|
+
head = written.split("[", 1)[0].split(".", 1)[0].strip()
|
|
126
|
+
return bool(head) and head not in _BUILTIN_NAMES and head not in known
|
|
80
127
|
|
|
81
128
|
|
|
82
129
|
def _base_resolver(mod: PyModule):
|
|
@@ -90,7 +137,8 @@ def _base_resolver(mod: PyModule):
|
|
|
90
137
|
aliases: Dict[str, str] = {}
|
|
91
138
|
for imp in mod.imports or []:
|
|
92
139
|
original = imp.alias or imp.name
|
|
93
|
-
|
|
140
|
+
module = imp.resolved_module or imp.module
|
|
141
|
+
aliases[imp.name] = module if imp.module == original else f"{module}.{original}"
|
|
94
142
|
|
|
95
143
|
def resolve(written: str) -> str:
|
|
96
144
|
head, _, rest = written.partition(".")
|
|
@@ -22,7 +22,7 @@ _CONFIDENCE = {"declared", "certain", "heuristic"}
|
|
|
22
22
|
# blocks (Units 4-5) not implemented yet; they are deliberately absent here
|
|
23
23
|
# rather than accepted-and-ignored, so a user file using them fails loudly
|
|
24
24
|
# instead of loading clean and doing nothing.
|
|
25
|
-
_TOP_LEVEL_KEYS = {"version", "frameworks", "disable"}
|
|
25
|
+
_TOP_LEVEL_KEYS = {"version", "frameworks", "heuristics", "disable"}
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
class RulesError(Exception):
|
|
@@ -60,6 +60,10 @@ class Framework:
|
|
|
60
60
|
@dataclass
|
|
61
61
|
class RuleSet:
|
|
62
62
|
frameworks: Dict[str, Framework] = field(default_factory=dict)
|
|
63
|
+
# Framework-independent decorator rules matched on the WRITTEN spelling,
|
|
64
|
+
# confidence `heuristic` by default. They run on every node regardless of
|
|
65
|
+
# `frameworks_detected` and never double a record a framework rule made.
|
|
66
|
+
heuristics: List[DecoratorRule] = field(default_factory=list)
|
|
63
67
|
rulesets: List[str] = field(default_factory=list)
|
|
64
68
|
|
|
65
69
|
|
|
@@ -103,9 +107,16 @@ def _merge(out: RuleSet, data: Dict[str, Any], origin: str) -> None:
|
|
|
103
107
|
for raw in body.get("bases") or []:
|
|
104
108
|
fw.bases.append(_base_rule(raw, origin))
|
|
105
109
|
|
|
110
|
+
heuristics = data.get("heuristics") or {}
|
|
111
|
+
if not isinstance(heuristics, dict):
|
|
112
|
+
raise RulesError(f"{origin}: `heuristics` must be a mapping")
|
|
113
|
+
for raw in heuristics.get("decorators") or []:
|
|
114
|
+
out.heuristics.append(_decorator_rule({"confidence": "heuristic", **raw}, origin))
|
|
115
|
+
|
|
106
116
|
for fw in out.frameworks.values():
|
|
107
117
|
fw.decorators = [r for r in fw.decorators if r.id not in disabled]
|
|
108
118
|
fw.bases = [r for r in fw.bases if r.id not in disabled]
|
|
119
|
+
out.heuristics = [r for r in out.heuristics if r.id not in disabled]
|
|
109
120
|
|
|
110
121
|
|
|
111
122
|
def _disable_list(data: Dict[str, Any], origin: str) -> List[str]:
|
|
@@ -22,6 +22,24 @@ frameworks:
|
|
|
22
22
|
transitive: true
|
|
23
23
|
dispatch: [get, post, put, delete, patch]
|
|
24
24
|
|
|
25
|
+
odoo:
|
|
26
|
+
detect: [odoo]
|
|
27
|
+
decorators:
|
|
28
|
+
# `route` is a plain function in odoo/http.py, so Jedi's definition path
|
|
29
|
+
# and the import-table fallback (`from odoo import http` + `@http.route`,
|
|
30
|
+
# the shape every --no-venv run sees) both spell it `odoo.http.route`.
|
|
31
|
+
# The first positional may be one route or a list of them. Odoo serves
|
|
32
|
+
# GET and POST on a route unless `methods=` narrows it (json-typed routes
|
|
33
|
+
# are POST), so the default is both, not GET.
|
|
34
|
+
- id: odoo.route
|
|
35
|
+
match: "odoo.http.route"
|
|
36
|
+
route: {from: positional, index: 0}
|
|
37
|
+
methods: {from: keyword, name: methods, default: [GET, POST]}
|
|
38
|
+
bases:
|
|
39
|
+
- id: odoo.controller
|
|
40
|
+
match: "odoo.http.Controller"
|
|
41
|
+
transitive: true
|
|
42
|
+
|
|
25
43
|
fastapi:
|
|
26
44
|
detect: [fastapi]
|
|
27
45
|
decorators:
|
|
@@ -86,3 +104,19 @@ frameworks:
|
|
|
86
104
|
match: "django.views.generic.*"
|
|
87
105
|
transitive: true
|
|
88
106
|
dispatch: [get, post, put, patch, delete, head, options]
|
|
107
|
+
|
|
108
|
+
# Framework-independent tier. Matched on the decorator's WRITTEN spelling, never
|
|
109
|
+
# on a resolved name, so a shape that reads as an HTTP entrypoint is flagged even
|
|
110
|
+
# when the library behind it has no `frameworks:` block above (or is not
|
|
111
|
+
# importable). Confidence `heuristic`; a consumer wanting only certain hits
|
|
112
|
+
# filters on it. A node a framework rule already matched gets no heuristic record.
|
|
113
|
+
heuristics:
|
|
114
|
+
decorators:
|
|
115
|
+
- id: heuristic.http-route
|
|
116
|
+
match: "{route,*.route,*.*.route}"
|
|
117
|
+
route: {from: positional, index: 0}
|
|
118
|
+
methods: {from: keyword, name: methods}
|
|
119
|
+
- id: heuristic.http-verb
|
|
120
|
+
match: "{*,*.*}.{get,post,put,patch,delete,head,options,websocket}"
|
|
121
|
+
route: {from: positional, index: 0}
|
|
122
|
+
methods: {from: match_suffix}
|
codeanalyzer/neo4j/bolt.py
CHANGED
|
@@ -37,13 +37,16 @@ opt-in under the same flag that already forces a clean analysis rebuild.
|
|
|
37
37
|
|
|
38
38
|
Nodes are MERGE-upserted, never blindly deleted, so a declaration another
|
|
39
39
|
(unchanged) module still references survives and its incoming edges stay valid.
|
|
40
|
-
``:PyExternal`` / ``:PyPackage`` / ``:PyDecorator``
|
|
40
|
+
``:PyExternal`` / ``:PyPackage`` / ``:PyDecorator`` have no owning module and are
|
|
41
41
|
MERGE-only.
|
|
42
42
|
|
|
43
|
-
Every
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
43
|
+
**Every destructive statement is scoped on the ``can://`` id prefix** (#173). The id is a
|
|
44
|
+
path — ``can://<app>/python/<file>/...`` — so ``id = <module-id> OR id STARTS WITH
|
|
45
|
+
<module-id> + '/'`` is containment, and it is one application, one language and one
|
|
46
|
+
module at once. That is what neither a label anchor nor the retired ``_module`` property
|
|
47
|
+
could give: two python applications sharing ``src/foo.py`` carry identical labels and an
|
|
48
|
+
identical file key, and only the id tells them apart. ``:PyCanNode`` anchors the predicate
|
|
49
|
+
so it seeks an index instead of scanning the store; it carries no safety claim.
|
|
47
50
|
|
|
48
51
|
The ``neo4j`` driver is imported lazily so it stays an optional dependency and
|
|
49
52
|
off the default (json) output path entirely.
|
|
@@ -53,16 +56,34 @@ from __future__ import annotations
|
|
|
53
56
|
from dataclasses import dataclass
|
|
54
57
|
from typing import Dict, List, Optional
|
|
55
58
|
|
|
56
|
-
from codeanalyzer.neo4j.rows import
|
|
57
|
-
|
|
59
|
+
from codeanalyzer.neo4j.rows import (
|
|
60
|
+
CAN_NODE, EdgeRow, GraphRows, NodeRow, application_prefix, chunk, descendant_prefix,
|
|
61
|
+
)
|
|
62
|
+
from codeanalyzer.neo4j.schema import CONSTRAINTS, INDEXES
|
|
58
63
|
from codeanalyzer.utils import logger
|
|
59
64
|
|
|
60
|
-
DESCENDANTS = (
|
|
61
|
-
"[:PY_DECLARES|PY_HAS_METHOD|PY_HAS_ATTRIBUTE|PY_DECLARES_VAR"
|
|
62
|
-
"|PY_HAS_CALLSITE|PY_HAS_BODY_NODE*1..]"
|
|
63
|
-
)
|
|
64
65
|
BATCH = 1000
|
|
65
66
|
|
|
67
|
+
# The per-module purge (#173): the module by equality, its subtree by prefix. Anchored
|
|
68
|
+
# on :PyCanNode only so the predicate can seek (see ``rows.CAN_NODE``).
|
|
69
|
+
PURGE_MODULE_EDGES = (
|
|
70
|
+
f"MATCH (x:{CAN_NODE}) WHERE x.id = $mid OR x.id STARTS WITH $pre "
|
|
71
|
+
"MATCH (x)-[r]->() DELETE r"
|
|
72
|
+
)
|
|
73
|
+
PURGE_VANISHED_NODES = (
|
|
74
|
+
f"MATCH (x:{CAN_NODE}) WHERE (x.id = $mid OR x.id STARTS WITH $pre) "
|
|
75
|
+
"AND NOT x.id IN $keys DETACH DELETE x"
|
|
76
|
+
)
|
|
77
|
+
# The orphan prune: modules inside this application's prefix that the run no longer
|
|
78
|
+
# emits, and everything under each. Batched — deleting a large application in one
|
|
79
|
+
# transaction exhausts dbms.memory.transaction.total.max (typescript#116).
|
|
80
|
+
PRUNE_VANISHED_MODULES = (
|
|
81
|
+
f"MATCH (m:PyModule:{CAN_NODE}) WHERE m.id STARTS WITH $app AND NOT m.id IN $present "
|
|
82
|
+
f"CALL {{ WITH m MATCH (x:{CAN_NODE}) WHERE x.id = m.id OR x.id STARTS WITH m.id + '/' "
|
|
83
|
+
"DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS "
|
|
84
|
+
"RETURN count(*) AS pruned"
|
|
85
|
+
)
|
|
86
|
+
|
|
66
87
|
|
|
67
88
|
@dataclass
|
|
68
89
|
class BoltConfig:
|
|
@@ -93,35 +114,50 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool, eager: bool =
|
|
|
93
114
|
for stmt in [*CONSTRAINTS, *INDEXES]:
|
|
94
115
|
s.run(stmt)
|
|
95
116
|
|
|
96
|
-
# The application anchor
|
|
97
|
-
#
|
|
117
|
+
# The application anchor. Every destructive statement below is scoped to
|
|
118
|
+
# ``can://<app>/``; an empty application is refused up front rather than
|
|
119
|
+
# becoming ``STARTS WITH ''`` (every node in the database). The root row is
|
|
120
|
+
# keyed on its ``can://`` id now, so the name comes off its props — reading
|
|
121
|
+
# ``n.value`` here would build ``can://can://<app>/``.
|
|
98
122
|
app_name = next(
|
|
99
|
-
(
|
|
123
|
+
(
|
|
124
|
+
n.props.get("name")
|
|
125
|
+
for n in rows.nodes
|
|
126
|
+
if n.labels and n.labels[0] == "PyApplication"
|
|
127
|
+
),
|
|
100
128
|
None,
|
|
101
129
|
)
|
|
130
|
+
app_prefix = application_prefix(app_name)
|
|
102
131
|
|
|
103
|
-
# Partition nodes by owning module
|
|
132
|
+
# Partition nodes by owning module (an in-memory field, never emitted, #173);
|
|
133
|
+
# shared nodes have none.
|
|
104
134
|
by_module: Dict[str, List[NodeRow]] = {}
|
|
105
135
|
shared: List[NodeRow] = []
|
|
106
136
|
module_of: Dict[str, str] = {} # node value → owning module
|
|
107
137
|
for n in rows.nodes:
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
module_of[n.value] = m
|
|
138
|
+
if n.module is not None:
|
|
139
|
+
by_module.setdefault(n.module, []).append(n)
|
|
140
|
+
module_of[n.value] = n.module
|
|
112
141
|
else:
|
|
113
142
|
shared.append(n)
|
|
114
143
|
|
|
115
|
-
# 2. diff content_hash.
|
|
144
|
+
# 2. diff content_hash, keyed by module id inside this application's prefix.
|
|
145
|
+
# Keyed by file key it was application-blind: a second application whose
|
|
146
|
+
# module shares the path and the hash looked "unchanged" and was never written.
|
|
116
147
|
db_hash: Dict[str, Optional[str]] = {}
|
|
117
148
|
with session() as s:
|
|
118
|
-
res = s.run(
|
|
149
|
+
res = s.run(
|
|
150
|
+
f"MATCH (m:PyModule:{CAN_NODE}) WHERE m.id STARTS WITH $app "
|
|
151
|
+
"RETURN m.id AS k, m.content_hash AS h",
|
|
152
|
+
app=app_prefix,
|
|
153
|
+
)
|
|
119
154
|
for rec in res:
|
|
120
155
|
db_hash[rec["k"]] = rec["h"]
|
|
121
156
|
changed = set()
|
|
122
157
|
for m, nodes in by_module.items():
|
|
123
|
-
|
|
124
|
-
|
|
158
|
+
mid = _module_id_of(nodes)
|
|
159
|
+
row_hash = _hash_of(nodes)
|
|
160
|
+
if mid not in db_hash or row_hash is None or row_hash != db_hash.get(mid):
|
|
125
161
|
changed.add(m)
|
|
126
162
|
logger.info(
|
|
127
163
|
f"neo4j(bolt): {len(by_module)} modules ({len(changed)} changed), "
|
|
@@ -139,23 +175,19 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool, eager: bool =
|
|
|
139
175
|
if not eager:
|
|
140
176
|
_upsert_nodes(session, neo4j, nodes)
|
|
141
177
|
continue
|
|
178
|
+
# The module id comes from the module's own row, never by splitting a
|
|
179
|
+
# declaration's id: a file key may itself contain '/'.
|
|
180
|
+
module_id = _module_id_of(nodes)
|
|
181
|
+
if module_id is None or not module_id.startswith(app_prefix):
|
|
182
|
+
raise ValueError(
|
|
183
|
+
f"neo4j: module {m!r} has no can:// id under {app_prefix!r}; "
|
|
184
|
+
"refusing to purge"
|
|
185
|
+
)
|
|
142
186
|
with session() as s:
|
|
143
|
-
def _purge(tx,
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
tx.run(
|
|
148
|
-
f"MATCH (x:{MODULE_OWNED_PATTERN}) WHERE x._module = $m "
|
|
149
|
-
"MATCH (x)-[r]->() DELETE r",
|
|
150
|
-
m=module,
|
|
151
|
-
)
|
|
152
|
-
tx.run(
|
|
153
|
-
f"MATCH (x:{MODULE_OWNED_PATTERN}) WHERE x._module = $m "
|
|
154
|
-
"AND NOT coalesce(x.signature, x.id, x.file_key) IN $keys "
|
|
155
|
-
"DETACH DELETE x",
|
|
156
|
-
m=module,
|
|
157
|
-
keys=node_keys,
|
|
158
|
-
)
|
|
187
|
+
def _purge(tx, mid=module_id, node_keys=keys):
|
|
188
|
+
params = {"mid": mid, "pre": descendant_prefix(mid)}
|
|
189
|
+
tx.run(PURGE_MODULE_EDGES, **params)
|
|
190
|
+
tx.run(PURGE_VANISHED_NODES, keys=node_keys, **params)
|
|
159
191
|
|
|
160
192
|
s.execute_write(_purge)
|
|
161
193
|
_upsert_nodes(session, neo4j, nodes)
|
|
@@ -169,19 +201,13 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool, eager: bool =
|
|
|
169
201
|
_upsert_edges(session, neo4j, edges)
|
|
170
202
|
|
|
171
203
|
# 6. orphan prune — only safe on a full run (a targeted run can't tell deleted from untargeted).
|
|
172
|
-
#
|
|
173
|
-
#
|
|
174
|
-
|
|
175
|
-
|
|
204
|
+
# Scoped to ``can://<app>/`` so a full run for application B never deletes
|
|
205
|
+
# application A's modules from a shared database — even when both are python and
|
|
206
|
+
# share a module path.
|
|
207
|
+
if full_run and eager:
|
|
208
|
+
present = [mid for mid in (_module_id_of(ns) for ns in by_module.values()) if mid]
|
|
176
209
|
with session() as s:
|
|
177
|
-
res = s.run(
|
|
178
|
-
"MATCH (:PyApplication {name: $app})-[:PY_HAS_MODULE]->(m:PyModule) "
|
|
179
|
-
"WHERE NOT m.file_key IN $present "
|
|
180
|
-
f"OPTIONAL MATCH (m)-{DESCENDANTS}->(x) DETACH DELETE x, m "
|
|
181
|
-
"RETURN count(m) AS pruned",
|
|
182
|
-
app=app_name,
|
|
183
|
-
present=present,
|
|
184
|
-
)
|
|
210
|
+
res = s.run(PRUNE_VANISHED_MODULES, app=app_prefix, present=present)
|
|
185
211
|
pruned = res.single()
|
|
186
212
|
pruned_count = pruned["pruned"] if pruned else 0
|
|
187
213
|
logger.info(f"neo4j(bolt): pruned {pruned_count} vanished module(s)")
|
|
@@ -263,12 +289,19 @@ def _upsert_edges(session, neo4j, edges: List[EdgeRow]) -> None:
|
|
|
263
289
|
# ----------------------------------------------------------------------------------------------
|
|
264
290
|
|
|
265
291
|
|
|
266
|
-
def
|
|
267
|
-
for n in nodes
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
292
|
+
def _module_row(nodes: List[NodeRow]) -> Optional[NodeRow]:
|
|
293
|
+
return next((n for n in nodes if n.labels[0] == "PyModule"), None)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _module_id_of(nodes: List[NodeRow]) -> Optional[str]:
|
|
297
|
+
row = _module_row(nodes)
|
|
298
|
+
return row.value if row is not None else None
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _hash_of(nodes: List[NodeRow]) -> Optional[str]:
|
|
302
|
+
row = _module_row(nodes)
|
|
303
|
+
h = row.props.get("content_hash") if row is not None else None
|
|
304
|
+
return h if isinstance(h, str) else None
|
|
272
305
|
|
|
273
306
|
|
|
274
307
|
def _to_params(props, neo4j) -> dict:
|
codeanalyzer/neo4j/cypher.py
CHANGED
|
@@ -28,14 +28,17 @@ from __future__ import annotations
|
|
|
28
28
|
from typing import Dict, List
|
|
29
29
|
|
|
30
30
|
from codeanalyzer.neo4j.rows import (
|
|
31
|
+
CAN_NODE,
|
|
31
32
|
EdgeRow,
|
|
32
33
|
GraphRows,
|
|
33
34
|
NodeRow,
|
|
35
|
+
application_prefix,
|
|
34
36
|
chunk,
|
|
35
37
|
cypher_map,
|
|
36
38
|
cypher_value,
|
|
37
39
|
)
|
|
38
40
|
from codeanalyzer.neo4j.schema import CONSTRAINTS, INDEXES
|
|
41
|
+
from codeanalyzer.schema.ids import application_id
|
|
39
42
|
|
|
40
43
|
BATCH = 500
|
|
41
44
|
|
|
@@ -66,13 +69,24 @@ def render_cypher(rows: GraphRows, app_name: str) -> str:
|
|
|
66
69
|
|
|
67
70
|
|
|
68
71
|
def _wipe(app_name: str) -> str:
|
|
69
|
-
|
|
72
|
+
"""The application root by equality plus everything under ``can://<app>/`` (#173).
|
|
73
|
+
Scoped by id prefix, so it is one application by construction — a second python
|
|
74
|
+
app sharing a module path, and a sibling analyzer's :Py* graph, are outside it.
|
|
75
|
+
:Package nodes (``pkg:`` purls) stay outside too.
|
|
76
|
+
|
|
77
|
+
Two things moved with the app-outermost grammar. The root is matched by its
|
|
78
|
+
``can://<app>`` id rather than by the free-text ``--app-name``, so two apps
|
|
79
|
+
sharing a name no longer wipe each other's root; and the app's :Artifact /
|
|
80
|
+
:ConfigKey nodes are now *inside* the prefix, so the snapshot rebuilds them
|
|
81
|
+
instead of leaving them to accumulate. That is deliberate, and it is the one
|
|
82
|
+
behavioural widening here: a cross-language edge into a shared :Artifact is
|
|
83
|
+
dropped by a python snapshot and restored on that analyzer's next push."""
|
|
84
|
+
prefix = cypher_value(application_prefix(app_name))
|
|
85
|
+
app_id = cypher_value(application_id(app_name))
|
|
70
86
|
return "\n".join(
|
|
71
87
|
[
|
|
72
|
-
f"MATCH (
|
|
73
|
-
"
|
|
74
|
-
"OPTIONAL MATCH (m)-[:PY_DECLARES|PY_HAS_METHOD|PY_HAS_ATTRIBUTE|PY_DECLARES_VAR|PY_HAS_CALLSITE*1..]->(x)",
|
|
75
|
-
"DETACH DELETE x, m, a;",
|
|
88
|
+
f"MATCH (x:{CAN_NODE}) WHERE x.id = {app_id} OR x.id STARTS WITH {prefix}",
|
|
89
|
+
"CALL { WITH x DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS;",
|
|
76
90
|
]
|
|
77
91
|
)
|
|
78
92
|
|