codeanalyzer-python 1.3.0__py3-none-any.whl → 1.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +7 -13
- codeanalyzer/artifacts/dependencies.py +3 -3
- codeanalyzer/artifacts/discovery.py +8 -34
- codeanalyzer/core.py +3 -4
- codeanalyzer/dataflow/builder.py +10 -2
- codeanalyzer/dataflow/identity.py +3 -3
- codeanalyzer/dataflow/sdg.py +17 -2
- codeanalyzer/dataflow/summaries.py +27 -4
- codeanalyzer/entrypoints/matching.py +36 -7
- codeanalyzer/entrypoints/pipeline.py +52 -4
- codeanalyzer/entrypoints/rules.py +12 -1
- codeanalyzer/entrypoints/rules.yml +34 -0
- codeanalyzer/neo4j/bolt.py +101 -45
- codeanalyzer/neo4j/cypher.py +10 -4
- codeanalyzer/neo4j/emit.py +3 -2
- codeanalyzer/neo4j/project.py +97 -42
- codeanalyzer/neo4j/rows.py +36 -1
- codeanalyzer/neo4j/schema.py +10 -8
- codeanalyzer/options/options.py +2 -3
- codeanalyzer/schema/ids.py +17 -0
- codeanalyzer/schema/l1_body.py +2 -0
- codeanalyzer/schema/py_schema.py +8 -3
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/METADATA +33 -18
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/RECORD +28 -28
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.3.0.dist-info → codeanalyzer_python-1.4.1.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/neo4j/bolt.py
CHANGED
|
@@ -27,11 +27,27 @@ Algorithm (the module subgraph is the unit of idempotent replacement):
|
|
|
27
27
|
4. upsert edges owned by changed modules (+ the shared edges).
|
|
28
28
|
5. on a FULL run only, prune modules whose source file vanished.
|
|
29
29
|
|
|
30
|
+
**A push never deletes by default** (#171). Steps 3 and 5 are the only destructive
|
|
31
|
+
ones and both run on ``eager`` (``--eager``) only; a default ``--lazy`` push is purely
|
|
32
|
+
additive — MERGE-upsert of nodes and edges, nothing removed. The cost of the default is
|
|
33
|
+
staleness: a declaration or a call edge the source no longer has stays in the graph until
|
|
34
|
+
an ``--eager`` push reconciles it. That is the deliberate trade — an incremental push into
|
|
35
|
+
a shared database should not be able to destroy anything, and the destructive rebuild is
|
|
36
|
+
opt-in under the same flag that already forces a clean analysis rebuild.
|
|
37
|
+
|
|
30
38
|
Nodes are MERGE-upserted, never blindly deleted, so a declaration another
|
|
31
39
|
(unchanged) module still references survives and its incoming edges stay valid.
|
|
32
|
-
``:PyExternal`` / ``:PyPackage`` / ``:PyDecorator``
|
|
40
|
+
``:PyExternal`` / ``:PyPackage`` / ``:PyDecorator`` have no owning module and are
|
|
33
41
|
MERGE-only.
|
|
34
42
|
|
|
43
|
+
**Every destructive statement is scoped on the ``can://`` id prefix** (#173). The id is a
|
|
44
|
+
path — ``can://python/<app>/<file>/...`` — so ``id = <module-id> OR id STARTS WITH
|
|
45
|
+
<module-id> + '/'`` is containment, and it is one language, one application and one
|
|
46
|
+
module at once. That is what neither a label anchor nor the retired ``_module`` property
|
|
47
|
+
could give: two python applications sharing ``src/foo.py`` carry identical labels and an
|
|
48
|
+
identical file key, and only the id tells them apart. ``:PyCanNode`` anchors the predicate
|
|
49
|
+
so it seeks an index instead of scanning the store; it carries no safety claim.
|
|
50
|
+
|
|
35
51
|
The ``neo4j`` driver is imported lazily so it stays an optional dependency and
|
|
36
52
|
off the default (json) output path entirely.
|
|
37
53
|
"""
|
|
@@ -40,16 +56,34 @@ from __future__ import annotations
|
|
|
40
56
|
from dataclasses import dataclass
|
|
41
57
|
from typing import Dict, List, Optional
|
|
42
58
|
|
|
43
|
-
from codeanalyzer.neo4j.rows import
|
|
59
|
+
from codeanalyzer.neo4j.rows import (
|
|
60
|
+
CAN_NODE, EdgeRow, GraphRows, NodeRow, application_prefix, chunk, descendant_prefix,
|
|
61
|
+
)
|
|
44
62
|
from codeanalyzer.neo4j.schema import CONSTRAINTS, INDEXES
|
|
45
63
|
from codeanalyzer.utils import logger
|
|
46
64
|
|
|
47
|
-
DESCENDANTS = (
|
|
48
|
-
"[:PY_DECLARES|PY_HAS_METHOD|PY_HAS_ATTRIBUTE|PY_DECLARES_VAR"
|
|
49
|
-
"|PY_HAS_CALLSITE|PY_HAS_BODY_NODE*1..]"
|
|
50
|
-
)
|
|
51
65
|
BATCH = 1000
|
|
52
66
|
|
|
67
|
+
# The per-module purge (#173): the module by equality, its subtree by prefix. Anchored
|
|
68
|
+
# on :PyCanNode only so the predicate can seek (see ``rows.CAN_NODE``).
|
|
69
|
+
PURGE_MODULE_EDGES = (
|
|
70
|
+
f"MATCH (x:{CAN_NODE}) WHERE x.id = $mid OR x.id STARTS WITH $pre "
|
|
71
|
+
"MATCH (x)-[r]->() DELETE r"
|
|
72
|
+
)
|
|
73
|
+
PURGE_VANISHED_NODES = (
|
|
74
|
+
f"MATCH (x:{CAN_NODE}) WHERE (x.id = $mid OR x.id STARTS WITH $pre) "
|
|
75
|
+
"AND NOT x.id IN $keys DETACH DELETE x"
|
|
76
|
+
)
|
|
77
|
+
# The orphan prune: modules inside this application's prefix that the run no longer
|
|
78
|
+
# emits, and everything under each. Batched — deleting a large application in one
|
|
79
|
+
# transaction exhausts dbms.memory.transaction.total.max (typescript#116).
|
|
80
|
+
PRUNE_VANISHED_MODULES = (
|
|
81
|
+
f"MATCH (m:PyModule:{CAN_NODE}) WHERE m.id STARTS WITH $app AND NOT m.id IN $present "
|
|
82
|
+
f"CALL {{ WITH m MATCH (x:{CAN_NODE}) WHERE x.id = m.id OR x.id STARTS WITH m.id + '/' "
|
|
83
|
+
"DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS "
|
|
84
|
+
"RETURN count(*) AS pruned"
|
|
85
|
+
)
|
|
86
|
+
|
|
53
87
|
|
|
54
88
|
@dataclass
|
|
55
89
|
class BoltConfig:
|
|
@@ -59,7 +93,7 @@ class BoltConfig:
|
|
|
59
93
|
database: Optional[str] = None
|
|
60
94
|
|
|
61
95
|
|
|
62
|
-
def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
96
|
+
def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool, eager: bool = False) -> None:
|
|
63
97
|
try:
|
|
64
98
|
import neo4j # noqa: WPS433 (lazy, optional dependency)
|
|
65
99
|
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
@@ -80,35 +114,44 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
|
80
114
|
for stmt in [*CONSTRAINTS, *INDEXES]:
|
|
81
115
|
s.run(stmt)
|
|
82
116
|
|
|
83
|
-
# The application anchor
|
|
84
|
-
#
|
|
117
|
+
# The application anchor. Every destructive statement below is scoped to
|
|
118
|
+
# ``can://python/<app>/``; an empty application id is refused up front rather
|
|
119
|
+
# than becoming ``STARTS WITH ''`` (every node in the database).
|
|
85
120
|
app_name = next(
|
|
86
121
|
(n.value for n in rows.nodes if n.labels and n.labels[0] == "PyApplication"),
|
|
87
122
|
None,
|
|
88
123
|
)
|
|
124
|
+
app_prefix = application_prefix(app_name)
|
|
89
125
|
|
|
90
|
-
# Partition nodes by owning module
|
|
126
|
+
# Partition nodes by owning module (an in-memory field, never emitted, #173);
|
|
127
|
+
# shared nodes have none.
|
|
91
128
|
by_module: Dict[str, List[NodeRow]] = {}
|
|
92
129
|
shared: List[NodeRow] = []
|
|
93
130
|
module_of: Dict[str, str] = {} # node value → owning module
|
|
94
131
|
for n in rows.nodes:
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
module_of[n.value] = m
|
|
132
|
+
if n.module is not None:
|
|
133
|
+
by_module.setdefault(n.module, []).append(n)
|
|
134
|
+
module_of[n.value] = n.module
|
|
99
135
|
else:
|
|
100
136
|
shared.append(n)
|
|
101
137
|
|
|
102
|
-
# 2. diff content_hash.
|
|
138
|
+
# 2. diff content_hash, keyed by module id inside this application's prefix.
|
|
139
|
+
# Keyed by file key it was application-blind: a second application whose
|
|
140
|
+
# module shares the path and the hash looked "unchanged" and was never written.
|
|
103
141
|
db_hash: Dict[str, Optional[str]] = {}
|
|
104
142
|
with session() as s:
|
|
105
|
-
res = s.run(
|
|
143
|
+
res = s.run(
|
|
144
|
+
f"MATCH (m:PyModule:{CAN_NODE}) WHERE m.id STARTS WITH $app "
|
|
145
|
+
"RETURN m.id AS k, m.content_hash AS h",
|
|
146
|
+
app=app_prefix,
|
|
147
|
+
)
|
|
106
148
|
for rec in res:
|
|
107
149
|
db_hash[rec["k"]] = rec["h"]
|
|
108
150
|
changed = set()
|
|
109
151
|
for m, nodes in by_module.items():
|
|
110
|
-
|
|
111
|
-
|
|
152
|
+
mid = _module_id_of(nodes)
|
|
153
|
+
row_hash = _hash_of(nodes)
|
|
154
|
+
if mid not in db_hash or row_hash is None or row_hash != db_hash.get(mid):
|
|
112
155
|
changed.add(m)
|
|
113
156
|
logger.info(
|
|
114
157
|
f"neo4j(bolt): {len(by_module)} modules ({len(changed)} changed), "
|
|
@@ -119,19 +162,26 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
|
119
162
|
_upsert_nodes(session, neo4j, shared)
|
|
120
163
|
|
|
121
164
|
# 4. per changed module: purge owned edges + vanished decls, then upsert its nodes.
|
|
165
|
+
# The purge is the only destructive step in a push, so it runs on --eager only.
|
|
122
166
|
for m in changed:
|
|
123
167
|
nodes = by_module[m]
|
|
124
168
|
keys = [n.value for n in nodes]
|
|
169
|
+
if not eager:
|
|
170
|
+
_upsert_nodes(session, neo4j, nodes)
|
|
171
|
+
continue
|
|
172
|
+
# The module id comes from the module's own row, never by splitting a
|
|
173
|
+
# declaration's id: a file key may itself contain '/'.
|
|
174
|
+
module_id = _module_id_of(nodes)
|
|
175
|
+
if module_id is None or not module_id.startswith(app_prefix):
|
|
176
|
+
raise ValueError(
|
|
177
|
+
f"neo4j: module {m!r} has no can:// id under {app_prefix!r}; "
|
|
178
|
+
"refusing to purge"
|
|
179
|
+
)
|
|
125
180
|
with session() as s:
|
|
126
|
-
def _purge(tx,
|
|
127
|
-
|
|
128
|
-
tx.run(
|
|
129
|
-
|
|
130
|
-
"WHERE NOT coalesce(x.signature, x.id, x.file_key) IN $keys "
|
|
131
|
-
"DETACH DELETE x",
|
|
132
|
-
m=module,
|
|
133
|
-
keys=node_keys,
|
|
134
|
-
)
|
|
181
|
+
def _purge(tx, mid=module_id, node_keys=keys):
|
|
182
|
+
params = {"mid": mid, "pre": descendant_prefix(mid)}
|
|
183
|
+
tx.run(PURGE_MODULE_EDGES, **params)
|
|
184
|
+
tx.run(PURGE_VANISHED_NODES, keys=node_keys, **params)
|
|
135
185
|
|
|
136
186
|
s.execute_write(_purge)
|
|
137
187
|
_upsert_nodes(session, neo4j, nodes)
|
|
@@ -145,22 +195,21 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
|
145
195
|
_upsert_edges(session, neo4j, edges)
|
|
146
196
|
|
|
147
197
|
# 6. orphan prune — only safe on a full run (a targeted run can't tell deleted from untargeted).
|
|
148
|
-
#
|
|
149
|
-
#
|
|
150
|
-
|
|
151
|
-
|
|
198
|
+
# Scoped to ``can://python/<app>/`` so a full run for application B never deletes
|
|
199
|
+
# application A's modules from a shared database — even when both are python and
|
|
200
|
+
# share a module path.
|
|
201
|
+
if full_run and eager:
|
|
202
|
+
present = [mid for mid in (_module_id_of(ns) for ns in by_module.values()) if mid]
|
|
152
203
|
with session() as s:
|
|
153
|
-
res = s.run(
|
|
154
|
-
"MATCH (:PyApplication {name: $app})-[:PY_HAS_MODULE]->(m:PyModule) "
|
|
155
|
-
"WHERE NOT m.file_key IN $present "
|
|
156
|
-
f"OPTIONAL MATCH (m)-{DESCENDANTS}->(x) DETACH DELETE x, m "
|
|
157
|
-
"RETURN count(m) AS pruned",
|
|
158
|
-
app=app_name,
|
|
159
|
-
present=present,
|
|
160
|
-
)
|
|
204
|
+
res = s.run(PRUNE_VANISHED_MODULES, app=app_prefix, present=present)
|
|
161
205
|
pruned = res.single()
|
|
162
206
|
pruned_count = pruned["pruned"] if pruned else 0
|
|
163
207
|
logger.info(f"neo4j(bolt): pruned {pruned_count} vanished module(s)")
|
|
208
|
+
elif not eager:
|
|
209
|
+
logger.info(
|
|
210
|
+
"neo4j(bolt): additive push (--lazy) — nothing deleted; "
|
|
211
|
+
"re-run with --eager to reconcile removed declarations and edges"
|
|
212
|
+
)
|
|
164
213
|
else:
|
|
165
214
|
logger.info(
|
|
166
215
|
"neo4j(bolt): targeted run — orphan pruning skipped (deleted files not removed)"
|
|
@@ -234,12 +283,19 @@ def _upsert_edges(session, neo4j, edges: List[EdgeRow]) -> None:
|
|
|
234
283
|
# ----------------------------------------------------------------------------------------------
|
|
235
284
|
|
|
236
285
|
|
|
237
|
-
def
|
|
238
|
-
for n in nodes
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
286
|
+
def _module_row(nodes: List[NodeRow]) -> Optional[NodeRow]:
|
|
287
|
+
return next((n for n in nodes if n.labels[0] == "PyModule"), None)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _module_id_of(nodes: List[NodeRow]) -> Optional[str]:
|
|
291
|
+
row = _module_row(nodes)
|
|
292
|
+
return row.value if row is not None else None
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _hash_of(nodes: List[NodeRow]) -> Optional[str]:
|
|
296
|
+
row = _module_row(nodes)
|
|
297
|
+
h = row.props.get("content_hash") if row is not None else None
|
|
298
|
+
return h if isinstance(h, str) else None
|
|
243
299
|
|
|
244
300
|
|
|
245
301
|
def _to_params(props, neo4j) -> dict:
|
codeanalyzer/neo4j/cypher.py
CHANGED
|
@@ -28,9 +28,11 @@ from __future__ import annotations
|
|
|
28
28
|
from typing import Dict, List
|
|
29
29
|
|
|
30
30
|
from codeanalyzer.neo4j.rows import (
|
|
31
|
+
CAN_NODE,
|
|
31
32
|
EdgeRow,
|
|
32
33
|
GraphRows,
|
|
33
34
|
NodeRow,
|
|
35
|
+
application_prefix,
|
|
34
36
|
chunk,
|
|
35
37
|
cypher_map,
|
|
36
38
|
cypher_value,
|
|
@@ -66,13 +68,17 @@ def render_cypher(rows: GraphRows, app_name: str) -> str:
|
|
|
66
68
|
|
|
67
69
|
|
|
68
70
|
def _wipe(app_name: str) -> str:
|
|
71
|
+
"""Everything under ``can://python/<app>/`` plus the application anchor (#173).
|
|
72
|
+
Scoped by id prefix, so it is one language and one application by construction —
|
|
73
|
+
a second python app sharing a module path, a sibling analyzer's graph, and the
|
|
74
|
+
cross-language :Artifact / :Package nodes are all outside it."""
|
|
75
|
+
prefix = cypher_value(application_prefix(app_name))
|
|
69
76
|
name = cypher_value(app_name)
|
|
70
77
|
return "\n".join(
|
|
71
78
|
[
|
|
72
|
-
f"MATCH (
|
|
73
|
-
"
|
|
74
|
-
"
|
|
75
|
-
"DETACH DELETE x, m, a;",
|
|
79
|
+
f"MATCH (x:{CAN_NODE}) WHERE x.id STARTS WITH {prefix}",
|
|
80
|
+
"CALL { WITH x DETACH DELETE x } IN TRANSACTIONS OF 1000 ROWS;",
|
|
81
|
+
f"MATCH (a:PyApplication {{name: {name}}}) DETACH DELETE a;",
|
|
76
82
|
]
|
|
77
83
|
)
|
|
78
84
|
|
codeanalyzer/neo4j/emit.py
CHANGED
|
@@ -67,9 +67,10 @@ def emit_neo4j(analysis: Analysis, options: AnalysisOptions) -> None:
|
|
|
67
67
|
password=options.neo4j_password,
|
|
68
68
|
database=options.neo4j_database,
|
|
69
69
|
)
|
|
70
|
-
# A full run (no single-file restriction) makes orphan pruning safe
|
|
70
|
+
# A full run (no single-file restriction) makes orphan pruning safe; --eager
|
|
71
|
+
# is what permits any deletion at all (#171).
|
|
71
72
|
full_run = options.file_name is None
|
|
72
|
-
bolt_writer(rows, cfg, full_run)
|
|
73
|
+
bolt_writer(rows, cfg, full_run, eager=options.rebuild_analysis)
|
|
73
74
|
return
|
|
74
75
|
|
|
75
76
|
out_dir = options.output if options.output is not None else Path.cwd()
|
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -27,14 +27,15 @@ Modelling decisions (mirror of the TypeScript backend):
|
|
|
27
27
|
- call-graph endpoints absent from the symbol table become ``:PyExternal`` ghost
|
|
28
28
|
nodes, so RPC / third-party / framework edges are preserved (matching the
|
|
29
29
|
analyzer's own ghost-node behaviour).
|
|
30
|
-
- every project-owned node
|
|
30
|
+
- every project-owned node names its owning module (``_module`` in the props it
|
|
31
|
+
hands RowBuilder, lifted to ``NodeRow.module`` and never emitted, #173), so
|
|
31
32
|
the incremental writer can delete exactly what a re-analyzed module emitted.
|
|
32
33
|
"""
|
|
33
34
|
from __future__ import annotations
|
|
34
35
|
|
|
35
36
|
import json
|
|
36
37
|
from pathlib import Path
|
|
37
|
-
from typing import Any, List, Optional
|
|
38
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
38
39
|
|
|
39
40
|
from codeanalyzer.neo4j.schema import SCHEMA_VERSION
|
|
40
41
|
from codeanalyzer.neo4j.rows import GraphRows, NodeRef, Props, RowBuilder, prune
|
|
@@ -47,7 +48,8 @@ from codeanalyzer.schema import (
|
|
|
47
48
|
PyModule,
|
|
48
49
|
PyVariableDeclaration,
|
|
49
50
|
)
|
|
50
|
-
from codeanalyzer.schema
|
|
51
|
+
from codeanalyzer.schema import model_dump
|
|
52
|
+
from codeanalyzer.schema.ids import application_id, global_ordinal, purl_pypi
|
|
51
53
|
from codeanalyzer.schema.py_schema import PyDecorator
|
|
52
54
|
|
|
53
55
|
|
|
@@ -70,6 +72,13 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
70
72
|
"repo_uri": app.repository.uri if app.repository else None,
|
|
71
73
|
"source_revision": app.repository.revision if app.repository else None,
|
|
72
74
|
"repo_dirty": app.repository.dirty if app.repository else None,
|
|
75
|
+
# #177: the entrypoint pass under-approximates by design, so a
|
|
76
|
+
# graph consumer must be able to tell "no entrypoints" from "the
|
|
77
|
+
# pass found nothing". Always present, even when empty.
|
|
78
|
+
"entrypoint_frameworks": list(app.entrypoint_report.frameworks_detected),
|
|
79
|
+
"entrypoint_report_json": json.dumps(
|
|
80
|
+
model_dump(app.entrypoint_report, mode="json"), sort_keys=True
|
|
81
|
+
),
|
|
73
82
|
}
|
|
74
83
|
),
|
|
75
84
|
)
|
|
@@ -87,19 +96,21 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
87
96
|
for file_key, mod in app.symbol_table.items():
|
|
88
97
|
mod_ref = b.node(["PyModule"], "id", mod.id, _module_props(mod, file_key))
|
|
89
98
|
b.edge("PY_HAS_MODULE", app_ref, mod_ref)
|
|
90
|
-
_project_module_body(b, file_key, mod_ref, mod, externals, sig_to_id, module_id_by_key
|
|
99
|
+
_project_module_body(b, file_key, mod_ref, mod, externals, sig_to_id, module_id_by_key,
|
|
100
|
+
application_id(app_name))
|
|
91
101
|
|
|
92
102
|
# The aggregated :PY_CALLS twin.
|
|
103
|
+
app_can_id = application_id(app_name)
|
|
93
104
|
for e in app.call_graph:
|
|
94
|
-
src = _call_endpoint(b, e.src, externals, sig_to_id)
|
|
95
|
-
tgt = _call_endpoint(b, e.dst, externals, sig_to_id)
|
|
105
|
+
src = _call_endpoint(b, e.src, externals, sig_to_id, app_can_id)
|
|
106
|
+
tgt = _call_endpoint(b, e.dst, externals, sig_to_id, app_can_id)
|
|
96
107
|
b.edge(
|
|
97
108
|
"PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.prov or []))
|
|
98
109
|
)
|
|
99
110
|
|
|
100
111
|
# Level-3 CPG overlay: each callable's v2 body/cfg/cdg/ddg. Idempotent under
|
|
101
112
|
# MERGE — a no-op when no callable carries L3 fields (levels 1/2).
|
|
102
|
-
_project_program_graphs(b, app, externals, sig_to_id)
|
|
113
|
+
_project_program_graphs(b, app, externals, sig_to_id, app_can_id)
|
|
103
114
|
|
|
104
115
|
# Neutral artifact/dependency subgraph (Task 6). L1 data — always present,
|
|
105
116
|
# full-depth-always regardless of -a.
|
|
@@ -109,7 +120,7 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
109
120
|
# _project_program_graphs above) into the config-key subgraph
|
|
110
121
|
# (ConfigKey from _project_artifacts above), plus first-class unresolved
|
|
111
122
|
# reads.
|
|
112
|
-
_project_config_uses(b, app, app_ref, externals, sig_to_id)
|
|
123
|
+
_project_config_uses(b, app, app_ref, externals, sig_to_id, app_can_id)
|
|
113
124
|
|
|
114
125
|
return b.finish()
|
|
115
126
|
|
|
@@ -128,11 +139,7 @@ def _global_ordinal(callable_id: str, local_key: str) -> str:
|
|
|
128
139
|
This MUST agree with :meth:`IdentityMap.global_id` for the same node, so the
|
|
129
140
|
JSON ``body``/``cfg`` projection and this Neo4j projection land on one node
|
|
130
141
|
identity (two-projection agreement)."""
|
|
131
|
-
return (
|
|
132
|
-
f"{callable_id}{local_key}"
|
|
133
|
-
if local_key.startswith("@")
|
|
134
|
-
else f"{callable_id}@{local_key}"
|
|
135
|
-
)
|
|
142
|
+
return global_ordinal(callable_id, local_key)
|
|
136
143
|
|
|
137
144
|
|
|
138
145
|
def _body_ref(callable_id: str, local_key: str) -> NodeRef:
|
|
@@ -140,7 +147,7 @@ def _body_ref(callable_id: str, local_key: str) -> NodeRef:
|
|
|
140
147
|
|
|
141
148
|
|
|
142
149
|
def _project_program_graphs(
|
|
143
|
-
b: RowBuilder, app: PyApplication, externals: dict, sig_to_id: dict
|
|
150
|
+
b: RowBuilder, app: PyApplication, externals: dict, sig_to_id: dict, app_can_id: str,
|
|
144
151
|
) -> None:
|
|
145
152
|
"""Level-3 CPG overlay, projected off each callable's v2 ``body``/``cfg``/
|
|
146
153
|
``cdg``/``ddg`` (populated by ``emit_l3_body`` at ``-a 3``; empty otherwise).
|
|
@@ -209,7 +216,7 @@ def _project_program_graphs(
|
|
|
209
216
|
b.edge(
|
|
210
217
|
"PY_RESOLVES_TO",
|
|
211
218
|
ref,
|
|
212
|
-
_call_endpoint(b, node.callee, externals, sig_to_id),
|
|
219
|
+
_call_endpoint(b, node.callee, externals, sig_to_id, app_can_id),
|
|
213
220
|
)
|
|
214
221
|
for e in c.cfg or []:
|
|
215
222
|
# kind-discriminated: a conditional's true/false pair between one
|
|
@@ -309,7 +316,6 @@ def _project_artifacts(b: RowBuilder, app: PyApplication, app_name: str, app_ref
|
|
|
309
316
|
"size_bytes": art.size_bytes,
|
|
310
317
|
"sha256": art.sha256,
|
|
311
318
|
"source": art.source,
|
|
312
|
-
"text_truncated": art.text_truncated,
|
|
313
319
|
"extraction": art.extraction,
|
|
314
320
|
}
|
|
315
321
|
),
|
|
@@ -400,6 +406,7 @@ def _project_artifacts(b: RowBuilder, app: PyApplication, app_name: str, app_ref
|
|
|
400
406
|
|
|
401
407
|
def _project_config_uses(
|
|
402
408
|
b: RowBuilder, app: PyApplication, app_ref: NodeRef, externals: dict, sig_to_id: dict,
|
|
409
|
+
app_can_id: str,
|
|
403
410
|
) -> None:
|
|
404
411
|
"""config_use (#162): PY_USES_CONFIG (`app.config_uses`) and
|
|
405
412
|
PY_READS_CONFIG_UNRESOLVED (`app.config_reads_unresolved`).
|
|
@@ -429,7 +436,7 @@ def _project_config_uses(
|
|
|
429
436
|
prune({"prov": list(e.prov) if e.prov else None}),
|
|
430
437
|
)
|
|
431
438
|
for r in app.config_reads_unresolved:
|
|
432
|
-
ghost_ref = _call_endpoint(b, r.callee, externals, sig_to_id)
|
|
439
|
+
ghost_ref = _call_endpoint(b, r.callee, externals, sig_to_id, app_can_id)
|
|
433
440
|
b.edge(
|
|
434
441
|
"PY_READS_CONFIG_UNRESOLVED",
|
|
435
442
|
app_ref,
|
|
@@ -454,8 +461,54 @@ def _symbol_ref(signature: str, externals: dict, sig_to_id: dict) -> NodeRef:
|
|
|
454
461
|
return NodeRef("PySymbol", "signature", signature)
|
|
455
462
|
|
|
456
463
|
|
|
464
|
+
def _base_ref_resolver(
|
|
465
|
+
b: RowBuilder, mod: PyModule, externals: dict, sig_to_id: dict, app_can_id: str,
|
|
466
|
+
) -> Callable[[str], NodeRef]:
|
|
467
|
+
"""Per-module: the written base spelling → the NodeRef PY_EXTENDS lands on (#178).
|
|
468
|
+
|
|
469
|
+
``base_classes`` stores the spelling as written (``Base``, ``views.View``),
|
|
470
|
+
while ``sig_to_id`` is keyed by signature (``pkg.mod.Base``), so the two never
|
|
471
|
+
met and every PY_EXTENDS row was dropped as dangling. Resolution order: a class
|
|
472
|
+
declared in this module (bare name or ``Outer.Inner`` path) → its can:// id; a
|
|
473
|
+
name the module's import table maps (same resolver the entrypoint pass uses)
|
|
474
|
+
that is a declared class elsewhere → its can:// id; otherwise an ``@external``
|
|
475
|
+
ghost with the id shape ``_home_external_symbols`` uses, so a call to the same
|
|
476
|
+
symbol MERGEs onto the same node."""
|
|
477
|
+
from codeanalyzer.entrypoints.pipeline import _base_resolver
|
|
478
|
+
|
|
479
|
+
local: Dict[str, str] = {}
|
|
480
|
+
|
|
481
|
+
def index(cl: PyClass, path: str) -> None:
|
|
482
|
+
local.setdefault(cl.name, cl.signature)
|
|
483
|
+
local[path] = cl.signature
|
|
484
|
+
for ic in (cl.types or {}).values():
|
|
485
|
+
index(ic, f"{path}.{ic.name}")
|
|
486
|
+
|
|
487
|
+
for cl in (mod.types or {}).values():
|
|
488
|
+
index(cl, cl.name)
|
|
489
|
+
resolve = _base_resolver(mod)
|
|
490
|
+
|
|
491
|
+
def base_ref(written: str) -> NodeRef:
|
|
492
|
+
sig = local.get(written) or resolve(written)
|
|
493
|
+
can_id = sig_to_id.get(sig)
|
|
494
|
+
if can_id is not None:
|
|
495
|
+
return _sym(can_id)
|
|
496
|
+
return _external_ghost(b, app_can_id, sig)
|
|
497
|
+
|
|
498
|
+
return base_ref
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _external_ghost(b: RowBuilder, app_can_id: str, signature: str) -> NodeRef:
|
|
502
|
+
"""A :PyExternal ghost for a dotted signature nobody homed, with the id shape
|
|
503
|
+
``_home_external_symbols`` uses — ``<app>/@external/<module>/<name>`` — so it
|
|
504
|
+
sits inside the application prefix (#173) and MERGEs with a homed twin."""
|
|
505
|
+
module, name = signature.rsplit(".", 1) if "." in signature else (None, signature)
|
|
506
|
+
ext_id = f"{app_can_id}/@external/{module}/{name}" if module else f"{app_can_id}/@external/{name}"
|
|
507
|
+
return b.node(["PySymbol", "PyExternal"], "id", ext_id, prune({"name": name, "module": module}))
|
|
508
|
+
|
|
509
|
+
|
|
457
510
|
def _call_endpoint(
|
|
458
|
-
b: RowBuilder, signature: str, externals: dict, sig_to_id: dict
|
|
511
|
+
b: RowBuilder, signature: str, externals: dict, sig_to_id: dict, app_can_id: str,
|
|
459
512
|
) -> NodeRef:
|
|
460
513
|
"""A call-graph endpoint: a declared callable already emitted (keyed by its
|
|
461
514
|
canonical ``can://`` id, resolved through ``sig_to_id``), or an external symbol
|
|
@@ -485,13 +538,7 @@ def _call_endpoint(
|
|
|
485
538
|
ext.id or signature,
|
|
486
539
|
prune({"name": ext.name, "module": ext.module}),
|
|
487
540
|
)
|
|
488
|
-
|
|
489
|
-
return b.node(
|
|
490
|
-
["PySymbol", "PyExternal"],
|
|
491
|
-
"id",
|
|
492
|
-
signature,
|
|
493
|
-
prune({"name": name}),
|
|
494
|
-
)
|
|
541
|
+
return _external_ghost(b, app_can_id, signature)
|
|
495
542
|
|
|
496
543
|
|
|
497
544
|
# ----------------------------------------------------------------------------------------------
|
|
@@ -501,16 +548,17 @@ def _call_endpoint(
|
|
|
501
548
|
|
|
502
549
|
def _project_module_body(
|
|
503
550
|
b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: PyModule,
|
|
504
|
-
externals: dict, sig_to_id: dict, module_id_by_key: dict,
|
|
551
|
+
externals: dict, sig_to_id: dict, module_id_by_key: dict, app_can_id: str,
|
|
505
552
|
) -> None:
|
|
553
|
+
base_ref = _base_ref_resolver(b, mod, externals, sig_to_id, app_can_id)
|
|
506
554
|
for fn in (mod.functions or {}).values():
|
|
507
555
|
_project_callable(b, file_key, mod_ref, "PY_DECLARES", fn, externals, sig_to_id,
|
|
508
|
-
mod.source)
|
|
556
|
+
mod.source, base_ref)
|
|
509
557
|
for cl in (mod.types or {}).values():
|
|
510
558
|
_project_class(b, file_key, mod_ref, "PY_DECLARES", cl, externals, sig_to_id,
|
|
511
|
-
mod.source)
|
|
559
|
+
mod.source, base_ref)
|
|
512
560
|
for v in mod.variables or []:
|
|
513
|
-
_project_variable(b, file_key, mod_ref,
|
|
561
|
+
_project_variable(b, file_key, mod_ref, v)
|
|
514
562
|
_project_imports(b, mod_ref, mod, module_id_by_key)
|
|
515
563
|
|
|
516
564
|
|
|
@@ -576,7 +624,7 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule,
|
|
|
576
624
|
|
|
577
625
|
def _project_class(
|
|
578
626
|
b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass,
|
|
579
|
-
externals: dict, sig_to_id: dict, source: str,
|
|
627
|
+
externals: dict, sig_to_id: dict, source: str, base_ref: Callable[[str], NodeRef],
|
|
580
628
|
) -> None:
|
|
581
629
|
ref = b.node(
|
|
582
630
|
["PySymbol", "PyClass"], "id", cl.id, _class_props(cl, file_key, source)
|
|
@@ -588,20 +636,21 @@ def _project_class(
|
|
|
588
636
|
|
|
589
637
|
for base in cl.base_classes or []:
|
|
590
638
|
if base:
|
|
591
|
-
b.edge_to_symbol("PY_EXTENDS", ref,
|
|
639
|
+
b.edge_to_symbol("PY_EXTENDS", ref, base_ref(base))
|
|
592
640
|
|
|
593
641
|
for m in (cl.callables or {}).values():
|
|
594
642
|
_project_callable(b, file_key, ref, "PY_HAS_METHOD", m, externals, sig_to_id,
|
|
595
|
-
source)
|
|
643
|
+
source, base_ref)
|
|
596
644
|
for a in (cl.attributes or {}).values():
|
|
597
|
-
_project_attribute(b, file_key, ref,
|
|
645
|
+
_project_attribute(b, file_key, ref, a)
|
|
598
646
|
for ic in (cl.types or {}).values():
|
|
599
|
-
_project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id, source
|
|
647
|
+
_project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id, source,
|
|
648
|
+
base_ref)
|
|
600
649
|
|
|
601
650
|
|
|
602
651
|
def _project_callable(
|
|
603
652
|
b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable,
|
|
604
|
-
externals: dict, sig_to_id: dict, source: str,
|
|
653
|
+
externals: dict, sig_to_id: dict, source: str, base_ref: Callable[[str], NodeRef],
|
|
605
654
|
) -> None:
|
|
606
655
|
ref = b.node(
|
|
607
656
|
["PySymbol", "PyCallable"],
|
|
@@ -615,18 +664,22 @@ def _project_callable(
|
|
|
615
664
|
_project_decorator(b, ref, d)
|
|
616
665
|
|
|
617
666
|
for v in c.local_variables or []:
|
|
618
|
-
_project_variable(b, file_key, ref,
|
|
667
|
+
_project_variable(b, file_key, ref, v)
|
|
619
668
|
for ic in (c.callables or {}).values():
|
|
620
669
|
_project_callable(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id,
|
|
621
|
-
source)
|
|
670
|
+
source, base_ref)
|
|
622
671
|
for cl in (c.types or {}).values():
|
|
623
|
-
_project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id, source
|
|
672
|
+
_project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id, source,
|
|
673
|
+
base_ref)
|
|
624
674
|
|
|
625
675
|
|
|
626
676
|
def _project_attribute(
|
|
627
|
-
b: RowBuilder, file_key: str, owner: NodeRef,
|
|
677
|
+
b: RowBuilder, file_key: str, owner: NodeRef, a: PyClassAttribute
|
|
628
678
|
) -> None:
|
|
629
|
-
|
|
679
|
+
# ``<class can:// id>/<name>`` (#173): minted from the owner's id so it carries
|
|
680
|
+
# the application segment. The signature-minted ``service.Service.name`` it
|
|
681
|
+
# replaced was identical across applications, so two apps MERGEd onto one node.
|
|
682
|
+
attr_id = f"{owner.value}/{a.name}"
|
|
630
683
|
ref = b.node(["PyAttribute"], "id", attr_id, _attribute_props(a, attr_id, file_key))
|
|
631
684
|
b.edge("PY_HAS_ATTRIBUTE", owner, ref)
|
|
632
685
|
|
|
@@ -635,10 +688,12 @@ def _project_variable(
|
|
|
635
688
|
b: RowBuilder,
|
|
636
689
|
file_key: str,
|
|
637
690
|
owner: NodeRef,
|
|
638
|
-
owner_id: str,
|
|
639
691
|
v: PyVariableDeclaration,
|
|
640
692
|
) -> None:
|
|
641
|
-
|
|
693
|
+
# ``<owner can:// id>/<name>@<line>`` (#173) — the owner is the module or the
|
|
694
|
+
# callable, so a module-level variable sits under ``<module-id>/`` like every
|
|
695
|
+
# other declaration and the module's prefix purge reaches it.
|
|
696
|
+
var_id = f"{owner.value}/{v.name}@{v.start_line}"
|
|
642
697
|
ref = b.node(["PyVariable"], "id", var_id, _variable_props(v, var_id, file_key))
|
|
643
698
|
b.edge("PY_DECLARES_VAR", owner, ref)
|
|
644
699
|
|
codeanalyzer/neo4j/rows.py
CHANGED
|
@@ -50,6 +50,35 @@ class NodeRow:
|
|
|
50
50
|
key_prop: str
|
|
51
51
|
value: str
|
|
52
52
|
props: Props
|
|
53
|
+
# The owning module's file key, for the incremental writer's per-module diff.
|
|
54
|
+
# In memory only (#173): it used to be emitted as ``_module`` and every
|
|
55
|
+
# destructive statement matched on it, which is application-blind. Scope now
|
|
56
|
+
# comes from the ``can://`` id prefix; this field only groups rows.
|
|
57
|
+
module: Optional[str] = None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# The marker label on every node keyed by a ``can://python/`` id (#173). It is an
|
|
61
|
+
# INDEX ANCHOR, nothing more: Neo4j property indexes are label-scoped, so the
|
|
62
|
+
# prefix predicate ``id STARTS WITH $p`` needs a label to seek on. Safety comes
|
|
63
|
+
# from the prefix, which carries language, application and module.
|
|
64
|
+
CAN_NODE = "PyCanNode"
|
|
65
|
+
_PY_CAN_PREFIX = "can://python/"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def descendant_prefix(can_id: str) -> str:
|
|
69
|
+
"""The prefix that matches a node's descendants and nothing else. The separator
|
|
70
|
+
is the point: ``can://python/app/src/foo.py`` is also a prefix of
|
|
71
|
+
``can://python/app/src/foo.pyX``, so descendants match on ``id + '/'`` and the
|
|
72
|
+
node itself by equality."""
|
|
73
|
+
return f"{can_id}/"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def application_prefix(app_name: Optional[str]) -> str:
|
|
77
|
+
"""``can://python/<app>/`` — the scope of every destructive statement. Refuses an
|
|
78
|
+
empty application: ``STARTS WITH ''`` would match every node in the database."""
|
|
79
|
+
if not app_name:
|
|
80
|
+
raise ValueError("neo4j: refusing a destructive statement without an application id")
|
|
81
|
+
return descendant_prefix(f"{_PY_CAN_PREFIX}{app_name}")
|
|
53
82
|
|
|
54
83
|
|
|
55
84
|
@dataclass
|
|
@@ -100,14 +129,20 @@ class RowBuilder:
|
|
|
100
129
|
(last write wins) and unions labels — the in-memory analog of
|
|
101
130
|
``MERGE (n:Label {key}) SET n += props``."""
|
|
102
131
|
node_id = f"{labels[0]} {value}"
|
|
132
|
+
props = dict(props)
|
|
133
|
+
module = props.pop("_module", None) # lifted off the graph (#173)
|
|
134
|
+
if key_prop == "id" and value.startswith(_PY_CAN_PREFIX) and CAN_NODE not in labels:
|
|
135
|
+
labels = [*labels, CAN_NODE]
|
|
103
136
|
existing = self._nodes.get(node_id)
|
|
104
137
|
if existing is not None:
|
|
105
138
|
existing.props.update(props)
|
|
139
|
+
if module is not None:
|
|
140
|
+
existing.module = module
|
|
106
141
|
for label in labels:
|
|
107
142
|
if label not in existing.labels:
|
|
108
143
|
existing.labels.append(label)
|
|
109
144
|
else:
|
|
110
|
-
self._nodes[node_id] = NodeRow(list(labels), key_prop, value,
|
|
145
|
+
self._nodes[node_id] = NodeRow(list(labels), key_prop, value, props, module)
|
|
111
146
|
self._keys.add((labels[0], value))
|
|
112
147
|
return NodeRef(labels[0], key_prop, value)
|
|
113
148
|
|