codeanalyzer-python 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +24 -1
- codeanalyzer/artifacts/__init__.py +20 -0
- codeanalyzer/artifacts/config_keys.py +588 -0
- codeanalyzer/artifacts/config_use.py +597 -0
- codeanalyzer/artifacts/config_use_rules.yml +58 -0
- codeanalyzer/artifacts/dependencies.py +237 -0
- codeanalyzer/artifacts/discovery.py +141 -0
- codeanalyzer/artifacts/parsers.py +248 -0
- codeanalyzer/core.py +90 -0
- codeanalyzer/dataflow/builder.py +22 -3
- codeanalyzer/dataflow/sdg.py +17 -2
- codeanalyzer/dataflow/summaries.py +27 -4
- codeanalyzer/neo4j/bolt.py +35 -6
- codeanalyzer/neo4j/emit.py +3 -2
- codeanalyzer/neo4j/project.py +191 -0
- codeanalyzer/neo4j/schema.py +74 -0
- codeanalyzer/options/options.py +4 -0
- codeanalyzer/schema/ids.py +21 -0
- codeanalyzer/schema/py_schema.py +117 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +11 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.4.0.dist-info}/METADATA +117 -7
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.4.0.dist-info}/RECORD +26 -19
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.4.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.4.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.4.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.4.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/core.py
CHANGED
|
@@ -37,6 +37,21 @@ from codeanalyzer.utils import ProgressBar
|
|
|
37
37
|
from codeanalyzer.options import AnalysisOptions
|
|
38
38
|
from codeanalyzer.provenance import analyzer_info, repository_info
|
|
39
39
|
|
|
40
|
+
def _artifact_full_text(project_dir: Path, path: str, art) -> str:
|
|
41
|
+
"""Mirrors ``artifacts.dependencies._full_text`` verbatim (not imported
|
|
42
|
+
-- that name is module-private to ``dependencies.py``): config-key
|
|
43
|
+
extraction (#152) must never depend on the stored ``source`` -- emptied
|
|
44
|
+
by ``capture_text=False`` (a payload-size control, not an extraction
|
|
45
|
+
control). Read the real file fresh instead;
|
|
46
|
+
fall back to ``art.source`` only if it's gone (e.g. a synthetic artifact
|
|
47
|
+
in a unit test, or the file vanished mid-run). Keep the two in sync if
|
|
48
|
+
this logic changes."""
|
|
49
|
+
try:
|
|
50
|
+
return (project_dir / path).read_bytes().decode("utf-8")
|
|
51
|
+
except (OSError, UnicodeDecodeError):
|
|
52
|
+
return art.source
|
|
53
|
+
|
|
54
|
+
|
|
40
55
|
def _ensure_ray() -> None:
|
|
41
56
|
"""Initialize Ray with the driver's pinned hash seed in the workers.
|
|
42
57
|
|
|
@@ -648,6 +663,58 @@ class Codeanalyzer:
|
|
|
648
663
|
|
|
649
664
|
detect_entrypoints(app, self.project_dir, self.options.entrypoint_rules)
|
|
650
665
|
|
|
666
|
+
# Artifacts + dependencies: L1 data, every level, never varies with -a
|
|
667
|
+
# (spec 2026-08-27). Deterministic by default; venv probing is opt-in.
|
|
668
|
+
from codeanalyzer.artifacts import (
|
|
669
|
+
build_dependency_view, discover_artifacts, extract_config_keys,
|
|
670
|
+
is_config_eligible,
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
app.artifacts = discover_artifacts(
|
|
674
|
+
self.project_dir, app_name,
|
|
675
|
+
capture_text=self.options.artifact_text,
|
|
676
|
+
)
|
|
677
|
+
app.dependencies, app.unresolved_imports = build_dependency_view(
|
|
678
|
+
app.artifacts,
|
|
679
|
+
app.symbol_table,
|
|
680
|
+
self.project_dir,
|
|
681
|
+
self.virtualenv if self.options.resolve_installed else None,
|
|
682
|
+
self.options.resolve_installed,
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
# Config keys (#152): L1 data, layered onto the same artifacts, every
|
|
686
|
+
# level. Namespace-eligible artifacts only (env-family by basename,
|
|
687
|
+
# else by format). `extraction` combines with any prior
|
|
688
|
+
# dependency-manifest pass on the same artifact: a parse failure here
|
|
689
|
+
# always downgrades to "partial" (never drops the artifact); a clean
|
|
690
|
+
# parse upgrades an untouched "none" to "full", but never overwrites
|
|
691
|
+
# an existing "partial" (e.g. from a failed dependency-manifest parse
|
|
692
|
+
# on the same artifact) -- a successful pass here must not silently
|
|
693
|
+
# erase an unrelated failure already recorded on the artifact.
|
|
694
|
+
for path in sorted(app.artifacts):
|
|
695
|
+
art = app.artifacts[path]
|
|
696
|
+
if not is_config_eligible(art):
|
|
697
|
+
continue
|
|
698
|
+
full_text = _artifact_full_text(self.project_dir, path, art)
|
|
699
|
+
keys, ok = extract_config_keys(art, full_text, self.options.artifact_text)
|
|
700
|
+
art.config_keys = keys
|
|
701
|
+
if not ok:
|
|
702
|
+
art.extraction = "partial"
|
|
703
|
+
elif art.extraction == "none":
|
|
704
|
+
art.extraction = "full"
|
|
705
|
+
|
|
706
|
+
# config_use (#162) detection: needs callees resolved (backfill_
|
|
707
|
+
# callees above, gated the same >= 2) and config_keys just extracted
|
|
708
|
+
# above, so this runs after both rather than immediately following
|
|
709
|
+
# the callee backfill call itself. Resolution is deferred past the
|
|
710
|
+
# L3/L4 blocks below (the dataflow tiers need DDG and call_graph/
|
|
711
|
+
# caller-argument substrate those blocks populate) and gated by
|
|
712
|
+
# level there, literal tier included.
|
|
713
|
+
if self.analysis_level >= 2:
|
|
714
|
+
from codeanalyzer.artifacts import detect_config_reads
|
|
715
|
+
|
|
716
|
+
config_reads = detect_config_reads(app)
|
|
717
|
+
|
|
651
718
|
# L3: intraprocedural dataflow (CFG/CDG/DDG) emitted onto the v2 tree.
|
|
652
719
|
if self.analysis_level >= 3:
|
|
653
720
|
from codeanalyzer.dataflow.builder import (
|
|
@@ -688,6 +755,29 @@ class Codeanalyzer:
|
|
|
688
755
|
# ``infos`` are the syntactic (L3) PDGs from the >=3 block above.
|
|
689
756
|
emit_ddg_pointsto_delta(app, infos, ir, sig_to_id)
|
|
690
757
|
|
|
758
|
+
# config_use (#162) resolution: literal tier always runs at >= 2;
|
|
759
|
+
# the dataflow tiers widen the set as substrate comes online -- intra
|
|
760
|
+
# needs the DDG the L3 block above just emitted, interproc needs
|
|
761
|
+
# call_graph + caller PyCallArgument values (already present since
|
|
762
|
+
# L2) plus, for its one-hop caller-side closure, the caller's own
|
|
763
|
+
# DDG -- populated for every callable by the same L3 block, which
|
|
764
|
+
# always runs before this when analysis_level >= 4. A read resolved
|
|
765
|
+
# at a lower tier is never recomputed, so `-a 2 ⊆ -a 3 ⊆ -a 4` holds
|
|
766
|
+
# by construction.
|
|
767
|
+
if self.analysis_level >= 2:
|
|
768
|
+
from codeanalyzer.artifacts import (
|
|
769
|
+
dataflow_intra_tier, dataflow_interproc_tier, resolve_uses,
|
|
770
|
+
)
|
|
771
|
+
|
|
772
|
+
tier_fns = []
|
|
773
|
+
if self.analysis_level >= 3:
|
|
774
|
+
tier_fns.append(dataflow_intra_tier)
|
|
775
|
+
if self.analysis_level >= 4:
|
|
776
|
+
tier_fns.append(dataflow_interproc_tier)
|
|
777
|
+
app.config_uses, app.config_reads_unresolved = resolve_uses(
|
|
778
|
+
config_reads, app, tier_fns=tier_fns
|
|
779
|
+
)
|
|
780
|
+
|
|
691
781
|
# Build the v2 envelope, then persist it (the cache stores the full
|
|
692
782
|
# ``Analysis`` envelope so a reused cache round-trips schema_version).
|
|
693
783
|
# k_limit is an L3+ envelope key: below the dataflow levels it stays
|
codeanalyzer/dataflow/builder.py
CHANGED
|
@@ -353,6 +353,10 @@ def build_program_graphs(
|
|
|
353
353
|
"""
|
|
354
354
|
class_idx = _class_index(app)
|
|
355
355
|
callable_idx = _callable_index(app)
|
|
356
|
+
# Reverse of callable_idx: a callsite's already-L2-backfilled BodyNode.callee
|
|
357
|
+
# (a can:// id, folding in the defuse linker's resolution -- not just Jedi's
|
|
358
|
+
# own callee_signature side channel) resolved back to a dotted signature.
|
|
359
|
+
id_to_sig = {c.id: sig for sig, c in callable_idx.items()}
|
|
356
360
|
|
|
357
361
|
infos, func_asts = build_function_pdgs(app, k, oracle_factory=oracle_factory)
|
|
358
362
|
|
|
@@ -373,7 +377,17 @@ def build_program_graphs(
|
|
|
373
377
|
calls_by_line.setdefault(call.lineno, (node.id, call))
|
|
374
378
|
|
|
375
379
|
for site in pycallable.call_sites or []:
|
|
376
|
-
|
|
380
|
+
# Prefer the callsite's body-backfilled callee over Jedi's own
|
|
381
|
+
# callee_signature side channel: under cross-test parso/Jedi cache
|
|
382
|
+
# pressure that inference can silently degrade (full-suite-only
|
|
383
|
+
# loss of SDG summary/global stitching -- test_dataflow_sdg.py),
|
|
384
|
+
# while BodyNode.callee is the deterministic L2 path (Jedi filtered
|
|
385
|
+
# the same way, PLUS the defuse linker's fallback -- l2_callees.py).
|
|
386
|
+
# Only a resolved INTERNAL target counts (id_to_sig misses on an
|
|
387
|
+
# external/unresolved callee); falls through to callee_signature
|
|
388
|
+
# exactly as before whenever the body doesn't have an answer.
|
|
389
|
+
body_node = pycallable.body.get(f"{site.start_line}:{site.start_column}")
|
|
390
|
+
target = (id_to_sig.get(body_node.callee) if body_node else None) or site.callee_signature
|
|
377
391
|
if not target:
|
|
378
392
|
continue
|
|
379
393
|
if target in class_idx and target not in infos:
|
|
@@ -428,8 +442,13 @@ def build_program_graphs(
|
|
|
428
442
|
for t in cs.targets:
|
|
429
443
|
call_edges.append((sig, t))
|
|
430
444
|
|
|
431
|
-
|
|
432
|
-
|
|
445
|
+
# The converged (facts, ddg) per function are threaded straight into the
|
|
446
|
+
# assembler rather than re-derived there (#155).
|
|
447
|
+
solutions: Dict[str, object] = {}
|
|
448
|
+
summaries = compute_summaries(
|
|
449
|
+
infos, sorted(set(call_edges)), solutions=solutions
|
|
450
|
+
)
|
|
451
|
+
return assemble_sdg(infos, summaries, k, solutions=solutions)
|
|
433
452
|
|
|
434
453
|
|
|
435
454
|
def emit_l4(
|
codeanalyzer/dataflow/sdg.py
CHANGED
|
@@ -386,8 +386,18 @@ def assemble_sdg(
|
|
|
386
386
|
infos: Dict[str, FunctionInfo],
|
|
387
387
|
summaries: Dict[str, FunctionSummary],
|
|
388
388
|
k: int,
|
|
389
|
+
*,
|
|
390
|
+
solutions: Optional[Dict[str, Tuple[Dict[int, object], List[object]]]] = None,
|
|
389
391
|
) -> ProgramGraphsIR:
|
|
390
|
-
"""Stitch every function's PDG into the whole-program SDG.
|
|
392
|
+
"""Stitch every function's PDG into the whole-program SDG.
|
|
393
|
+
|
|
394
|
+
*solutions* optionally carries the converged ``(facts, ddg)`` that
|
|
395
|
+
:func:`~codeanalyzer.dataflow.summaries.compute_summaries` already
|
|
396
|
+
derived, sparing a second identical solve per function (#155). Omit it and
|
|
397
|
+
every function is re-solved, which is the historical behaviour and the
|
|
398
|
+
right posture whenever *summaries* did not come from an immediately
|
|
399
|
+
preceding run over these same *infos*.
|
|
400
|
+
"""
|
|
391
401
|
ir = ProgramGraphsIR(k_limit=k)
|
|
392
402
|
|
|
393
403
|
# Pass 1: solve each function against the final summaries and lay out its
|
|
@@ -396,7 +406,12 @@ def assemble_sdg(
|
|
|
396
406
|
formal_ids: Dict[str, Dict[str, int]] = {}
|
|
397
407
|
for sig in sorted(infos):
|
|
398
408
|
info = infos[sig]
|
|
399
|
-
|
|
409
|
+
cached = solutions.get(sig) if solutions is not None else None
|
|
410
|
+
if cached is None:
|
|
411
|
+
summary, facts, ddg = solve_function(info, summaries)
|
|
412
|
+
else:
|
|
413
|
+
facts, ddg = cached
|
|
414
|
+
summary = summaries[sig]
|
|
400
415
|
asm = _FunctionAssembler(info, summary, facts, ddg)
|
|
401
416
|
asm.build_formals()
|
|
402
417
|
assemblers[sig] = asm
|
|
@@ -199,19 +199,42 @@ def solve_function(
|
|
|
199
199
|
def compute_summaries(
|
|
200
200
|
infos: Dict[str, FunctionInfo],
|
|
201
201
|
call_edges: List[Tuple[str, str]],
|
|
202
|
+
*,
|
|
203
|
+
solutions: Optional[Dict[str, Tuple[Dict[int, object], List[DDGEdge]]]] = None,
|
|
202
204
|
) -> Dict[str, FunctionSummary]:
|
|
203
205
|
"""Bottom-up composition over the SCC condensation DAG, monotone fixpoint
|
|
204
|
-
within each SCC.
|
|
206
|
+
within each SCC.
|
|
207
|
+
|
|
208
|
+
A **singleton SCC with no self-edge** is solved exactly once: the
|
|
209
|
+
condensation is processed bottom-up, so every callee summary it reads is
|
|
210
|
+
already final and a second pass could only recompute the same answer to
|
|
211
|
+
observe that nothing changed. Genuinely recursive SCCs (several members,
|
|
212
|
+
or one member calling itself) still iterate to fixpoint.
|
|
213
|
+
|
|
214
|
+
When *solutions* is supplied it receives each signature's converged
|
|
215
|
+
``(facts, ddg)`` — the by-products of the final solve, which
|
|
216
|
+
:func:`~codeanalyzer.dataflow.sdg.assemble_sdg` would otherwise recompute
|
|
217
|
+
from scratch. They are the same values that a fresh solve against the
|
|
218
|
+
final summaries produces, because a converged pass is by definition one
|
|
219
|
+
in which no member's summary changed (#155).
|
|
220
|
+
"""
|
|
205
221
|
order = strongly_connected_components(sorted(infos), call_edges)
|
|
222
|
+
self_calls = {src for src, dst in call_edges if src == dst}
|
|
206
223
|
summaries: Dict[str, FunctionSummary] = {}
|
|
207
224
|
for scc in order:
|
|
208
225
|
members = [s for s in scc if s in infos]
|
|
209
|
-
|
|
210
|
-
|
|
226
|
+
if not members:
|
|
227
|
+
continue
|
|
228
|
+
recursive = len(members) > 1 or members[0] in self_calls
|
|
229
|
+
while True:
|
|
211
230
|
changed = False
|
|
212
231
|
for sig in members:
|
|
213
|
-
new,
|
|
232
|
+
new, facts, ddg = solve_function(infos[sig], summaries)
|
|
233
|
+
if solutions is not None:
|
|
234
|
+
solutions[sig] = (facts, ddg)
|
|
214
235
|
if summaries.get(sig) != new:
|
|
215
236
|
summaries[sig] = new
|
|
216
237
|
changed = True
|
|
238
|
+
if not (recursive and changed):
|
|
239
|
+
break
|
|
217
240
|
return summaries
|
codeanalyzer/neo4j/bolt.py
CHANGED
|
@@ -27,11 +27,24 @@ Algorithm (the module subgraph is the unit of idempotent replacement):
|
|
|
27
27
|
4. upsert edges owned by changed modules (+ the shared edges).
|
|
28
28
|
5. on a FULL run only, prune modules whose source file vanished.
|
|
29
29
|
|
|
30
|
+
**A push never deletes by default** (#171). Steps 3 and 5 are the only destructive
|
|
31
|
+
ones and both run on ``eager`` (``--eager``) only; a default ``--lazy`` push is purely
|
|
32
|
+
additive — MERGE-upsert of nodes and edges, nothing removed. The cost of the default is
|
|
33
|
+
staleness: a declaration or a call edge the source no longer has stays in the graph until
|
|
34
|
+
an ``--eager`` push reconciles it. That is the deliberate trade — an incremental push into
|
|
35
|
+
a shared database should not be able to destroy anything, and the destructive rebuild is
|
|
36
|
+
opt-in under the same flag that already forces a clean analysis rebuild.
|
|
37
|
+
|
|
30
38
|
Nodes are MERGE-upserted, never blindly deleted, so a declaration another
|
|
31
39
|
(unchanged) module still references survives and its incoming edges stay valid.
|
|
32
40
|
``:PyExternal`` / ``:PyPackage`` / ``:PyDecorator`` are shared (no ``_module``) and are
|
|
33
41
|
MERGE-only.
|
|
34
42
|
|
|
43
|
+
Every ``_module`` match is anchored on the python-owned labels
|
|
44
|
+
(``schema.MODULE_OWNED_PATTERN``). ``_module`` is a shared convention, not a python-private
|
|
45
|
+
one -- codeanalyzer-java and codeanalyzer-typescript set it on their nodes too -- so an
|
|
46
|
+
unlabelled match reaches a sibling analyzer's graph in a shared database (#171).
|
|
47
|
+
|
|
35
48
|
The ``neo4j`` driver is imported lazily so it stays an optional dependency and
|
|
36
49
|
off the default (json) output path entirely.
|
|
37
50
|
"""
|
|
@@ -41,7 +54,7 @@ from dataclasses import dataclass
|
|
|
41
54
|
from typing import Dict, List, Optional
|
|
42
55
|
|
|
43
56
|
from codeanalyzer.neo4j.rows import EdgeRow, GraphRows, NodeRow, chunk
|
|
44
|
-
from codeanalyzer.neo4j.schema import CONSTRAINTS, INDEXES
|
|
57
|
+
from codeanalyzer.neo4j.schema import CONSTRAINTS, INDEXES, MODULE_OWNED_PATTERN
|
|
45
58
|
from codeanalyzer.utils import logger
|
|
46
59
|
|
|
47
60
|
DESCENDANTS = (
|
|
@@ -59,7 +72,7 @@ class BoltConfig:
|
|
|
59
72
|
database: Optional[str] = None
|
|
60
73
|
|
|
61
74
|
|
|
62
|
-
def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
75
|
+
def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool, eager: bool = False) -> None:
|
|
63
76
|
try:
|
|
64
77
|
import neo4j # noqa: WPS433 (lazy, optional dependency)
|
|
65
78
|
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
@@ -119,15 +132,26 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
|
119
132
|
_upsert_nodes(session, neo4j, shared)
|
|
120
133
|
|
|
121
134
|
# 4. per changed module: purge owned edges + vanished decls, then upsert its nodes.
|
|
135
|
+
# The purge is the only destructive step in a push, so it runs on --eager only.
|
|
122
136
|
for m in changed:
|
|
123
137
|
nodes = by_module[m]
|
|
124
138
|
keys = [n.value for n in nodes]
|
|
139
|
+
if not eager:
|
|
140
|
+
_upsert_nodes(session, neo4j, nodes)
|
|
141
|
+
continue
|
|
125
142
|
with session() as s:
|
|
126
143
|
def _purge(tx, module=m, node_keys=keys):
|
|
127
|
-
|
|
144
|
+
# Anchored on python-owned labels: `_module` is also set by the java
|
|
145
|
+
# and typescript analyzers, so an unlabelled match would delete a
|
|
146
|
+
# sibling's nodes wherever a file key collides (#171).
|
|
128
147
|
tx.run(
|
|
129
|
-
"MATCH (x
|
|
130
|
-
"
|
|
148
|
+
f"MATCH (x:{MODULE_OWNED_PATTERN}) WHERE x._module = $m "
|
|
149
|
+
"MATCH (x)-[r]->() DELETE r",
|
|
150
|
+
m=module,
|
|
151
|
+
)
|
|
152
|
+
tx.run(
|
|
153
|
+
f"MATCH (x:{MODULE_OWNED_PATTERN}) WHERE x._module = $m "
|
|
154
|
+
"AND NOT coalesce(x.signature, x.id, x.file_key) IN $keys "
|
|
131
155
|
"DETACH DELETE x",
|
|
132
156
|
m=module,
|
|
133
157
|
keys=node_keys,
|
|
@@ -147,7 +171,7 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
|
147
171
|
# 6. orphan prune — only safe on a full run (a targeted run can't tell deleted from untargeted).
|
|
148
172
|
# Scope to THIS application's anchor so a full run for application B never
|
|
149
173
|
# deletes application A's modules from a shared database.
|
|
150
|
-
if full_run and app_name is not None:
|
|
174
|
+
if full_run and eager and app_name is not None:
|
|
151
175
|
present = list(by_module.keys())
|
|
152
176
|
with session() as s:
|
|
153
177
|
res = s.run(
|
|
@@ -161,6 +185,11 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
|
161
185
|
pruned = res.single()
|
|
162
186
|
pruned_count = pruned["pruned"] if pruned else 0
|
|
163
187
|
logger.info(f"neo4j(bolt): pruned {pruned_count} vanished module(s)")
|
|
188
|
+
elif not eager:
|
|
189
|
+
logger.info(
|
|
190
|
+
"neo4j(bolt): additive push (--lazy) — nothing deleted; "
|
|
191
|
+
"re-run with --eager to reconcile removed declarations and edges"
|
|
192
|
+
)
|
|
164
193
|
else:
|
|
165
194
|
logger.info(
|
|
166
195
|
"neo4j(bolt): targeted run — orphan pruning skipped (deleted files not removed)"
|
codeanalyzer/neo4j/emit.py
CHANGED
|
@@ -67,9 +67,10 @@ def emit_neo4j(analysis: Analysis, options: AnalysisOptions) -> None:
|
|
|
67
67
|
password=options.neo4j_password,
|
|
68
68
|
database=options.neo4j_database,
|
|
69
69
|
)
|
|
70
|
-
# A full run (no single-file restriction) makes orphan pruning safe
|
|
70
|
+
# A full run (no single-file restriction) makes orphan pruning safe; --eager
|
|
71
|
+
# is what permits any deletion at all (#171).
|
|
71
72
|
full_run = options.file_name is None
|
|
72
|
-
bolt_writer(rows, cfg, full_run)
|
|
73
|
+
bolt_writer(rows, cfg, full_run, eager=options.rebuild_analysis)
|
|
73
74
|
return
|
|
74
75
|
|
|
75
76
|
out_dir = options.output if options.output is not None else Path.cwd()
|
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -47,6 +47,7 @@ from codeanalyzer.schema import (
|
|
|
47
47
|
PyModule,
|
|
48
48
|
PyVariableDeclaration,
|
|
49
49
|
)
|
|
50
|
+
from codeanalyzer.schema.ids import application_id, purl_pypi
|
|
50
51
|
from codeanalyzer.schema.py_schema import PyDecorator
|
|
51
52
|
|
|
52
53
|
|
|
@@ -100,6 +101,16 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
100
101
|
# MERGE — a no-op when no callable carries L3 fields (levels 1/2).
|
|
101
102
|
_project_program_graphs(b, app, externals, sig_to_id)
|
|
102
103
|
|
|
104
|
+
# Neutral artifact/dependency subgraph (Task 6). L1 data — always present,
|
|
105
|
+
# full-depth-always regardless of -a.
|
|
106
|
+
_project_artifacts(b, app, app_name, app_ref)
|
|
107
|
+
|
|
108
|
+
# config_use (#162): the resolved-read bridge (PyBodyNode from
|
|
109
|
+
# _project_program_graphs above) into the config-key subgraph
|
|
110
|
+
# (ConfigKey from _project_artifacts above), plus first-class unresolved
|
|
111
|
+
# reads.
|
|
112
|
+
_project_config_uses(b, app, app_ref, externals, sig_to_id)
|
|
113
|
+
|
|
103
114
|
return b.finish()
|
|
104
115
|
|
|
105
116
|
|
|
@@ -248,6 +259,186 @@ def _project_program_graphs(
|
|
|
248
259
|
)
|
|
249
260
|
|
|
250
261
|
|
|
262
|
+
# ----------------------------------------------------------------------------------------------
|
|
263
|
+
# Artifact / dependency subgraph (spec 2026-08-27, Task 6)
|
|
264
|
+
# ----------------------------------------------------------------------------------------------
|
|
265
|
+
|
|
266
|
+
_LOCK_BASENAMES = ("poetry.lock", "uv.lock", "Pipfile.lock")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _import_ghost(b: RowBuilder, app_can_id: str, name: str) -> NodeRef:
|
|
270
|
+
"""A ``:PyExternal`` ghost for a bare imported module name (``PY_PROVIDES``'s
|
|
271
|
+
``provides_imports`` entries, ``PY_UNRESOLVED_IMPORT``'s ``module``).
|
|
272
|
+
|
|
273
|
+
``app.external_symbols`` only homes call-graph endpoints (``_home_external_
|
|
274
|
+
symbols`` walks ``app.call_graph``), so a module that is imported but never
|
|
275
|
+
called — the overwhelmingly common case for ``provides_imports`` and the
|
|
276
|
+
*only* case for an unresolved import — has no existing ghost to MERGE onto.
|
|
277
|
+
This builds one with the same id shape ``_call_endpoint``/``_home_external_
|
|
278
|
+
symbols`` use for a dot-less (no ``.`` in the signature) call target:
|
|
279
|
+
``<app can:// id>/@external/<name>``, ``module=None``. Same two-label
|
|
280
|
+
``["PySymbol", "PyExternal"]`` idiom as ``_call_endpoint`` -- the schema
|
|
281
|
+
declares :PyExternal's merge label as PySymbol, and RowBuilder MERGEs by
|
|
282
|
+
``(labels[0], value)``, so if a call to that same bare name is ever
|
|
283
|
+
projected too, both rows collapse onto this one node — correctly, since
|
|
284
|
+
they name the same real-world symbol."""
|
|
285
|
+
return b.node(
|
|
286
|
+
["PySymbol", "PyExternal"], "id", f"{app_can_id}/@external/{name}", {"name": name}
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _project_artifacts(b: RowBuilder, app: PyApplication, app_name: str, app_ref: NodeRef) -> None:
|
|
291
|
+
"""Non-code artifacts, declared dependencies and undeclared imports (Tasks
|
|
292
|
+
1-5) -- neutral ``Artifact``/``Package`` nodes with no ``Py`` prefix
|
|
293
|
+
(deliberate: cross-language merge targets, unlike everything else this
|
|
294
|
+
module projects). Always emitted regardless of ``-a`` -- this section is
|
|
295
|
+
L1 data, identical at every analysis level (mirrors ``analysis.json``)."""
|
|
296
|
+
app_can_id = application_id(app_name)
|
|
297
|
+
|
|
298
|
+
for path in sorted(app.artifacts or {}):
|
|
299
|
+
art = app.artifacts[path]
|
|
300
|
+
art_ref = b.node(
|
|
301
|
+
["Artifact"],
|
|
302
|
+
"id",
|
|
303
|
+
art.id,
|
|
304
|
+
prune(
|
|
305
|
+
{
|
|
306
|
+
"path": art.path,
|
|
307
|
+
"format": art.format,
|
|
308
|
+
"roles": art.roles,
|
|
309
|
+
"size_bytes": art.size_bytes,
|
|
310
|
+
"sha256": art.sha256,
|
|
311
|
+
"source": art.source,
|
|
312
|
+
"extraction": art.extraction,
|
|
313
|
+
}
|
|
314
|
+
),
|
|
315
|
+
)
|
|
316
|
+
b.edge("HAS_ARTIFACT", app_ref, art_ref)
|
|
317
|
+
|
|
318
|
+
# Config keys flattened out of this artifact (#152) -- sorted by key
|
|
319
|
+
# for deterministic row order, matching the JSON side's L1 determinism.
|
|
320
|
+
for ck in sorted(art.config_keys or [], key=lambda k: k.key):
|
|
321
|
+
ck_ref = b.node(
|
|
322
|
+
["ConfigKey"],
|
|
323
|
+
"id",
|
|
324
|
+
ck.id,
|
|
325
|
+
prune(
|
|
326
|
+
{
|
|
327
|
+
"key": ck.key,
|
|
328
|
+
"namespace": ck.namespace,
|
|
329
|
+
"value": ck.value,
|
|
330
|
+
"references": list(ck.references or []),
|
|
331
|
+
"start_line": ck.span.start[0] if ck.span else None,
|
|
332
|
+
"end_line": ck.span.end[0] if ck.span else None,
|
|
333
|
+
}
|
|
334
|
+
),
|
|
335
|
+
)
|
|
336
|
+
b.edge("DEFINES_CONFIG", art_ref, ck_ref)
|
|
337
|
+
|
|
338
|
+
# Every lock artifact present LOCKS every dependency it pinned. The pins
|
|
339
|
+
# from all lock files are already merged into one `locked_version` per
|
|
340
|
+
# dependency upstream (Task 5 `build_dependency_view`) -- there is no
|
|
341
|
+
# per-lock-file attribution to split on, so (like the JSON projection) a
|
|
342
|
+
# dependency locked with N lock artifacts present gets N LOCKS edges.
|
|
343
|
+
lock_ids = [
|
|
344
|
+
app.artifacts[p].id
|
|
345
|
+
for p in sorted(app.artifacts or {})
|
|
346
|
+
if p.rsplit("/", 1)[-1] in _LOCK_BASENAMES
|
|
347
|
+
]
|
|
348
|
+
|
|
349
|
+
# app.dependencies has one PyDependency per DECLARING MANIFEST, so a
|
|
350
|
+
# package declared in 2+ manifests (e.g. requirements.txt +
|
|
351
|
+
# requirements-dev.txt both listing "requests") walks this loop once per
|
|
352
|
+
# manifest. DECLARES_DEPENDENCY is correctly one row per declaration (its
|
|
353
|
+
# `from_ref` is the manifest, so those rows are already distinct) -- but
|
|
354
|
+
# LOCKS/PY_PROVIDES/PY_UNRESOLVED_IMPORT are per-PACKAGE facts, and
|
|
355
|
+
# RowBuilder.edge() is append-only (unlike node(), it does not MERGE-dedup)
|
|
356
|
+
# -- so without a guard they'd be emitted once per declaring manifest
|
|
357
|
+
# instead of once, violating GraphRows' documented deduped-bag contract.
|
|
358
|
+
seen: set = set()
|
|
359
|
+
|
|
360
|
+
for d in app.dependencies or []:
|
|
361
|
+
pkg_id = purl_pypi(d.name)
|
|
362
|
+
pkg_ref = b.node(["Package"], "id", pkg_id, {"ecosystem": "pypi", "name": d.name})
|
|
363
|
+
# kind-discriminated: the same manifest may declare one package twice
|
|
364
|
+
# under different kinds (e.g. requests in [project.dependencies] AND
|
|
365
|
+
# again under [project.optional-dependencies]) -- same endpoint pair,
|
|
366
|
+
# so a plain MERGE would collapse the two declarations into one row.
|
|
367
|
+
b.edge(
|
|
368
|
+
"DECLARES_DEPENDENCY",
|
|
369
|
+
NodeRef("Artifact", "id", d.declared_in),
|
|
370
|
+
pkg_ref,
|
|
371
|
+
prune({"spec": d.spec, "kind": d.kind, "extras": d.extras, "prov": d.prov, "direct": d.direct}),
|
|
372
|
+
key=d.kind,
|
|
373
|
+
)
|
|
374
|
+
if d.locked_version:
|
|
375
|
+
for lock_id in lock_ids:
|
|
376
|
+
key = ("LOCKS", lock_id, pkg_id)
|
|
377
|
+
if key not in seen:
|
|
378
|
+
seen.add(key)
|
|
379
|
+
b.edge(
|
|
380
|
+
"LOCKS",
|
|
381
|
+
NodeRef("Artifact", "id", lock_id),
|
|
382
|
+
pkg_ref,
|
|
383
|
+
{"version": d.locked_version},
|
|
384
|
+
)
|
|
385
|
+
for top in d.provides_imports:
|
|
386
|
+
ghost_ref = _import_ghost(b, app_can_id, top)
|
|
387
|
+
key = ("PY_PROVIDES", pkg_id, ghost_ref.value)
|
|
388
|
+
if key not in seen:
|
|
389
|
+
seen.add(key)
|
|
390
|
+
b.edge("PY_PROVIDES", pkg_ref, ghost_ref)
|
|
391
|
+
|
|
392
|
+
for u in app.unresolved_imports or []:
|
|
393
|
+
ghost_ref = _import_ghost(b, app_can_id, u.module)
|
|
394
|
+
key = ("PY_UNRESOLVED_IMPORT", app_ref.value, ghost_ref.value)
|
|
395
|
+
if key not in seen:
|
|
396
|
+
seen.add(key)
|
|
397
|
+
b.edge("PY_UNRESOLVED_IMPORT", app_ref, ghost_ref, prune({"prov": u.prov}))
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _project_config_uses(
|
|
401
|
+
b: RowBuilder, app: PyApplication, app_ref: NodeRef, externals: dict, sig_to_id: dict,
|
|
402
|
+
) -> None:
|
|
403
|
+
"""config_use (#162): PY_USES_CONFIG (`app.config_uses`) and
|
|
404
|
+
PY_READS_CONFIG_UNRESOLVED (`app.config_reads_unresolved`).
|
|
405
|
+
|
|
406
|
+
`PyConfigUseEdge.src`/`.dst` are already a GLOBAL ordinal id and a
|
|
407
|
+
ConfigKey id (both resolved upstream by `resolve_uses`), so — like
|
|
408
|
+
`param_in`/`param_out` — they address existing PyBodyNode/ConfigKey rows
|
|
409
|
+
directly with a plain :class:`NodeRef`; no defer-and-gate needed. Call
|
|
410
|
+
nodes enter a callable's `body` at L1 (before any config_use tier runs),
|
|
411
|
+
so the src PyBodyNode is always already projected by
|
|
412
|
+
`_project_program_graphs`, whatever level this ran at.
|
|
413
|
+
|
|
414
|
+
`PyConfigRead.callee` is already the full external `can://…/@external/…`
|
|
415
|
+
id (not a bare module name), so its ghost goes through `_call_endpoint`
|
|
416
|
+
(which looks it up in `externals` directly) rather than `_import_ghost`
|
|
417
|
+
(built for a bare imported name) — same inline node()+edge() shape
|
|
418
|
+
`PY_UNRESOLVED_IMPORT` uses. `_k` discriminates by (key, reason): the same
|
|
419
|
+
external callee (e.g. `os.getenv`) legitimately reads several distinct
|
|
420
|
+
undeclared/dynamic keys across a codebase, and without a discriminant a
|
|
421
|
+
plain endpoint-pair MERGE would collapse those onto one relationship.
|
|
422
|
+
"""
|
|
423
|
+
for e in app.config_uses:
|
|
424
|
+
b.edge(
|
|
425
|
+
"PY_USES_CONFIG",
|
|
426
|
+
NodeRef("PyBodyNode", "id", e.src),
|
|
427
|
+
NodeRef("ConfigKey", "id", e.dst),
|
|
428
|
+
prune({"prov": list(e.prov) if e.prov else None}),
|
|
429
|
+
)
|
|
430
|
+
for r in app.config_reads_unresolved:
|
|
431
|
+
ghost_ref = _call_endpoint(b, r.callee, externals, sig_to_id)
|
|
432
|
+
b.edge(
|
|
433
|
+
"PY_READS_CONFIG_UNRESOLVED",
|
|
434
|
+
app_ref,
|
|
435
|
+
ghost_ref,
|
|
436
|
+
prune({"key": r.key, "reason": r.reason, "prov": list(r.prov) if r.prov else None}),
|
|
437
|
+
# _k=(key,reason) does not per-site discriminate the non-literal bucket (accepted prop-list ceiling).
|
|
438
|
+
key=f"{r.key or ''}|{r.reason}",
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
|
|
251
442
|
def _sym(can_id: str) -> NodeRef:
|
|
252
443
|
return NodeRef("PySymbol", "id", can_id)
|
|
253
444
|
|
codeanalyzer/neo4j/schema.py
CHANGED
|
@@ -201,6 +201,30 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
201
201
|
"_module": "string",
|
|
202
202
|
},
|
|
203
203
|
),
|
|
204
|
+
# Neutral artifact/dependency subgraph (spec 2026-08-27, Task 6). No `Py`
|
|
205
|
+
# prefix -- deliberate: `Artifact`/`Package` are cross-language merge
|
|
206
|
+
# targets, so a sibling-language analyzer over the same repo lands on the
|
|
207
|
+
# same nodes instead of a per-language duplicate. `PY_PROVIDES` /
|
|
208
|
+
# `PY_UNRESOLVED_IMPORT` stay PY_-namespaced (this analyzer's own claim
|
|
209
|
+
# about what an import resolves to) and target `:PyExternal`.
|
|
210
|
+
NodeLabel("Artifact", "Artifact", "id", {
|
|
211
|
+
"id": "string", "path": "string", "format": "string",
|
|
212
|
+
"roles": "string[]", "size_bytes": "integer", "sha256": "string",
|
|
213
|
+
"source": "string", "extraction": "string",
|
|
214
|
+
}),
|
|
215
|
+
NodeLabel("Package", "Package", "id", {
|
|
216
|
+
"id": "string", "ecosystem": "string", "name": "string",
|
|
217
|
+
}),
|
|
218
|
+
# A configuration key flattened out of a config-bearing Artifact (#152).
|
|
219
|
+
# Neutral vocabulary like Artifact/Package -- a yaml/env/ini key is not a
|
|
220
|
+
# Python concept. `value` is omitted (not null) when the source model's
|
|
221
|
+
# value is None (--no-artifact-text, or a namespace with no value at that
|
|
222
|
+
# path); `references` is always present, possibly empty.
|
|
223
|
+
NodeLabel("ConfigKey", "ConfigKey", "id", {
|
|
224
|
+
"id": "string", "key": "string", "namespace": "string",
|
|
225
|
+
"value": "string", "references": "string[]",
|
|
226
|
+
**_SPAN,
|
|
227
|
+
}),
|
|
204
228
|
]
|
|
205
229
|
|
|
206
230
|
_DECL_TARGETS = ["PyClass", "PyCallable"]
|
|
@@ -250,6 +274,39 @@ REL_TYPES: List[RelType] = [
|
|
|
250
274
|
RelType("PY_PARAM_IN", ["PyBodyNode"], ["PyBodyNode"], {"var": "string"}),
|
|
251
275
|
RelType("PY_PARAM_OUT", ["PyBodyNode"], ["PyBodyNode"], {"var": "string"}),
|
|
252
276
|
RelType("PY_SUMMARY", ["PyBodyNode"], ["PyBodyNode"]),
|
|
277
|
+
# Neutral artifact/dependency subgraph (Task 6).
|
|
278
|
+
RelType("HAS_ARTIFACT", ["PyApplication"], ["Artifact"]),
|
|
279
|
+
# A config key nests under exactly one owning artifact (its id is
|
|
280
|
+
# `<artifact-id>@key/<dotted.key>`) -- a plain containment edge, no
|
|
281
|
+
# per-edge properties or discriminant needed (#152).
|
|
282
|
+
RelType("DEFINES_CONFIG", ["Artifact"], ["ConfigKey"]),
|
|
283
|
+
# ``_k`` (merges per ``kind``): the same manifest may declare one package
|
|
284
|
+
# twice under different kinds (e.g. a runtime dep re-listed under an
|
|
285
|
+
# optional extra) -- same endpoint pair, so without the discriminant the
|
|
286
|
+
# plain MERGE collapses the two declarations into one row.
|
|
287
|
+
RelType("DECLARES_DEPENDENCY", ["Artifact"], ["Package"], {
|
|
288
|
+
"spec": "string", "kind": "string", "extras": "string[]", "prov": "string[]",
|
|
289
|
+
"direct": "boolean", "_k": "string",
|
|
290
|
+
}),
|
|
291
|
+
RelType("LOCKS", ["Artifact"], ["Package"], {"version": "string"}),
|
|
292
|
+
RelType("PY_PROVIDES", ["Package"], ["PyExternal"]),
|
|
293
|
+
RelType("PY_UNRESOLVED_IMPORT", ["PyApplication"], ["PyExternal"], {"prov": "string[]"}),
|
|
294
|
+
# config_use (#162): the resolved-read bridge from a call site's body node
|
|
295
|
+
# to the PyConfigKey it reads. `src`/`dst` are already GLOBAL ordinal /
|
|
296
|
+
# ConfigKey ids (resolved upstream by `resolve_uses`), so no discriminant
|
|
297
|
+
# is needed -- one call site reads one key per edge.
|
|
298
|
+
RelType("PY_USES_CONFIG", ["PyBodyNode"], ["ConfigKey"], {"prov": "string[]"}),
|
|
299
|
+
# A detector-matched read that never closed on exactly one declared key --
|
|
300
|
+
# first-class per #162, PyApplication -> PyExternal ghost of the callee
|
|
301
|
+
# (mirrors PY_UNRESOLVED_IMPORT's shape). `_k` discriminates by (key,
|
|
302
|
+
# reason): the same external callee (e.g. `os.getenv`) legitimately reads
|
|
303
|
+
# several distinct undeclared/dynamic keys across a codebase -- without a
|
|
304
|
+
# discriminant a plain endpoint-pair MERGE would collapse those onto one
|
|
305
|
+
# relationship and silently drop every key but the last one SET.
|
|
306
|
+
RelType(
|
|
307
|
+
"PY_READS_CONFIG_UNRESOLVED", ["PyApplication"], ["PyExternal"],
|
|
308
|
+
{"key": "string", "reason": "string", "prov": "string[]", "_k": "string"},
|
|
309
|
+
),
|
|
253
310
|
]
|
|
254
311
|
|
|
255
312
|
|
|
@@ -275,10 +332,27 @@ def uniqueness_constraints() -> list[str]:
|
|
|
275
332
|
|
|
276
333
|
CONSTRAINTS: List[str] = uniqueness_constraints()
|
|
277
334
|
|
|
335
|
+
# The labels this analyzer owns per module -- the ones carrying the internal ``_module``
|
|
336
|
+
# provenance property. Derived from NODE_LABELS so a new module-scoped label is covered
|
|
337
|
+
# without a second list to maintain. `_module` is NOT python-private: codeanalyzer-java
|
|
338
|
+
# and codeanalyzer-typescript set the same property on their nodes, so every statement
|
|
339
|
+
# matching on it must be anchored to these labels or it matches a sibling analyzer's graph
|
|
340
|
+
# in a shared database (#171).
|
|
341
|
+
MODULE_OWNED_LABELS: List[str] = [n.label for n in NODE_LABELS if "_module" in n.properties]
|
|
342
|
+
|
|
343
|
+
# The label disjunction to anchor such a statement with: ``MATCH (x:PyModule|PyClass|...)``.
|
|
344
|
+
MODULE_OWNED_PATTERN: str = "|".join(MODULE_OWNED_LABELS)
|
|
345
|
+
|
|
278
346
|
INDEXES: List[str] = [
|
|
279
347
|
"CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
|
|
280
348
|
"CREATE INDEX py_class_name IF NOT EXISTS FOR (c:PyClass) ON (c.name)",
|
|
281
349
|
"CREATE FULLTEXT INDEX py_code_fts IF NOT EXISTS FOR (c:PyCallable) ON EACH [c.code, c.docstring]",
|
|
350
|
+
] + [
|
|
351
|
+
# One per module-owned label: the incremental writer's per-module purge matches on
|
|
352
|
+
# `_module` once per changed module, which without these is a label scan per label per
|
|
353
|
+
# module -- quadratic on a full push (#171).
|
|
354
|
+
f"CREATE INDEX {label.lower()}_module IF NOT EXISTS FOR (x:{label}) ON (x._module)"
|
|
355
|
+
for label in MODULE_OWNED_LABELS
|
|
282
356
|
]
|
|
283
357
|
|
|
284
358
|
|