codeanalyzer-python 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/core.py +2 -2
- codeanalyzer/dataflow/builder.py +3 -0
- codeanalyzer/dataflow/identity.py +3 -3
- codeanalyzer/entrypoints/matching.py +36 -7
- codeanalyzer/entrypoints/pipeline.py +52 -4
- codeanalyzer/entrypoints/rules.py +12 -1
- codeanalyzer/entrypoints/rules.yml +34 -0
- codeanalyzer/neo4j/bolt.py +90 -57
- codeanalyzer/neo4j/cypher.py +19 -5
- codeanalyzer/neo4j/project.py +105 -44
- codeanalyzer/neo4j/rows.py +47 -1
- codeanalyzer/neo4j/schema.py +11 -25
- codeanalyzer/schema/ids.py +57 -10
- codeanalyzer/schema/l1_body.py +2 -0
- codeanalyzer/schema/py_schema.py +8 -2
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/METADATA +42 -11
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/RECORD +21 -21
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.4.0.dist-info → codeanalyzer_python-1.5.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -27,14 +27,15 @@ Modelling decisions (mirror of the TypeScript backend):
|
|
|
27
27
|
- call-graph endpoints absent from the symbol table become ``:PyExternal`` ghost
|
|
28
28
|
nodes, so RPC / third-party / framework edges are preserved (matching the
|
|
29
29
|
analyzer's own ghost-node behaviour).
|
|
30
|
-
- every project-owned node
|
|
30
|
+
- every project-owned node names its owning module (``_module`` in the props it
|
|
31
|
+
hands RowBuilder, lifted to ``NodeRow.module`` and never emitted, #173), so
|
|
31
32
|
the incremental writer can delete exactly what a re-analyzed module emitted.
|
|
32
33
|
"""
|
|
33
34
|
from __future__ import annotations
|
|
34
35
|
|
|
35
36
|
import json
|
|
36
37
|
from pathlib import Path
|
|
37
|
-
from typing import Any, List, Optional
|
|
38
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
38
39
|
|
|
39
40
|
from codeanalyzer.neo4j.schema import SCHEMA_VERSION
|
|
40
41
|
from codeanalyzer.neo4j.rows import GraphRows, NodeRef, Props, RowBuilder, prune
|
|
@@ -47,7 +48,8 @@ from codeanalyzer.schema import (
|
|
|
47
48
|
PyModule,
|
|
48
49
|
PyVariableDeclaration,
|
|
49
50
|
)
|
|
50
|
-
from codeanalyzer.schema
|
|
51
|
+
from codeanalyzer.schema import model_dump
|
|
52
|
+
from codeanalyzer.schema.ids import application_id, external_id, global_ordinal, purl_pypi
|
|
51
53
|
from codeanalyzer.schema.py_schema import PyDecorator
|
|
52
54
|
|
|
53
55
|
|
|
@@ -58,18 +60,30 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
58
60
|
passes it through so the :PyApplication node carries it as props."""
|
|
59
61
|
b = RowBuilder()
|
|
60
62
|
|
|
63
|
+
# Keyed on the ``can://<app>`` id, not on ``--app-name``: two applications
|
|
64
|
+
# analyzed under the same free-text name used to MERGE onto one root, with
|
|
65
|
+
# no diagnostic. ``name`` survives as a display property.
|
|
61
66
|
app_ref = b.node(
|
|
62
67
|
["PyApplication"],
|
|
63
|
-
"
|
|
64
|
-
app_name,
|
|
68
|
+
"id",
|
|
69
|
+
application_id(app_name),
|
|
65
70
|
prune(
|
|
66
71
|
{
|
|
72
|
+
"id": application_id(app_name),
|
|
73
|
+
"name": app_name,
|
|
67
74
|
"schema_version": SCHEMA_VERSION,
|
|
68
75
|
"analyzer_name": analyzer.name if analyzer else None,
|
|
69
76
|
"analyzer_version": analyzer.version if analyzer else None,
|
|
70
77
|
"repo_uri": app.repository.uri if app.repository else None,
|
|
71
78
|
"source_revision": app.repository.revision if app.repository else None,
|
|
72
79
|
"repo_dirty": app.repository.dirty if app.repository else None,
|
|
80
|
+
# #177: the entrypoint pass under-approximates by design, so a
|
|
81
|
+
# graph consumer must be able to tell "no entrypoints" from "the
|
|
82
|
+
# pass found nothing". Always present, even when empty.
|
|
83
|
+
"entrypoint_frameworks": list(app.entrypoint_report.frameworks_detected),
|
|
84
|
+
"entrypoint_report_json": json.dumps(
|
|
85
|
+
model_dump(app.entrypoint_report, mode="json"), sort_keys=True
|
|
86
|
+
),
|
|
73
87
|
}
|
|
74
88
|
),
|
|
75
89
|
)
|
|
@@ -87,19 +101,21 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
87
101
|
for file_key, mod in app.symbol_table.items():
|
|
88
102
|
mod_ref = b.node(["PyModule"], "id", mod.id, _module_props(mod, file_key))
|
|
89
103
|
b.edge("PY_HAS_MODULE", app_ref, mod_ref)
|
|
90
|
-
_project_module_body(b, file_key, mod_ref, mod, externals, sig_to_id, module_id_by_key
|
|
104
|
+
_project_module_body(b, file_key, mod_ref, mod, externals, sig_to_id, module_id_by_key,
|
|
105
|
+
application_id(app_name))
|
|
91
106
|
|
|
92
107
|
# The aggregated :PY_CALLS twin.
|
|
108
|
+
app_can_id = application_id(app_name)
|
|
93
109
|
for e in app.call_graph:
|
|
94
|
-
src = _call_endpoint(b, e.src, externals, sig_to_id)
|
|
95
|
-
tgt = _call_endpoint(b, e.dst, externals, sig_to_id)
|
|
110
|
+
src = _call_endpoint(b, e.src, externals, sig_to_id, app_can_id)
|
|
111
|
+
tgt = _call_endpoint(b, e.dst, externals, sig_to_id, app_can_id)
|
|
96
112
|
b.edge(
|
|
97
113
|
"PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.prov or []))
|
|
98
114
|
)
|
|
99
115
|
|
|
100
116
|
# Level-3 CPG overlay: each callable's v2 body/cfg/cdg/ddg. Idempotent under
|
|
101
117
|
# MERGE — a no-op when no callable carries L3 fields (levels 1/2).
|
|
102
|
-
_project_program_graphs(b, app, externals, sig_to_id)
|
|
118
|
+
_project_program_graphs(b, app, externals, sig_to_id, app_can_id)
|
|
103
119
|
|
|
104
120
|
# Neutral artifact/dependency subgraph (Task 6). L1 data — always present,
|
|
105
121
|
# full-depth-always regardless of -a.
|
|
@@ -109,7 +125,7 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
109
125
|
# _project_program_graphs above) into the config-key subgraph
|
|
110
126
|
# (ConfigKey from _project_artifacts above), plus first-class unresolved
|
|
111
127
|
# reads.
|
|
112
|
-
_project_config_uses(b, app, app_ref, externals, sig_to_id)
|
|
128
|
+
_project_config_uses(b, app, app_ref, externals, sig_to_id, app_can_id)
|
|
113
129
|
|
|
114
130
|
return b.finish()
|
|
115
131
|
|
|
@@ -128,11 +144,7 @@ def _global_ordinal(callable_id: str, local_key: str) -> str:
|
|
|
128
144
|
This MUST agree with :meth:`IdentityMap.global_id` for the same node, so the
|
|
129
145
|
JSON ``body``/``cfg`` projection and this Neo4j projection land on one node
|
|
130
146
|
identity (two-projection agreement)."""
|
|
131
|
-
return (
|
|
132
|
-
f"{callable_id}{local_key}"
|
|
133
|
-
if local_key.startswith("@")
|
|
134
|
-
else f"{callable_id}@{local_key}"
|
|
135
|
-
)
|
|
147
|
+
return global_ordinal(callable_id, local_key)
|
|
136
148
|
|
|
137
149
|
|
|
138
150
|
def _body_ref(callable_id: str, local_key: str) -> NodeRef:
|
|
@@ -140,7 +152,7 @@ def _body_ref(callable_id: str, local_key: str) -> NodeRef:
|
|
|
140
152
|
|
|
141
153
|
|
|
142
154
|
def _project_program_graphs(
|
|
143
|
-
b: RowBuilder, app: PyApplication, externals: dict, sig_to_id: dict
|
|
155
|
+
b: RowBuilder, app: PyApplication, externals: dict, sig_to_id: dict, app_can_id: str,
|
|
144
156
|
) -> None:
|
|
145
157
|
"""Level-3 CPG overlay, projected off each callable's v2 ``body``/``cfg``/
|
|
146
158
|
``cdg``/``ddg`` (populated by ``emit_l3_body`` at ``-a 3``; empty otherwise).
|
|
@@ -209,7 +221,7 @@ def _project_program_graphs(
|
|
|
209
221
|
b.edge(
|
|
210
222
|
"PY_RESOLVES_TO",
|
|
211
223
|
ref,
|
|
212
|
-
_call_endpoint(b, node.callee, externals, sig_to_id),
|
|
224
|
+
_call_endpoint(b, node.callee, externals, sig_to_id, app_can_id),
|
|
213
225
|
)
|
|
214
226
|
for e in c.cfg or []:
|
|
215
227
|
# kind-discriminated: a conditional's true/false pair between one
|
|
@@ -283,7 +295,7 @@ def _import_ghost(b: RowBuilder, app_can_id: str, name: str) -> NodeRef:
|
|
|
283
295
|
projected too, both rows collapse onto this one node — correctly, since
|
|
284
296
|
they name the same real-world symbol."""
|
|
285
297
|
return b.node(
|
|
286
|
-
["PySymbol", "PyExternal"], "id",
|
|
298
|
+
["PySymbol", "PyExternal"], "id", external_id(app_can_id, None, name), {"name": name}
|
|
287
299
|
)
|
|
288
300
|
|
|
289
301
|
|
|
@@ -399,6 +411,7 @@ def _project_artifacts(b: RowBuilder, app: PyApplication, app_name: str, app_ref
|
|
|
399
411
|
|
|
400
412
|
def _project_config_uses(
|
|
401
413
|
b: RowBuilder, app: PyApplication, app_ref: NodeRef, externals: dict, sig_to_id: dict,
|
|
414
|
+
app_can_id: str,
|
|
402
415
|
) -> None:
|
|
403
416
|
"""config_use (#162): PY_USES_CONFIG (`app.config_uses`) and
|
|
404
417
|
PY_READS_CONFIG_UNRESOLVED (`app.config_reads_unresolved`).
|
|
@@ -428,7 +441,7 @@ def _project_config_uses(
|
|
|
428
441
|
prune({"prov": list(e.prov) if e.prov else None}),
|
|
429
442
|
)
|
|
430
443
|
for r in app.config_reads_unresolved:
|
|
431
|
-
ghost_ref = _call_endpoint(b, r.callee, externals, sig_to_id)
|
|
444
|
+
ghost_ref = _call_endpoint(b, r.callee, externals, sig_to_id, app_can_id)
|
|
432
445
|
b.edge(
|
|
433
446
|
"PY_READS_CONFIG_UNRESOLVED",
|
|
434
447
|
app_ref,
|
|
@@ -453,8 +466,54 @@ def _symbol_ref(signature: str, externals: dict, sig_to_id: dict) -> NodeRef:
|
|
|
453
466
|
return NodeRef("PySymbol", "signature", signature)
|
|
454
467
|
|
|
455
468
|
|
|
469
|
+
def _base_ref_resolver(
|
|
470
|
+
b: RowBuilder, mod: PyModule, externals: dict, sig_to_id: dict, app_can_id: str,
|
|
471
|
+
) -> Callable[[str], NodeRef]:
|
|
472
|
+
"""Per-module: the written base spelling → the NodeRef PY_EXTENDS lands on (#178).
|
|
473
|
+
|
|
474
|
+
``base_classes`` stores the spelling as written (``Base``, ``views.View``),
|
|
475
|
+
while ``sig_to_id`` is keyed by signature (``pkg.mod.Base``), so the two never
|
|
476
|
+
met and every PY_EXTENDS row was dropped as dangling. Resolution order: a class
|
|
477
|
+
declared in this module (bare name or ``Outer.Inner`` path) → its can:// id; a
|
|
478
|
+
name the module's import table maps (same resolver the entrypoint pass uses)
|
|
479
|
+
that is a declared class elsewhere → its can:// id; otherwise an ``@external``
|
|
480
|
+
ghost with the id shape ``_home_external_symbols`` uses, so a call to the same
|
|
481
|
+
symbol MERGEs onto the same node."""
|
|
482
|
+
from codeanalyzer.entrypoints.pipeline import _base_resolver
|
|
483
|
+
|
|
484
|
+
local: Dict[str, str] = {}
|
|
485
|
+
|
|
486
|
+
def index(cl: PyClass, path: str) -> None:
|
|
487
|
+
local.setdefault(cl.name, cl.signature)
|
|
488
|
+
local[path] = cl.signature
|
|
489
|
+
for ic in (cl.types or {}).values():
|
|
490
|
+
index(ic, f"{path}.{ic.name}")
|
|
491
|
+
|
|
492
|
+
for cl in (mod.types or {}).values():
|
|
493
|
+
index(cl, cl.name)
|
|
494
|
+
resolve = _base_resolver(mod)
|
|
495
|
+
|
|
496
|
+
def base_ref(written: str) -> NodeRef:
|
|
497
|
+
sig = local.get(written) or resolve(written)
|
|
498
|
+
can_id = sig_to_id.get(sig)
|
|
499
|
+
if can_id is not None:
|
|
500
|
+
return _sym(can_id)
|
|
501
|
+
return _external_ghost(b, app_can_id, sig)
|
|
502
|
+
|
|
503
|
+
return base_ref
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _external_ghost(b: RowBuilder, app_can_id: str, signature: str) -> NodeRef:
|
|
507
|
+
"""A :PyExternal ghost for a dotted signature nobody homed, with the id shape
|
|
508
|
+
``_home_external_symbols`` uses — ``<app>/@external/<module>/<name>`` — so it
|
|
509
|
+
sits inside the application prefix (#173) and MERGEs with a homed twin."""
|
|
510
|
+
module, name = signature.rsplit(".", 1) if "." in signature else (None, signature)
|
|
511
|
+
ext_id = external_id(app_can_id, module, name)
|
|
512
|
+
return b.node(["PySymbol", "PyExternal"], "id", ext_id, prune({"name": name, "module": module}))
|
|
513
|
+
|
|
514
|
+
|
|
456
515
|
def _call_endpoint(
|
|
457
|
-
b: RowBuilder, signature: str, externals: dict, sig_to_id: dict
|
|
516
|
+
b: RowBuilder, signature: str, externals: dict, sig_to_id: dict, app_can_id: str,
|
|
458
517
|
) -> NodeRef:
|
|
459
518
|
"""A call-graph endpoint: a declared callable already emitted (keyed by its
|
|
460
519
|
canonical ``can://`` id, resolved through ``sig_to_id``), or an external symbol
|
|
@@ -484,13 +543,7 @@ def _call_endpoint(
|
|
|
484
543
|
ext.id or signature,
|
|
485
544
|
prune({"name": ext.name, "module": ext.module}),
|
|
486
545
|
)
|
|
487
|
-
|
|
488
|
-
return b.node(
|
|
489
|
-
["PySymbol", "PyExternal"],
|
|
490
|
-
"id",
|
|
491
|
-
signature,
|
|
492
|
-
prune({"name": name}),
|
|
493
|
-
)
|
|
546
|
+
return _external_ghost(b, app_can_id, signature)
|
|
494
547
|
|
|
495
548
|
|
|
496
549
|
# ----------------------------------------------------------------------------------------------
|
|
@@ -500,16 +553,17 @@ def _call_endpoint(
|
|
|
500
553
|
|
|
501
554
|
def _project_module_body(
|
|
502
555
|
b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: PyModule,
|
|
503
|
-
externals: dict, sig_to_id: dict, module_id_by_key: dict,
|
|
556
|
+
externals: dict, sig_to_id: dict, module_id_by_key: dict, app_can_id: str,
|
|
504
557
|
) -> None:
|
|
558
|
+
base_ref = _base_ref_resolver(b, mod, externals, sig_to_id, app_can_id)
|
|
505
559
|
for fn in (mod.functions or {}).values():
|
|
506
560
|
_project_callable(b, file_key, mod_ref, "PY_DECLARES", fn, externals, sig_to_id,
|
|
507
|
-
mod.source)
|
|
561
|
+
mod.source, base_ref)
|
|
508
562
|
for cl in (mod.types or {}).values():
|
|
509
563
|
_project_class(b, file_key, mod_ref, "PY_DECLARES", cl, externals, sig_to_id,
|
|
510
|
-
mod.source)
|
|
564
|
+
mod.source, base_ref)
|
|
511
565
|
for v in mod.variables or []:
|
|
512
|
-
_project_variable(b, file_key, mod_ref,
|
|
566
|
+
_project_variable(b, file_key, mod_ref, v)
|
|
513
567
|
_project_imports(b, mod_ref, mod, module_id_by_key)
|
|
514
568
|
|
|
515
569
|
|
|
@@ -575,7 +629,7 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule,
|
|
|
575
629
|
|
|
576
630
|
def _project_class(
|
|
577
631
|
b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass,
|
|
578
|
-
externals: dict, sig_to_id: dict, source: str,
|
|
632
|
+
externals: dict, sig_to_id: dict, source: str, base_ref: Callable[[str], NodeRef],
|
|
579
633
|
) -> None:
|
|
580
634
|
ref = b.node(
|
|
581
635
|
["PySymbol", "PyClass"], "id", cl.id, _class_props(cl, file_key, source)
|
|
@@ -587,20 +641,21 @@ def _project_class(
|
|
|
587
641
|
|
|
588
642
|
for base in cl.base_classes or []:
|
|
589
643
|
if base:
|
|
590
|
-
b.edge_to_symbol("PY_EXTENDS", ref,
|
|
644
|
+
b.edge_to_symbol("PY_EXTENDS", ref, base_ref(base))
|
|
591
645
|
|
|
592
646
|
for m in (cl.callables or {}).values():
|
|
593
647
|
_project_callable(b, file_key, ref, "PY_HAS_METHOD", m, externals, sig_to_id,
|
|
594
|
-
source)
|
|
648
|
+
source, base_ref)
|
|
595
649
|
for a in (cl.attributes or {}).values():
|
|
596
|
-
_project_attribute(b, file_key, ref,
|
|
650
|
+
_project_attribute(b, file_key, ref, a)
|
|
597
651
|
for ic in (cl.types or {}).values():
|
|
598
|
-
_project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id, source
|
|
652
|
+
_project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id, source,
|
|
653
|
+
base_ref)
|
|
599
654
|
|
|
600
655
|
|
|
601
656
|
def _project_callable(
|
|
602
657
|
b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable,
|
|
603
|
-
externals: dict, sig_to_id: dict, source: str,
|
|
658
|
+
externals: dict, sig_to_id: dict, source: str, base_ref: Callable[[str], NodeRef],
|
|
604
659
|
) -> None:
|
|
605
660
|
ref = b.node(
|
|
606
661
|
["PySymbol", "PyCallable"],
|
|
@@ -614,18 +669,22 @@ def _project_callable(
|
|
|
614
669
|
_project_decorator(b, ref, d)
|
|
615
670
|
|
|
616
671
|
for v in c.local_variables or []:
|
|
617
|
-
_project_variable(b, file_key, ref,
|
|
672
|
+
_project_variable(b, file_key, ref, v)
|
|
618
673
|
for ic in (c.callables or {}).values():
|
|
619
674
|
_project_callable(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id,
|
|
620
|
-
source)
|
|
675
|
+
source, base_ref)
|
|
621
676
|
for cl in (c.types or {}).values():
|
|
622
|
-
_project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id, source
|
|
677
|
+
_project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id, source,
|
|
678
|
+
base_ref)
|
|
623
679
|
|
|
624
680
|
|
|
625
681
|
def _project_attribute(
|
|
626
|
-
b: RowBuilder, file_key: str, owner: NodeRef,
|
|
682
|
+
b: RowBuilder, file_key: str, owner: NodeRef, a: PyClassAttribute
|
|
627
683
|
) -> None:
|
|
628
|
-
|
|
684
|
+
# ``<class can:// id>/<name>`` (#173): minted from the owner's id so it carries
|
|
685
|
+
# the application segment. The signature-minted ``service.Service.name`` it
|
|
686
|
+
# replaced was identical across applications, so two apps MERGEd onto one node.
|
|
687
|
+
attr_id = f"{owner.value}/{a.name}"
|
|
629
688
|
ref = b.node(["PyAttribute"], "id", attr_id, _attribute_props(a, attr_id, file_key))
|
|
630
689
|
b.edge("PY_HAS_ATTRIBUTE", owner, ref)
|
|
631
690
|
|
|
@@ -634,10 +693,12 @@ def _project_variable(
|
|
|
634
693
|
b: RowBuilder,
|
|
635
694
|
file_key: str,
|
|
636
695
|
owner: NodeRef,
|
|
637
|
-
owner_id: str,
|
|
638
696
|
v: PyVariableDeclaration,
|
|
639
697
|
) -> None:
|
|
640
|
-
|
|
698
|
+
# ``<owner can:// id>/<name>@<line>`` (#173) — the owner is the module or the
|
|
699
|
+
# callable, so a module-level variable sits under ``<module-id>/`` like every
|
|
700
|
+
# other declaration and the module's prefix purge reaches it.
|
|
701
|
+
var_id = f"{owner.value}/{v.name}@{v.start_line}"
|
|
641
702
|
ref = b.node(["PyVariable"], "id", var_id, _variable_props(v, var_id, file_key))
|
|
642
703
|
b.edge("PY_DECLARES_VAR", owner, ref)
|
|
643
704
|
|
codeanalyzer/neo4j/rows.py
CHANGED
|
@@ -28,6 +28,8 @@ from __future__ import annotations
|
|
|
28
28
|
from dataclasses import dataclass, field
|
|
29
29
|
from typing import Dict, List, Optional, Union
|
|
30
30
|
|
|
31
|
+
from codeanalyzer.schema.ids import SCHEME, application_id
|
|
32
|
+
|
|
31
33
|
# A property value: a primitive, or a homogeneous list of primitives.
|
|
32
34
|
Scalar = Union[str, int, float, bool]
|
|
33
35
|
Prop = Union[Scalar, List[str], List[int], List[float], List[bool]]
|
|
@@ -50,6 +52,44 @@ class NodeRow:
|
|
|
50
52
|
key_prop: str
|
|
51
53
|
value: str
|
|
52
54
|
props: Props
|
|
55
|
+
# The owning module's file key, for the incremental writer's per-module diff.
|
|
56
|
+
# In memory only (#173): it used to be emitted as ``_module`` and every
|
|
57
|
+
# destructive statement matched on it, which is application-blind. Scope now
|
|
58
|
+
# comes from the ``can://`` id prefix; this field only groups rows.
|
|
59
|
+
module: Optional[str] = None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# The marker label on every node keyed by a ``can://`` id (#173). It is an
|
|
63
|
+
# INDEX ANCHOR, nothing more: Neo4j property indexes are label-scoped, so the
|
|
64
|
+
# prefix predicate ``id STARTS WITH $p`` needs a label to seek on. Safety comes
|
|
65
|
+
# from the prefix, which carries application, language and module.
|
|
66
|
+
#
|
|
67
|
+
# The test is the SCHEME, never a language segment: since the app moved outermost
|
|
68
|
+
# an id no longer begins with the language, and ``can://python/`` now means "the
|
|
69
|
+
# application is called python". Testing that here would have quietly stripped the
|
|
70
|
+
# marker off every graph but one, taking the destructive statements' index — and
|
|
71
|
+
# their reach — with it.
|
|
72
|
+
CAN_NODE = "PyCanNode"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def descendant_prefix(can_id: str) -> str:
|
|
76
|
+
"""The prefix that matches a node's descendants and nothing else. The separator
|
|
77
|
+
is the point: ``can://app/python/src/foo.py`` is also a prefix of
|
|
78
|
+
``can://app/python/src/foo.pyX``, so descendants match on ``id + '/'`` and the
|
|
79
|
+
node itself by equality."""
|
|
80
|
+
return f"{can_id}/"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def application_prefix(app_name: Optional[str]) -> str:
|
|
84
|
+
"""``can://<app>/`` — the scope of every destructive statement. Refuses an
|
|
85
|
+
empty application: ``STARTS WITH ''`` would match every node in the database.
|
|
86
|
+
|
|
87
|
+
Since the app is the outermost segment this now covers the application's
|
|
88
|
+
artifacts and config keys too, which the old language-first prefix left
|
|
89
|
+
outside every scope and so never cleaned up."""
|
|
90
|
+
if not app_name:
|
|
91
|
+
raise ValueError("neo4j: refusing a destructive statement without an application id")
|
|
92
|
+
return descendant_prefix(application_id(app_name))
|
|
53
93
|
|
|
54
94
|
|
|
55
95
|
@dataclass
|
|
@@ -100,14 +140,20 @@ class RowBuilder:
|
|
|
100
140
|
(last write wins) and unions labels — the in-memory analog of
|
|
101
141
|
``MERGE (n:Label {key}) SET n += props``."""
|
|
102
142
|
node_id = f"{labels[0]} {value}"
|
|
143
|
+
props = dict(props)
|
|
144
|
+
module = props.pop("_module", None) # lifted off the graph (#173)
|
|
145
|
+
if key_prop == "id" and value.startswith(SCHEME) and CAN_NODE not in labels:
|
|
146
|
+
labels = [*labels, CAN_NODE]
|
|
103
147
|
existing = self._nodes.get(node_id)
|
|
104
148
|
if existing is not None:
|
|
105
149
|
existing.props.update(props)
|
|
150
|
+
if module is not None:
|
|
151
|
+
existing.module = module
|
|
106
152
|
for label in labels:
|
|
107
153
|
if label not in existing.labels:
|
|
108
154
|
existing.labels.append(label)
|
|
109
155
|
else:
|
|
110
|
-
self._nodes[node_id] = NodeRow(list(labels), key_prop, value,
|
|
156
|
+
self._nodes[node_id] = NodeRow(list(labels), key_prop, value, props, module)
|
|
111
157
|
self._keys.add((labels[0], value))
|
|
112
158
|
return NodeRef(labels[0], key_prop, value)
|
|
113
159
|
|
codeanalyzer/neo4j/schema.py
CHANGED
|
@@ -57,7 +57,9 @@ class RelType:
|
|
|
57
57
|
|
|
58
58
|
|
|
59
59
|
# Labels layered onto a node in addition to its primary/specific label.
|
|
60
|
-
|
|
60
|
+
# ``PyCanNode`` (#173) rides every node keyed by a ``can://`` id — an index
|
|
61
|
+
# anchor for the prefix-scoped destructive statements (see ``rows.CAN_NODE``).
|
|
62
|
+
MARKER_LABELS: List[str] = ["PyCanNode"]
|
|
61
63
|
|
|
62
64
|
_SPAN = {"start_line": "integer", "end_line": "integer"}
|
|
63
65
|
|
|
@@ -66,8 +68,9 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
66
68
|
NodeLabel(
|
|
67
69
|
"PyApplication",
|
|
68
70
|
"PyApplication",
|
|
69
|
-
"
|
|
71
|
+
"id",
|
|
70
72
|
{
|
|
73
|
+
"id": "string",
|
|
71
74
|
"name": "string",
|
|
72
75
|
"schema_version": "string",
|
|
73
76
|
"analyzer_name": "string",
|
|
@@ -75,6 +78,8 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
75
78
|
"repo_uri": "string",
|
|
76
79
|
"source_revision": "string",
|
|
77
80
|
"repo_dirty": "boolean",
|
|
81
|
+
"entrypoint_frameworks": "string[]",
|
|
82
|
+
"entrypoint_report_json": "string",
|
|
78
83
|
},
|
|
79
84
|
),
|
|
80
85
|
NodeLabel(
|
|
@@ -88,7 +93,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
88
93
|
"content_hash": "string",
|
|
89
94
|
"last_modified": "float",
|
|
90
95
|
"file_size": "integer",
|
|
91
|
-
"_module": "string",
|
|
92
96
|
},
|
|
93
97
|
),
|
|
94
98
|
NodeLabel(
|
|
@@ -104,7 +108,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
104
108
|
"decorators": "string[]",
|
|
105
109
|
"docstring": "string",
|
|
106
110
|
**_SPAN,
|
|
107
|
-
"_module": "string",
|
|
108
111
|
"is_entrypoint": "boolean",
|
|
109
112
|
"entrypoint_frameworks": "string[]",
|
|
110
113
|
},
|
|
@@ -128,7 +131,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
128
131
|
"modifiers": "string[]",
|
|
129
132
|
"parameters_json": "string",
|
|
130
133
|
"accessed_symbols_json": "string",
|
|
131
|
-
"_module": "string",
|
|
132
134
|
"is_entrypoint": "boolean",
|
|
133
135
|
"entrypoint_frameworks": "string[]",
|
|
134
136
|
},
|
|
@@ -157,7 +159,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
157
159
|
"initializer": "string",
|
|
158
160
|
"docstring": "string",
|
|
159
161
|
**_SPAN,
|
|
160
|
-
"_module": "string",
|
|
161
162
|
},
|
|
162
163
|
),
|
|
163
164
|
NodeLabel(
|
|
@@ -171,7 +172,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
171
172
|
"initializer": "string",
|
|
172
173
|
"scope": "string",
|
|
173
174
|
**_SPAN,
|
|
174
|
-
"_module": "string",
|
|
175
175
|
},
|
|
176
176
|
),
|
|
177
177
|
# Level-3 CPG overlay (present only at -a 3). The dataflow vocabulary is
|
|
@@ -198,7 +198,6 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
198
198
|
"is_constructor_call": "boolean",
|
|
199
199
|
"arguments_json": "string",
|
|
200
200
|
**_SPAN,
|
|
201
|
-
"_module": "string",
|
|
202
201
|
},
|
|
203
202
|
),
|
|
204
203
|
# Neutral artifact/dependency subgraph (spec 2026-08-27, Task 6). No `Py`
|
|
@@ -332,27 +331,14 @@ def uniqueness_constraints() -> list[str]:
|
|
|
332
331
|
|
|
333
332
|
CONSTRAINTS: List[str] = uniqueness_constraints()
|
|
334
333
|
|
|
335
|
-
# The labels this analyzer owns per module -- the ones carrying the internal ``_module``
|
|
336
|
-
# provenance property. Derived from NODE_LABELS so a new module-scoped label is covered
|
|
337
|
-
# without a second list to maintain. `_module` is NOT python-private: codeanalyzer-java
|
|
338
|
-
# and codeanalyzer-typescript set the same property on their nodes, so every statement
|
|
339
|
-
# matching on it must be anchored to these labels or it matches a sibling analyzer's graph
|
|
340
|
-
# in a shared database (#171).
|
|
341
|
-
MODULE_OWNED_LABELS: List[str] = [n.label for n in NODE_LABELS if "_module" in n.properties]
|
|
342
|
-
|
|
343
|
-
# The label disjunction to anchor such a statement with: ``MATCH (x:PyModule|PyClass|...)``.
|
|
344
|
-
MODULE_OWNED_PATTERN: str = "|".join(MODULE_OWNED_LABELS)
|
|
345
|
-
|
|
346
334
|
INDEXES: List[str] = [
|
|
347
335
|
"CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
|
|
348
336
|
"CREATE INDEX py_class_name IF NOT EXISTS FOR (c:PyClass) ON (c.name)",
|
|
349
337
|
"CREATE FULLTEXT INDEX py_code_fts IF NOT EXISTS FOR (c:PyCallable) ON EACH [c.code, c.docstring]",
|
|
350
|
-
|
|
351
|
-
#
|
|
352
|
-
#
|
|
353
|
-
|
|
354
|
-
f"CREATE INDEX {label.lower()}_module IF NOT EXISTS FOR (x:{label}) ON (x._module)"
|
|
355
|
-
for label in MODULE_OWNED_LABELS
|
|
338
|
+
# #173: every destructive statement is ``MATCH (x:PyCanNode) WHERE x.id STARTS WITH $p``.
|
|
339
|
+
# A range index on the marker makes that a prefix seek; without it, a store scan per
|
|
340
|
+
# changed module. STARTS WITH is index-backed; CONTAINS / ENDS WITH are not.
|
|
341
|
+
"CREATE INDEX py_can_node_id IF NOT EXISTS FOR (n:PyCanNode) ON (n.id)",
|
|
356
342
|
]
|
|
357
343
|
|
|
358
344
|
|
codeanalyzer/schema/ids.py
CHANGED
|
@@ -1,17 +1,30 @@
|
|
|
1
1
|
"""Canonical `can://` id construction for schema v2 (durable ids, ≥ callable).
|
|
2
2
|
Ordinal ids (< callable) are `ordinal_id(callable_id, tag)`. Pure functions;
|
|
3
|
-
ids are opaque handles (the <file> segment itself contains '/').
|
|
3
|
+
ids are opaque handles (the <file> segment itself contains '/').
|
|
4
|
+
|
|
5
|
+
The **application is the outermost segment** and the language sits inside it:
|
|
6
|
+
``can://<app>/python/<file>/<type>/<callable-sig>``. So ``can://<app>`` is a
|
|
7
|
+
prefix of every id this analyzer mints for that application — code, externals
|
|
8
|
+
and artifacts alike — which is what the prefix-scoped destructive statements
|
|
9
|
+
(#173) rely on. Nothing may be identified by its *language* prefix any more:
|
|
10
|
+
an application named ``python`` mints ``can://python/python/...``, so a test
|
|
11
|
+
for ``can://python/`` no longer means "a python id"; test the scheme instead."""
|
|
4
12
|
from __future__ import annotations
|
|
5
|
-
from typing import List
|
|
13
|
+
from typing import List, Optional
|
|
14
|
+
|
|
15
|
+
SCHEME = "can://"
|
|
6
16
|
|
|
7
|
-
|
|
17
|
+
# This analyzer's language segment, which sits INSIDE the app rather than above it.
|
|
18
|
+
LANG = "python"
|
|
8
19
|
|
|
9
20
|
def application_id(app_name: str) -> str:
|
|
10
|
-
|
|
21
|
+
"""``can://<app>`` — the application root, and the prefix every id below it shares."""
|
|
22
|
+
return f"{SCHEME}{app_name}"
|
|
11
23
|
|
|
12
24
|
def module_id(app_name: str, file_key: str) -> str:
|
|
25
|
+
"""``can://<app>/python/<relative-file-key>`` (separators normalized to ``/``)."""
|
|
13
26
|
rel = file_key.replace("\\", "/").lstrip("./")
|
|
14
|
-
return f"{application_id(app_name)}/{rel}"
|
|
27
|
+
return f"{application_id(app_name)}/{LANG}/{rel}"
|
|
15
28
|
|
|
16
29
|
def child_id(parent_id: str, segment: str) -> str:
|
|
17
30
|
return f"{parent_id}/{segment}"
|
|
@@ -23,13 +36,47 @@ def ordinal_id(callable_id: str, tag: str) -> str:
|
|
|
23
36
|
return f"{callable_id}@{tag}"
|
|
24
37
|
|
|
25
38
|
|
|
39
|
+
def external_id(app_id: str, module: Optional[str], name: str) -> str:
|
|
40
|
+
"""``can://<app>/@external/<module>/<name>`` — the home of a call-graph
|
|
41
|
+
endpoint that is not declared in the symbol table (an imported library or
|
|
42
|
+
builtin member). ``module`` is ``None`` for a dot-less signature, which drops
|
|
43
|
+
the segment.
|
|
44
|
+
|
|
45
|
+
Language-NEUTRAL, like ``artifact``: ``@external`` sits in the position the
|
|
46
|
+
language occupies for code nodes, so sibling analyzers over the same ``<app>``
|
|
47
|
+
name a library symbol identically and it is one node in a merged graph. The
|
|
48
|
+
cost is real and was accepted deliberately — two analyzers' notions of
|
|
49
|
+
``os.getcwd`` are not necessarily the same thing, and merging them says they
|
|
50
|
+
are. TypeScript's form; java follows it."""
|
|
51
|
+
base = f"{app_id}/@external"
|
|
52
|
+
return f"{base}/{module}/{name}" if module else f"{base}/{name}"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def global_ordinal(callable_id: str, local_key: str) -> str:
|
|
56
|
+
"""The GLOBAL ordinal id of a body node from its LOCAL key: synthetic keys
|
|
57
|
+
(`@entry`, `@formal_in:0`) already carry the `@`; positional keys (`15:2`,
|
|
58
|
+
`15:2/actual_in:0`) get one. This is the :PyBodyNode merge key and, since
|
|
59
|
+
#176, `BodyNode.id` — the one implementation both projections share."""
|
|
60
|
+
return f"{callable_id}{local_key}" if local_key.startswith("@") else f"{callable_id}@{local_key}"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def stamp_body_ids(callable) -> None:
|
|
64
|
+
"""Stamp `id` on every body node and parameter of one callable (#176).
|
|
65
|
+
Idempotent; each body emitter calls it after writing its nodes."""
|
|
66
|
+
for key, node in callable.body.items():
|
|
67
|
+
node.id = global_ordinal(callable.id, key)
|
|
68
|
+
for i, p in enumerate(callable.parameters or []):
|
|
69
|
+
p.id = ordinal_id(callable.id, f"formal_in:{i}")
|
|
70
|
+
|
|
71
|
+
|
|
26
72
|
def artifact_id(app_name: str, rel_path: str) -> str:
|
|
27
|
-
"""Language-neutral artifact id: ``can
|
|
73
|
+
"""Language-neutral artifact id: ``can://<app>/artifact/<rel-path>``.
|
|
28
74
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
same id for the same file
|
|
32
|
-
|
|
75
|
+
``artifact`` is a reserved segment in the same position the language
|
|
76
|
+
occupies for code nodes, so sibling analyzers over the same repo (same
|
|
77
|
+
``<app>``) still emit the same id for the same file — and, unlike the old
|
|
78
|
+
``can://artifact/<app>/...``, it now sits inside the application prefix."""
|
|
79
|
+
return f"{application_id(app_name)}/artifact/{rel_path}"
|
|
33
80
|
|
|
34
81
|
|
|
35
82
|
def config_key_id(artifact_id: str, dotted_key: str) -> str:
|
codeanalyzer/schema/l1_body.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""L1 body population: materialize `call` nodes from existing call sites.
|
|
2
2
|
`callee` is left None here — the sanctioned null→id refinement happens at L2."""
|
|
3
3
|
from __future__ import annotations
|
|
4
|
+
from codeanalyzer.schema.ids import stamp_body_ids
|
|
4
5
|
from codeanalyzer.schema.py_schema import PyApplication, PyClass, PyCallable, BodyNode, Span, byte_offsets
|
|
5
6
|
|
|
6
7
|
def _do_callable(source: str, c: PyCallable) -> None:
|
|
@@ -20,6 +21,7 @@ def _do_callable(source: str, c: PyCallable) -> None:
|
|
|
20
21
|
is_constructor_call=cs.is_constructor_call,
|
|
21
22
|
arguments=list(cs.arguments or []),
|
|
22
23
|
)
|
|
24
|
+
stamp_body_ids(c)
|
|
23
25
|
for ic in (c.callables or {}).values():
|
|
24
26
|
_do_callable(source, ic)
|
|
25
27
|
for icl in (c.types or {}).values():
|
codeanalyzer/schema/py_schema.py
CHANGED
|
@@ -128,6 +128,9 @@ class BodyNode(BaseModel):
|
|
|
128
128
|
"""A node in a callable's `body`: an AST region (statement/call/branch/…) or
|
|
129
129
|
a synthetic analysis vertex (entry/exit/formal_in/out/actual_in/out)."""
|
|
130
130
|
kind: str
|
|
131
|
+
# #176: the GLOBAL ordinal id — `<callable-id>@<local>` — the same value the
|
|
132
|
+
# Neo4j projection merges :PyBodyNode on. Stamped by `ids.stamp_body_ids`.
|
|
133
|
+
id: str = ""
|
|
131
134
|
span: Optional[Span] = None
|
|
132
135
|
callee: Optional[str] = None # only on `call` nodes; the sanctioned null→id slot
|
|
133
136
|
of: Optional[str] = None # param vertices: the variable/return they carry
|
|
@@ -287,6 +290,9 @@ class PyCallableParameter(BaseModel):
|
|
|
287
290
|
"""Represents a parameter of a Python callable (function/method)."""
|
|
288
291
|
|
|
289
292
|
name: str
|
|
293
|
+
# #176: `<callable-id>@formal_in:<i>` for position i — the L4 formal_in vertex
|
|
294
|
+
# that carries this parameter. A forward reference below level 4.
|
|
295
|
+
id: str = ""
|
|
290
296
|
type: Optional[str] = None
|
|
291
297
|
default_value: Optional[str] = None
|
|
292
298
|
decorators: List[PyDecorator] = []
|
|
@@ -475,7 +481,7 @@ class PyExternalSymbol(BaseModel):
|
|
|
475
481
|
builtin member. An edge-endpoint id home, not a tree node: keyed in
|
|
476
482
|
``PyApplication.external_symbols`` by its ``can://…/@external/…`` id."""
|
|
477
483
|
|
|
478
|
-
id: str = "" # can
|
|
484
|
+
id: str = "" # can://<app>/@external/<module>/<name>
|
|
479
485
|
kind: str = "external"
|
|
480
486
|
name: str # the member/short name, e.g. "get" for "requests.get"
|
|
481
487
|
module: Optional[str] = None # best-effort owning module, e.g. "requests"
|
|
@@ -504,7 +510,7 @@ class PyArtifact(BaseModel):
|
|
|
504
510
|
plain data/binary) -- never dropped from the walk. Captured broadly (node
|
|
505
511
|
+ verbatim ``source``); *meaning* is extracted narrowly -- only
|
|
506
512
|
``dependency-manifest`` roles feed ``dependencies`` today. ``id`` is
|
|
507
|
-
language-neutral (``can
|
|
513
|
+
language-neutral (``can://<app>/artifact/<path>``)."""
|
|
508
514
|
|
|
509
515
|
id: str = ""
|
|
510
516
|
kind: str = "artifact"
|