codeanalyzer-python 0.2.1__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +116 -6
- codeanalyzer/core.py +139 -213
- codeanalyzer/neo4j/__init__.py +1 -1
- codeanalyzer/neo4j/emit.py +2 -2
- codeanalyzer/neo4j/project.py +84 -18
- codeanalyzer/neo4j/schema.py +261 -15
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +20 -1
- codeanalyzer/provenance.py +61 -0
- codeanalyzer/schema/py_schema.py +39 -1
- codeanalyzer/semantic_analysis/call_graph.py +24 -3
- codeanalyzer/semantic_analysis/pycg/__init__.py +20 -0
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +1054 -0
- codeanalyzer/semantic_analysis/{codeql/__init__.py → pycg/pycg_exceptions.py} +5 -8
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +401 -0
- codeanalyzer/syntactic_analysis/import_resolver.py +67 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +74 -10
- codeanalyzer/utils/logging.py +5 -2
- codeanalyzer/utils/progress_bar.py +15 -7
- codeanalyzer_python-0.3.1.dist-info/METADATA +553 -0
- codeanalyzer_python-0.3.1.dist-info/RECORD +39 -0
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/WHEEL +1 -1
- codeanalyzer/neo4j/catalog.py +0 -245
- codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +0 -382
- codeanalyzer/semantic_analysis/codeql/codeql_exceptions.py +0 -12
- codeanalyzer/semantic_analysis/codeql/codeql_loader.py +0 -91
- codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py +0 -185
- codeanalyzer_python-0.2.1.dist-info/METADATA +0 -415
- codeanalyzer_python-0.2.1.dist-info/RECORD +0 -39
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/neo4j/__init__.py
CHANGED
|
@@ -19,7 +19,7 @@ plus the two writers (cypher snapshot / bolt incremental). Nothing here runs
|
|
|
19
19
|
unless ``--emit neo4j`` (or ``--emit schema``) is selected.
|
|
20
20
|
"""
|
|
21
21
|
from codeanalyzer.neo4j.bolt import BoltConfig, bolt_writer
|
|
22
|
-
from codeanalyzer.neo4j.
|
|
22
|
+
from codeanalyzer.neo4j.schema import (
|
|
23
23
|
MARKER_LABELS,
|
|
24
24
|
NODE_LABELS,
|
|
25
25
|
REL_TYPES,
|
codeanalyzer/neo4j/emit.py
CHANGED
|
@@ -28,7 +28,7 @@ from pathlib import Path
|
|
|
28
28
|
from typing import Optional
|
|
29
29
|
|
|
30
30
|
from codeanalyzer.neo4j.bolt import BoltConfig, bolt_writer
|
|
31
|
-
from codeanalyzer.neo4j.
|
|
31
|
+
from codeanalyzer.neo4j.schema import build_schema_document
|
|
32
32
|
from codeanalyzer.neo4j.cypher import render_cypher
|
|
33
33
|
from codeanalyzer.neo4j.project import project
|
|
34
34
|
from codeanalyzer.options import AnalysisOptions
|
|
@@ -38,7 +38,7 @@ from codeanalyzer.utils import logger
|
|
|
38
38
|
|
|
39
39
|
def emit_schema(output: Optional[Path]) -> None:
|
|
40
40
|
"""Emit the Neo4j schema contract (``schema.json``) — a static artifact derived
|
|
41
|
-
from the in-repo
|
|
41
|
+
from the in-repo schema, independent of any analyzed project. With no
|
|
42
42
|
``output`` it prints to stdout."""
|
|
43
43
|
doc = json.dumps(build_schema_document(), indent=2) + "\n"
|
|
44
44
|
if output is None:
|
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -36,7 +36,7 @@ import json
|
|
|
36
36
|
from pathlib import Path
|
|
37
37
|
from typing import Any, List, Optional
|
|
38
38
|
|
|
39
|
-
from codeanalyzer.neo4j.
|
|
39
|
+
from codeanalyzer.neo4j.schema import SCHEMA_VERSION
|
|
40
40
|
from codeanalyzer.neo4j.rows import GraphRows, NodeRef, Props, RowBuilder, prune
|
|
41
41
|
from codeanalyzer.schema import (
|
|
42
42
|
PyApplication,
|
|
@@ -53,10 +53,26 @@ from codeanalyzer.schema.py_schema import PyCallsite
|
|
|
53
53
|
def project(app: PyApplication, app_name: str) -> GraphRows:
|
|
54
54
|
b = RowBuilder()
|
|
55
55
|
|
|
56
|
-
app_ref = b.node(
|
|
56
|
+
app_ref = b.node(
|
|
57
|
+
["PyApplication"],
|
|
58
|
+
"name",
|
|
59
|
+
app_name,
|
|
60
|
+
prune(
|
|
61
|
+
{
|
|
62
|
+
"schema_version": SCHEMA_VERSION,
|
|
63
|
+
"analyzer_name": app.analyzer.name if app.analyzer else None,
|
|
64
|
+
"analyzer_version": app.analyzer.version if app.analyzer else None,
|
|
65
|
+
"repo_uri": app.repository.uri if app.repository else None,
|
|
66
|
+
"source_revision": app.repository.revision if app.repository else None,
|
|
67
|
+
"repo_dirty": app.repository.dirty if app.repository else None,
|
|
68
|
+
}
|
|
69
|
+
),
|
|
70
|
+
)
|
|
57
71
|
|
|
58
72
|
for file_key, mod in app.symbol_table.items():
|
|
59
|
-
mod_ref = b.node(
|
|
73
|
+
mod_ref = b.node(
|
|
74
|
+
["PyModule"], "file_key", file_key, _module_props(mod, file_key)
|
|
75
|
+
)
|
|
60
76
|
b.edge("PY_HAS_MODULE", app_ref, mod_ref)
|
|
61
77
|
_project_module_body(b, file_key, mod_ref, mod)
|
|
62
78
|
|
|
@@ -66,7 +82,9 @@ def project(app: PyApplication, app_name: str) -> GraphRows:
|
|
|
66
82
|
for e in app.call_graph:
|
|
67
83
|
src = _call_endpoint(b, e.source, externals)
|
|
68
84
|
tgt = _call_endpoint(b, e.target, externals)
|
|
69
|
-
b.edge(
|
|
85
|
+
b.edge(
|
|
86
|
+
"PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.provenance or []))
|
|
87
|
+
)
|
|
70
88
|
|
|
71
89
|
return b.finish()
|
|
72
90
|
|
|
@@ -86,9 +104,18 @@ def _call_endpoint(b: RowBuilder, signature: str, externals: dict) -> NodeRef:
|
|
|
86
104
|
ext = externals.get(signature)
|
|
87
105
|
if ext is None and b.has_key("PySymbol", signature):
|
|
88
106
|
return _sym(signature)
|
|
89
|
-
name =
|
|
107
|
+
name = (
|
|
108
|
+
ext.name
|
|
109
|
+
if ext is not None
|
|
110
|
+
else (signature.rsplit(".", 1)[-1] if "." in signature else signature)
|
|
111
|
+
)
|
|
90
112
|
module = ext.module if ext is not None else None
|
|
91
|
-
return b.node(
|
|
113
|
+
return b.node(
|
|
114
|
+
["PySymbol", "PyExternal"],
|
|
115
|
+
"signature",
|
|
116
|
+
signature,
|
|
117
|
+
prune({"name": name, "module": module}),
|
|
118
|
+
)
|
|
92
119
|
|
|
93
120
|
|
|
94
121
|
# ----------------------------------------------------------------------------------------------
|
|
@@ -96,7 +123,9 @@ def _call_endpoint(b: RowBuilder, signature: str, externals: dict) -> NodeRef:
|
|
|
96
123
|
# ----------------------------------------------------------------------------------------------
|
|
97
124
|
|
|
98
125
|
|
|
99
|
-
def _project_module_body(
|
|
126
|
+
def _project_module_body(
|
|
127
|
+
b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: PyModule
|
|
128
|
+
) -> None:
|
|
100
129
|
for fn in (mod.functions or {}).values():
|
|
101
130
|
_project_callable(b, file_key, mod_ref, "PY_DECLARES", fn)
|
|
102
131
|
for cl in (mod.classes or {}).values():
|
|
@@ -107,25 +136,47 @@ def _project_module_body(b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: Py
|
|
|
107
136
|
|
|
108
137
|
|
|
109
138
|
def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule) -> None:
|
|
110
|
-
#
|
|
111
|
-
#
|
|
139
|
+
# At most one PY_IMPORTS edge per (module, target) pair -- mirrors PY_CALLS,
|
|
140
|
+
# which pre-aggregates for the same reason: both writers MERGE edges on
|
|
141
|
+
# (type, from, to) and SET their props, so a second row for the same pair
|
|
142
|
+
# would silently overwrite the first in Neo4j instead of adding an edge.
|
|
143
|
+
# Buckets key on the edge's target identity (the resolved module, or the
|
|
144
|
+
# spelling itself for unresolved/external imports), so the SAME target
|
|
145
|
+
# imported under different spellings (``from pkg import util``,
|
|
146
|
+
# ``from . import util as u``, ``from .util import helper``) collapses
|
|
147
|
+
# onto one edge; the raw spellings ride along as a ``spellings`` array.
|
|
148
|
+
# Resolved internal imports point at the real :PyModule; externals keep
|
|
149
|
+
# the shared :PyPackage. Unresolved *relative* spellings (".", ".foo")
|
|
150
|
+
# name no package -- they are dropped from the graph (the spelling
|
|
151
|
+
# survives in analysis.json), instead of minting bogus
|
|
152
|
+
# :PyPackage{name: "."} nodes.
|
|
112
153
|
agg: dict = {}
|
|
113
154
|
for im in mod.imports or []:
|
|
114
155
|
if not im.module:
|
|
115
|
-
continue
|
|
116
|
-
|
|
156
|
+
continue
|
|
157
|
+
if im.resolved_module is None and im.module.startswith("."):
|
|
158
|
+
continue
|
|
159
|
+
key = im.resolved_module or im.module
|
|
160
|
+
a = agg.setdefault(
|
|
161
|
+
key, {"spellings": set(), "names": set(), "aliases": set(), "resolved": im.resolved_module}
|
|
162
|
+
)
|
|
163
|
+
a["spellings"].add(im.module)
|
|
117
164
|
if im.name:
|
|
118
165
|
a["names"].add(im.name)
|
|
119
166
|
if im.alias:
|
|
120
167
|
a["aliases"].add(im.alias)
|
|
121
|
-
for
|
|
122
|
-
|
|
168
|
+
for key, a in agg.items():
|
|
169
|
+
if a["resolved"] is not None:
|
|
170
|
+
target = NodeRef("PyModule", "file_key", a["resolved"])
|
|
171
|
+
else:
|
|
172
|
+
target = b.node(["PyPackage"], "name", key, {})
|
|
123
173
|
b.edge(
|
|
124
174
|
"PY_IMPORTS",
|
|
125
175
|
mod_ref,
|
|
126
|
-
|
|
176
|
+
target,
|
|
127
177
|
prune(
|
|
128
178
|
{
|
|
179
|
+
"spellings": sorted(a["spellings"]),
|
|
129
180
|
"imported_names": sorted(a["names"]) or None,
|
|
130
181
|
"aliases": sorted(a["aliases"]) or None,
|
|
131
182
|
}
|
|
@@ -141,7 +192,9 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule) -> None:
|
|
|
141
192
|
def _project_class(
|
|
142
193
|
b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass
|
|
143
194
|
) -> None:
|
|
144
|
-
ref = b.node(
|
|
195
|
+
ref = b.node(
|
|
196
|
+
["PySymbol", "PyClass"], "signature", cl.signature, _class_props(cl, file_key)
|
|
197
|
+
)
|
|
145
198
|
b.edge(parent_rel, parent, ref)
|
|
146
199
|
|
|
147
200
|
for base in cl.base_classes or []:
|
|
@@ -158,7 +211,12 @@ def _project_class(
|
|
|
158
211
|
def _project_callable(
|
|
159
212
|
b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable
|
|
160
213
|
) -> None:
|
|
161
|
-
ref = b.node(
|
|
214
|
+
ref = b.node(
|
|
215
|
+
["PySymbol", "PyCallable"],
|
|
216
|
+
"signature",
|
|
217
|
+
c.signature,
|
|
218
|
+
_callable_props(c, file_key),
|
|
219
|
+
)
|
|
162
220
|
b.edge(owner_rel, owner, ref)
|
|
163
221
|
|
|
164
222
|
for d in c.decorators or []:
|
|
@@ -166,7 +224,9 @@ def _project_callable(
|
|
|
166
224
|
|
|
167
225
|
for s in c.call_sites or []:
|
|
168
226
|
# Key off the relative file (a call site lives in its callable's file) so ids stay portable.
|
|
169
|
-
cs_id =
|
|
227
|
+
cs_id = (
|
|
228
|
+
f"{file_key}#{s.start_line}:{s.start_column}-{s.end_line}:{s.end_column}"
|
|
229
|
+
)
|
|
170
230
|
cs = b.node(["PyCallSite"], "id", cs_id, _call_site_props(s, file_key))
|
|
171
231
|
b.edge("PY_HAS_CALLSITE", ref, cs)
|
|
172
232
|
if s.callee_signature:
|
|
@@ -189,7 +249,11 @@ def _project_attribute(
|
|
|
189
249
|
|
|
190
250
|
|
|
191
251
|
def _project_variable(
|
|
192
|
-
b: RowBuilder,
|
|
252
|
+
b: RowBuilder,
|
|
253
|
+
file_key: str,
|
|
254
|
+
owner: NodeRef,
|
|
255
|
+
owner_id: str,
|
|
256
|
+
v: PyVariableDeclaration,
|
|
193
257
|
) -> None:
|
|
194
258
|
var_id = f"{owner_id}#{v.name}@{v.start_line}"
|
|
195
259
|
ref = b.node(["PyVariable"], "id", var_id, _variable_props(v, var_id, file_key))
|
|
@@ -258,6 +322,7 @@ def _attribute_props(a: PyClassAttribute, attr_id: str, file_key: str) -> Props:
|
|
|
258
322
|
"id": attr_id,
|
|
259
323
|
"name": a.name,
|
|
260
324
|
"type": a.type,
|
|
325
|
+
"initializer": a.initializer,
|
|
261
326
|
"docstring": _docstring_of(a.comments),
|
|
262
327
|
"start_line": a.start_line,
|
|
263
328
|
"end_line": a.end_line,
|
|
@@ -290,6 +355,7 @@ def _call_site_props(s: PyCallsite, file_key: str) -> Props:
|
|
|
290
355
|
"receiver_expr": s.receiver_expr,
|
|
291
356
|
"receiver_type": s.receiver_type,
|
|
292
357
|
"argument_types": list(s.argument_types or []),
|
|
358
|
+
"arguments_json": _stringify_if(s.arguments),
|
|
293
359
|
"return_type": s.return_type,
|
|
294
360
|
"callee_signature": s.callee_signature,
|
|
295
361
|
"is_constructor_call": s.is_constructor_call,
|
codeanalyzer/neo4j/schema.py
CHANGED
|
@@ -14,26 +14,272 @@
|
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
################################################################################
|
|
16
16
|
|
|
17
|
-
"""The Cypher DDL — uniqueness constraints and indexes — shared by both writers.
|
|
18
|
-
Run BEFORE any load so MERGE uses an index seek (not a label scan) and the
|
|
19
|
-
identity invariant is enforced by the database. Every statement is idempotent
|
|
20
|
-
(``IF NOT EXISTS``).
|
|
21
17
|
"""
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
18
|
+
The declarative Neo4j schema — the single in-repo source of truth for the graph contract: node
|
|
19
|
+
labels with their keys and typed properties, relationship types and their endpoints, and the
|
|
20
|
+
Cypher DDL (uniqueness constraints + indexes). The constraints are DERIVED from the node labels
|
|
21
|
+
(one per distinct mergeLabel/key) so a new label brings its own constraint — there is no second
|
|
22
|
+
list to keep in sync. `--emit schema` serializes all of this to a machine-readable schema.json,
|
|
23
|
+
and the conformance test (``test/test_neo4j_schema.py``) asserts the real emitter never produces a
|
|
24
|
+
label / relationship / property that isn't declared here — so this file cannot silently drift
|
|
25
|
+
from :mod:`codeanalyzer.neo4j.project`.
|
|
26
|
+
|
|
27
|
+
SCHEMA_VERSION is the contract version: bump MAJOR on a breaking change (renamed/removed label,
|
|
28
|
+
relationship or key), MINOR on an additive change (new label/rel/property). It is stamped onto
|
|
29
|
+
the :PyApplication node of every emitted graph so any consumer can detect a producer/consumer
|
|
30
|
+
mismatch at runtime.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
from dataclasses import dataclass, field
|
|
36
|
+
from typing import Dict, List
|
|
37
|
+
|
|
38
|
+
SCHEMA_VERSION = "1.2.0"
|
|
39
|
+
|
|
40
|
+
# PropType ∈ {"string", "integer", "float", "boolean", "string[]", "integer[]"}.
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class NodeLabel:
|
|
45
|
+
label: str # the specific label (also the catalog key)
|
|
46
|
+
merge_label: str # the label the uniqueness constraint / MERGE is on
|
|
47
|
+
key: str
|
|
48
|
+
properties: Dict[str, str]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class RelType:
|
|
53
|
+
type: str
|
|
54
|
+
from_labels: List[str]
|
|
55
|
+
to_labels: List[str]
|
|
56
|
+
properties: Dict[str, str] = field(default_factory=dict)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# Labels layered onto a node in addition to its primary/specific label.
|
|
60
|
+
MARKER_LABELS: List[str] = []
|
|
61
|
+
|
|
62
|
+
_SPAN = {"start_line": "integer", "end_line": "integer"}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
NODE_LABELS: List[NodeLabel] = [
|
|
66
|
+
NodeLabel(
|
|
67
|
+
"PyApplication",
|
|
68
|
+
"PyApplication",
|
|
69
|
+
"name",
|
|
70
|
+
{
|
|
71
|
+
"name": "string",
|
|
72
|
+
"schema_version": "string",
|
|
73
|
+
"analyzer_name": "string",
|
|
74
|
+
"analyzer_version": "string",
|
|
75
|
+
"repo_uri": "string",
|
|
76
|
+
"source_revision": "string",
|
|
77
|
+
"repo_dirty": "boolean",
|
|
78
|
+
},
|
|
79
|
+
),
|
|
80
|
+
NodeLabel(
|
|
81
|
+
"PyModule",
|
|
82
|
+
"PyModule",
|
|
83
|
+
"file_key",
|
|
84
|
+
{
|
|
85
|
+
"file_key": "string",
|
|
86
|
+
"module_name": "string",
|
|
87
|
+
"content_hash": "string",
|
|
88
|
+
"last_modified": "float",
|
|
89
|
+
"file_size": "integer",
|
|
90
|
+
"_module": "string",
|
|
91
|
+
},
|
|
92
|
+
),
|
|
93
|
+
NodeLabel(
|
|
94
|
+
"PyClass",
|
|
95
|
+
"PySymbol",
|
|
96
|
+
"signature",
|
|
97
|
+
{
|
|
98
|
+
"signature": "string",
|
|
99
|
+
"name": "string",
|
|
100
|
+
"code": "string",
|
|
101
|
+
"base_classes": "string[]",
|
|
102
|
+
"docstring": "string",
|
|
103
|
+
**_SPAN,
|
|
104
|
+
"_module": "string",
|
|
105
|
+
},
|
|
106
|
+
),
|
|
107
|
+
NodeLabel(
|
|
108
|
+
"PyCallable",
|
|
109
|
+
"PySymbol",
|
|
110
|
+
"signature",
|
|
111
|
+
{
|
|
112
|
+
"signature": "string",
|
|
113
|
+
"name": "string",
|
|
114
|
+
"path": "string",
|
|
115
|
+
"return_type": "string",
|
|
116
|
+
"cyclomatic_complexity": "integer",
|
|
117
|
+
"code": "string",
|
|
118
|
+
"code_start_line": "integer",
|
|
119
|
+
**_SPAN,
|
|
120
|
+
"docstring": "string",
|
|
121
|
+
"decorators": "string[]",
|
|
122
|
+
"parameters_json": "string",
|
|
123
|
+
"accessed_symbols_json": "string",
|
|
124
|
+
"_module": "string",
|
|
125
|
+
},
|
|
126
|
+
),
|
|
127
|
+
NodeLabel(
|
|
128
|
+
"PyExternal",
|
|
129
|
+
"PySymbol",
|
|
130
|
+
"signature",
|
|
131
|
+
{"signature": "string", "name": "string", "module": "string"},
|
|
132
|
+
),
|
|
133
|
+
NodeLabel("PyPackage", "PyPackage", "name", {"name": "string"}),
|
|
134
|
+
NodeLabel(
|
|
135
|
+
"PyDecorator",
|
|
136
|
+
"PyDecorator",
|
|
137
|
+
"name",
|
|
138
|
+
{"name": "string"},
|
|
139
|
+
),
|
|
140
|
+
NodeLabel(
|
|
141
|
+
"PyCallSite",
|
|
142
|
+
"PyCallSite",
|
|
143
|
+
"id",
|
|
144
|
+
{
|
|
145
|
+
"id": "string",
|
|
146
|
+
"method_name": "string",
|
|
147
|
+
"receiver_expr": "string",
|
|
148
|
+
"receiver_type": "string",
|
|
149
|
+
"argument_types": "string[]",
|
|
150
|
+
"arguments_json": "string",
|
|
151
|
+
"return_type": "string",
|
|
152
|
+
"callee_signature": "string",
|
|
153
|
+
"is_constructor_call": "boolean",
|
|
154
|
+
"start_line": "integer",
|
|
155
|
+
"start_column": "integer",
|
|
156
|
+
"end_line": "integer",
|
|
157
|
+
"end_column": "integer",
|
|
158
|
+
"_module": "string",
|
|
159
|
+
},
|
|
160
|
+
),
|
|
161
|
+
NodeLabel(
|
|
162
|
+
"PyAttribute",
|
|
163
|
+
"PyAttribute",
|
|
164
|
+
"id",
|
|
165
|
+
{
|
|
166
|
+
"id": "string",
|
|
167
|
+
"name": "string",
|
|
168
|
+
"type": "string",
|
|
169
|
+
"initializer": "string",
|
|
170
|
+
"docstring": "string",
|
|
171
|
+
**_SPAN,
|
|
172
|
+
"_module": "string",
|
|
173
|
+
},
|
|
174
|
+
),
|
|
175
|
+
NodeLabel(
|
|
176
|
+
"PyVariable",
|
|
177
|
+
"PyVariable",
|
|
178
|
+
"id",
|
|
179
|
+
{
|
|
180
|
+
"id": "string",
|
|
181
|
+
"name": "string",
|
|
182
|
+
"type": "string",
|
|
183
|
+
"initializer": "string",
|
|
184
|
+
"scope": "string",
|
|
185
|
+
**_SPAN,
|
|
186
|
+
"_module": "string",
|
|
187
|
+
},
|
|
188
|
+
),
|
|
33
189
|
]
|
|
34
190
|
|
|
191
|
+
_DECL_TARGETS = ["PyClass", "PyCallable"]
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
REL_TYPES: List[RelType] = [
|
|
195
|
+
RelType("PY_HAS_MODULE", ["PyApplication"], ["PyModule"]),
|
|
196
|
+
RelType("PY_DECLARES", ["PyModule", "PyClass", "PyCallable"], _DECL_TARGETS),
|
|
197
|
+
RelType("PY_HAS_METHOD", ["PyClass"], ["PyCallable"]),
|
|
198
|
+
RelType("PY_HAS_ATTRIBUTE", ["PyClass"], ["PyAttribute"]),
|
|
199
|
+
RelType("PY_DECLARES_VAR", ["PyModule", "PyCallable"], ["PyVariable"]),
|
|
200
|
+
RelType("PY_HAS_CALLSITE", ["PyCallable"], ["PyCallSite"]),
|
|
201
|
+
RelType("PY_RESOLVES_TO", ["PyCallSite"], ["PyCallable", "PyExternal"]),
|
|
202
|
+
RelType(
|
|
203
|
+
"PY_CALLS",
|
|
204
|
+
["PyCallable", "PyExternal"],
|
|
205
|
+
["PyCallable", "PyExternal"],
|
|
206
|
+
{"weight": "integer", "provenance": "string[]"},
|
|
207
|
+
),
|
|
208
|
+
RelType("PY_EXTENDS", ["PyClass"], ["PyClass"]),
|
|
209
|
+
RelType(
|
|
210
|
+
"PY_IMPORTS",
|
|
211
|
+
["PyModule"],
|
|
212
|
+
["PyModule", "PyPackage"],
|
|
213
|
+
{"spellings": "string[]", "imported_names": "string[]", "aliases": "string[]"},
|
|
214
|
+
),
|
|
215
|
+
RelType("PY_DECORATED_BY", ["PyCallable"], ["PyDecorator"]),
|
|
216
|
+
]
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def uniqueness_constraints() -> list[str]:
|
|
220
|
+
"""One uniqueness constraint per distinct (merge_label, key)."""
|
|
221
|
+
seen: set[tuple[str, str]] = set()
|
|
222
|
+
out: list[str] = []
|
|
223
|
+
|
|
224
|
+
for node in NODE_LABELS:
|
|
225
|
+
identifier = (node.merge_label, node.key)
|
|
226
|
+
if identifier in seen:
|
|
227
|
+
continue
|
|
228
|
+
|
|
229
|
+
seen.add(identifier)
|
|
230
|
+
out.append(
|
|
231
|
+
f"CREATE CONSTRAINT {node.merge_label.lower()}_{node.key} "
|
|
232
|
+
f"IF NOT EXISTS FOR (x:{node.merge_label}) "
|
|
233
|
+
f"REQUIRE x.{node.key} IS UNIQUE"
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
return out
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
CONSTRAINTS: List[str] = uniqueness_constraints()
|
|
240
|
+
|
|
35
241
|
INDEXES: List[str] = [
|
|
36
242
|
"CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
|
|
37
243
|
"CREATE INDEX py_class_name IF NOT EXISTS FOR (c:PyClass) ON (c.name)",
|
|
38
244
|
"CREATE FULLTEXT INDEX py_code_fts IF NOT EXISTS FOR (c:PyCallable) ON EACH [c.code, c.docstring]",
|
|
39
245
|
]
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@dataclass
|
|
249
|
+
class SchemaDocument:
|
|
250
|
+
schema_version: str
|
|
251
|
+
generator: str
|
|
252
|
+
marker_labels: List[str]
|
|
253
|
+
node_labels: List[NodeLabel]
|
|
254
|
+
relationship_types: List[RelType]
|
|
255
|
+
constraints: List[str]
|
|
256
|
+
indexes: List[str]
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def build_schema_document() -> dict:
|
|
260
|
+
"""Build the full machine-readable schema document emitted by ``--emit schema``."""
|
|
261
|
+
return {
|
|
262
|
+
"schema_version": SCHEMA_VERSION,
|
|
263
|
+
"generator": "codeanalyzer-python",
|
|
264
|
+
"marker_labels": list(MARKER_LABELS),
|
|
265
|
+
"node_labels": [
|
|
266
|
+
{
|
|
267
|
+
"label": n.label,
|
|
268
|
+
"merge_label": n.merge_label,
|
|
269
|
+
"key": n.key,
|
|
270
|
+
"properties": n.properties,
|
|
271
|
+
}
|
|
272
|
+
for n in NODE_LABELS
|
|
273
|
+
],
|
|
274
|
+
"relationship_types": [
|
|
275
|
+
{
|
|
276
|
+
"type": r.type,
|
|
277
|
+
"from": r.from_labels,
|
|
278
|
+
"to": r.to_labels,
|
|
279
|
+
"properties": r.properties,
|
|
280
|
+
}
|
|
281
|
+
for r in REL_TYPES
|
|
282
|
+
],
|
|
283
|
+
"constraints": list(CONSTRAINTS),
|
|
284
|
+
"indexes": list(INDEXES),
|
|
285
|
+
}
|
codeanalyzer/options/__init__.py
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
from .options import AnalysisOptions, EmitTarget, OutputFormat
|
|
1
|
+
from .options import AnalysisOptions, EmitTarget, OutputFormat, ShardStrategy
|
|
2
2
|
|
|
3
|
-
__all__ = ["AnalysisOptions", "EmitTarget", "OutputFormat"]
|
|
3
|
+
__all__ = ["AnalysisOptions", "EmitTarget", "OutputFormat", "ShardStrategy"]
|
codeanalyzer/options/options.py
CHANGED
|
@@ -23,6 +23,20 @@ class EmitTarget(str, Enum):
|
|
|
23
23
|
SCHEMA = "schema"
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
class ShardStrategy(str, Enum):
|
|
27
|
+
"""How ``--pycg-shard`` groups files into shards (level 2 only).
|
|
28
|
+
|
|
29
|
+
- ``jedi`` : partition the Jedi module-dependency graph (strongly-
|
|
30
|
+
connected-component condensation + Louvain) so tightly-
|
|
31
|
+
coupled modules co-compute and few call edges are severed
|
|
32
|
+
between shards. Import cycles are never split.
|
|
33
|
+
- ``package`` : legacy one-shard-per-package-directory grouping.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
JEDI = "jedi"
|
|
37
|
+
PACKAGE = "package"
|
|
38
|
+
|
|
39
|
+
|
|
26
40
|
@dataclass
|
|
27
41
|
class AnalysisOptions:
|
|
28
42
|
input: Path
|
|
@@ -34,7 +48,7 @@ class AnalysisOptions:
|
|
|
34
48
|
neo4j_user: str = "neo4j"
|
|
35
49
|
neo4j_password: str = "neo4j"
|
|
36
50
|
neo4j_database: Optional[str] = None
|
|
37
|
-
|
|
51
|
+
analysis_level: int = 1
|
|
38
52
|
using_ray: bool = False
|
|
39
53
|
rebuild_analysis: bool = False
|
|
40
54
|
skip_tests: bool = True
|
|
@@ -43,3 +57,8 @@ class AnalysisOptions:
|
|
|
43
57
|
cache_dir: Optional[Path] = None
|
|
44
58
|
clear_cache: bool = False
|
|
45
59
|
verbosity: int = 0
|
|
60
|
+
pycg_shard: bool = False
|
|
61
|
+
pycg_shard_ceiling: int = 100
|
|
62
|
+
pycg_shard_timeout: int = 120
|
|
63
|
+
pycg_shard_strategy: ShardStrategy = ShardStrategy.JEDI
|
|
64
|
+
pycg_max_iter: int = 50
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Provenance capture: repository (git) and analyzer identity for a snapshot.
|
|
2
|
+
|
|
3
|
+
Runs git only as a subprocess query against the analyzed project directory —
|
|
4
|
+
never mutates anything. Absence of git (no repo, no binary) degrades to None.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import subprocess
|
|
9
|
+
from importlib.metadata import PackageNotFoundError, version as _pkg_version
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Optional, Union
|
|
12
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
13
|
+
|
|
14
|
+
from codeanalyzer.schema.py_schema import PyAnalyzerInfo, PyRepositoryInfo
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _strip_userinfo(uri: str) -> str:
|
|
18
|
+
"""Drop credentials from URL-style remotes; scp-style git@host:path is a
|
|
19
|
+
username, not a secret, and parses with no netloc — left intact."""
|
|
20
|
+
parts = urlsplit(uri)
|
|
21
|
+
if parts.netloc and "@" in parts.netloc:
|
|
22
|
+
return urlunsplit(parts._replace(netloc=parts.netloc.rpartition("@")[2]))
|
|
23
|
+
return uri
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _git(project_dir: Union[Path, str], *args: str) -> Optional[str]:
|
|
27
|
+
"""One git query; None on any failure (no repo, no git, timeout)."""
|
|
28
|
+
try:
|
|
29
|
+
result = subprocess.run(
|
|
30
|
+
["git", "-C", str(project_dir), *args],
|
|
31
|
+
capture_output=True, text=True, timeout=10, check=False,
|
|
32
|
+
)
|
|
33
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
34
|
+
return None
|
|
35
|
+
if result.returncode != 0:
|
|
36
|
+
return None
|
|
37
|
+
return result.stdout.strip()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def repository_info(project_dir: Union[Path, str]) -> Optional[PyRepositoryInfo]:
|
|
41
|
+
"""Git provenance of ``project_dir``, or None when it isn't a git checkout."""
|
|
42
|
+
revision = _git(project_dir, "rev-parse", "HEAD")
|
|
43
|
+
if not revision:
|
|
44
|
+
return None
|
|
45
|
+
raw_uri = _git(project_dir, "remote", "get-url", "origin")
|
|
46
|
+
uri = _strip_userinfo(raw_uri) if raw_uri else None
|
|
47
|
+
status = _git(project_dir, "status", "--porcelain", "--untracked-files=no")
|
|
48
|
+
return PyRepositoryInfo(uri=uri, revision=revision, dirty=bool(status))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def analyzer_info(analysis_level: int) -> PyAnalyzerInfo:
|
|
52
|
+
"""Identity + configuration of this analyzer run."""
|
|
53
|
+
try:
|
|
54
|
+
version = _pkg_version("codeanalyzer-python")
|
|
55
|
+
except PackageNotFoundError:
|
|
56
|
+
version = "unknown"
|
|
57
|
+
return PyAnalyzerInfo(
|
|
58
|
+
name="codeanalyzer-python",
|
|
59
|
+
version=version,
|
|
60
|
+
config={"analysis_level": analysis_level},
|
|
61
|
+
)
|