codeanalyzer-python 0.1.14__py3-none-any.whl → 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +108 -6
- codeanalyzer/core.py +100 -18
- codeanalyzer/neo4j/__init__.py +46 -0
- codeanalyzer/neo4j/bolt.py +234 -0
- codeanalyzer/neo4j/catalog.py +245 -0
- codeanalyzer/neo4j/cypher.py +138 -0
- codeanalyzer/neo4j/emit.py +74 -0
- codeanalyzer/neo4j/project.py +330 -0
- codeanalyzer/neo4j/rows.py +181 -0
- codeanalyzer/neo4j/schema.py +39 -0
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +21 -0
- codeanalyzer/schema/__init__.py +2 -0
- codeanalyzer/schema/py_schema.py +15 -0
- codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +109 -27
- codeanalyzer_python-0.2.1.dist-info/METADATA +415 -0
- codeanalyzer_python-0.2.1.dist-info/RECORD +39 -0
- {codeanalyzer_python-0.1.14.dist-info → codeanalyzer_python-0.2.1.dist-info}/WHEEL +1 -1
- codeanalyzer_python-0.2.1.dist-info/entry_points.txt +3 -0
- codeanalyzer_python-0.1.14.dist-info/METADATA +0 -392
- codeanalyzer_python-0.1.14.dist-info/RECORD +0 -31
- codeanalyzer_python-0.1.14.dist-info/entry_points.txt +0 -2
- {codeanalyzer_python-0.1.14.dist-info → codeanalyzer_python-0.2.1.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.1.14.dist-info → codeanalyzer_python-0.2.1.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/__main__.py
CHANGED
|
@@ -7,13 +7,18 @@ from codeanalyzer.core import Codeanalyzer
|
|
|
7
7
|
from codeanalyzer.utils import _set_log_level, logger
|
|
8
8
|
from codeanalyzer.config import OutputFormat
|
|
9
9
|
from codeanalyzer.schema import model_dump_json
|
|
10
|
-
from codeanalyzer.options import AnalysisOptions
|
|
10
|
+
from codeanalyzer.options import AnalysisOptions, EmitTarget
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def main(
|
|
14
14
|
input: Annotated[
|
|
15
|
-
Path,
|
|
16
|
-
|
|
15
|
+
Optional[Path],
|
|
16
|
+
typer.Option(
|
|
17
|
+
"-i",
|
|
18
|
+
"--input",
|
|
19
|
+
help="Path to the project root directory (not required for --emit schema).",
|
|
20
|
+
),
|
|
21
|
+
] = None,
|
|
17
22
|
output: Annotated[
|
|
18
23
|
Optional[Path],
|
|
19
24
|
typer.Option("-o", "--output", help="Output directory for artifacts."),
|
|
@@ -23,10 +28,61 @@ def main(
|
|
|
23
28
|
typer.Option(
|
|
24
29
|
"-f",
|
|
25
30
|
"--format",
|
|
26
|
-
help="Output format: json or msgpack.",
|
|
31
|
+
help="Output format for --emit json: json or msgpack.",
|
|
27
32
|
case_sensitive=False,
|
|
28
33
|
),
|
|
29
34
|
] = OutputFormat.JSON,
|
|
35
|
+
emit: Annotated[
|
|
36
|
+
EmitTarget,
|
|
37
|
+
typer.Option(
|
|
38
|
+
"--emit",
|
|
39
|
+
help="Output target: json (analysis.json, default) | neo4j (graph.cypher or live "
|
|
40
|
+
"Bolt push) | schema (the Neo4j schema.json contract).",
|
|
41
|
+
case_sensitive=False,
|
|
42
|
+
),
|
|
43
|
+
] = EmitTarget.JSON,
|
|
44
|
+
app_name: Annotated[
|
|
45
|
+
Optional[str],
|
|
46
|
+
typer.Option(
|
|
47
|
+
"--app-name",
|
|
48
|
+
help="Logical application name for the graph :PyApplication anchor "
|
|
49
|
+
"(default: input dir name).",
|
|
50
|
+
),
|
|
51
|
+
] = None,
|
|
52
|
+
neo4j_uri: Annotated[
|
|
53
|
+
Optional[str],
|
|
54
|
+
typer.Option(
|
|
55
|
+
"--neo4j-uri",
|
|
56
|
+
envvar="NEO4J_URI",
|
|
57
|
+
help="Push the graph to a live Neo4j over Bolt (incremental); omit to write "
|
|
58
|
+
"graph.cypher. [env: NEO4J_URI]",
|
|
59
|
+
),
|
|
60
|
+
] = None,
|
|
61
|
+
neo4j_user: Annotated[
|
|
62
|
+
str,
|
|
63
|
+
typer.Option(
|
|
64
|
+
"--neo4j-user",
|
|
65
|
+
envvar="NEO4J_USERNAME",
|
|
66
|
+
help="Neo4j username. [env: NEO4J_USERNAME]",
|
|
67
|
+
),
|
|
68
|
+
] = "neo4j",
|
|
69
|
+
neo4j_password: Annotated[
|
|
70
|
+
str,
|
|
71
|
+
typer.Option(
|
|
72
|
+
"--neo4j-password",
|
|
73
|
+
envvar="NEO4J_PASSWORD",
|
|
74
|
+
help="Neo4j password. Prefer the env var over the flag (the flag is visible in shell "
|
|
75
|
+
"history / process list). [env: NEO4J_PASSWORD]",
|
|
76
|
+
),
|
|
77
|
+
] = "neo4j",
|
|
78
|
+
neo4j_database: Annotated[
|
|
79
|
+
Optional[str],
|
|
80
|
+
typer.Option(
|
|
81
|
+
"--neo4j-database",
|
|
82
|
+
envvar="NEO4J_DATABASE",
|
|
83
|
+
help="Neo4j database name (default: server default). [env: NEO4J_DATABASE]",
|
|
84
|
+
),
|
|
85
|
+
] = None,
|
|
30
86
|
using_codeql: Annotated[
|
|
31
87
|
bool, typer.Option("--codeql/--no-codeql", help="Enable CodeQL-based analysis.")
|
|
32
88
|
] = False,
|
|
@@ -48,6 +104,14 @@ def main(
|
|
|
48
104
|
help="Skip test files in analysis.",
|
|
49
105
|
),
|
|
50
106
|
] = True,
|
|
107
|
+
no_venv: Annotated[
|
|
108
|
+
bool,
|
|
109
|
+
typer.Option(
|
|
110
|
+
"--no-venv/--venv",
|
|
111
|
+
help="Skip virtualenv creation and dependency installation; resolve "
|
|
112
|
+
"imports against the ambient Python environment instead.",
|
|
113
|
+
),
|
|
114
|
+
] = False,
|
|
51
115
|
file_name: Annotated[
|
|
52
116
|
Optional[Path],
|
|
53
117
|
typer.Option(
|
|
@@ -78,10 +142,17 @@ def main(
|
|
|
78
142
|
input=input,
|
|
79
143
|
output=output,
|
|
80
144
|
format=format,
|
|
145
|
+
emit=emit,
|
|
146
|
+
app_name=app_name,
|
|
147
|
+
neo4j_uri=neo4j_uri,
|
|
148
|
+
neo4j_user=neo4j_user,
|
|
149
|
+
neo4j_password=neo4j_password,
|
|
150
|
+
neo4j_database=neo4j_database,
|
|
81
151
|
using_codeql=using_codeql,
|
|
82
152
|
using_ray=using_ray,
|
|
83
153
|
rebuild_analysis=rebuild_analysis,
|
|
84
154
|
skip_tests=skip_tests,
|
|
155
|
+
no_venv=no_venv,
|
|
85
156
|
file_name=file_name,
|
|
86
157
|
cache_dir=cache_dir,
|
|
87
158
|
clear_cache=clear_cache,
|
|
@@ -89,6 +160,18 @@ def main(
|
|
|
89
160
|
)
|
|
90
161
|
|
|
91
162
|
_set_log_level(options.verbosity)
|
|
163
|
+
|
|
164
|
+
# The schema contract is a static artifact — no project analysis required.
|
|
165
|
+
if options.emit == EmitTarget.SCHEMA:
|
|
166
|
+
from codeanalyzer.neo4j.emit import emit_schema
|
|
167
|
+
|
|
168
|
+
emit_schema(options.output)
|
|
169
|
+
return
|
|
170
|
+
|
|
171
|
+
# Every other target requires an input project.
|
|
172
|
+
if options.input is None:
|
|
173
|
+
logger.error("Missing option '-i' / '--input' (required for --emit json | neo4j).")
|
|
174
|
+
raise typer.Exit(code=1)
|
|
92
175
|
if not options.input.exists():
|
|
93
176
|
logger.error(f"Input path '{options.input}' does not exist.")
|
|
94
177
|
raise typer.Exit(code=1)
|
|
@@ -112,7 +195,11 @@ def main(
|
|
|
112
195
|
with Codeanalyzer(options) as analyzer:
|
|
113
196
|
artifacts = analyzer.analyze()
|
|
114
197
|
|
|
115
|
-
if options.
|
|
198
|
+
if options.emit == EmitTarget.NEO4J:
|
|
199
|
+
from codeanalyzer.neo4j.emit import emit_neo4j
|
|
200
|
+
|
|
201
|
+
emit_neo4j(artifacts, options)
|
|
202
|
+
elif options.output is None:
|
|
116
203
|
print(model_dump_json(artifacts, separators=(",", ":")))
|
|
117
204
|
else:
|
|
118
205
|
options.output.mkdir(parents=True, exist_ok=True)
|
|
@@ -142,7 +229,7 @@ def _write_output(artifacts, output_dir: Path, format: OutputFormat):
|
|
|
142
229
|
|
|
143
230
|
app = typer.Typer(
|
|
144
231
|
callback=main,
|
|
145
|
-
name="
|
|
232
|
+
name="canpy",
|
|
146
233
|
help="Static Analysis on Python source code using Jedi, CodeQL and Tree sitter.",
|
|
147
234
|
invoke_without_command=True,
|
|
148
235
|
no_args_is_help=True,
|
|
@@ -151,5 +238,20 @@ app = typer.Typer(
|
|
|
151
238
|
pretty_exceptions_show_locals=False,
|
|
152
239
|
)
|
|
153
240
|
|
|
241
|
+
def deprecated_main() -> None:
|
|
242
|
+
"""Entry point for the legacy ``codeanalyzer`` command. Prints a one-line
|
|
243
|
+
deprecation notice to stderr (so piped stdout — e.g. ``--emit schema`` — stays
|
|
244
|
+
clean) and then runs the CLI unchanged. Kept for backwards compatibility; will
|
|
245
|
+
be removed in a future release."""
|
|
246
|
+
import sys
|
|
247
|
+
|
|
248
|
+
print(
|
|
249
|
+
"codeanalyzer: this command has been renamed to `canpy`. The `codeanalyzer` "
|
|
250
|
+
"alias is deprecated and will be removed in a future release — please use `canpy`.",
|
|
251
|
+
file=sys.stderr,
|
|
252
|
+
)
|
|
253
|
+
app()
|
|
254
|
+
|
|
255
|
+
|
|
154
256
|
if __name__ == "__main__":
|
|
155
257
|
app()
|
codeanalyzer/core.py
CHANGED
|
@@ -8,7 +8,13 @@ from typing import Any, Dict, Optional, Union, List
|
|
|
8
8
|
|
|
9
9
|
import ray
|
|
10
10
|
from codeanalyzer.utils import logger
|
|
11
|
-
from codeanalyzer.schema import
|
|
11
|
+
from codeanalyzer.schema import (
|
|
12
|
+
PyApplication,
|
|
13
|
+
PyExternalSymbol,
|
|
14
|
+
PyModule,
|
|
15
|
+
model_dump_json,
|
|
16
|
+
model_validate_json,
|
|
17
|
+
)
|
|
12
18
|
from codeanalyzer.schema.py_schema import PyCallEdge
|
|
13
19
|
from codeanalyzer.semantic_analysis.call_graph import (
|
|
14
20
|
jedi_call_graph_edges,
|
|
@@ -60,6 +66,7 @@ class Codeanalyzer:
|
|
|
60
66
|
self.skip_tests = options.skip_tests
|
|
61
67
|
self.using_codeql = options.using_codeql
|
|
62
68
|
self.rebuild_analysis = options.rebuild_analysis
|
|
69
|
+
self.no_venv = options.no_venv
|
|
63
70
|
self.cache_dir = (
|
|
64
71
|
options.cache_dir.resolve() if options.cache_dir is not None else self.project_dir
|
|
65
72
|
) / ".codeanalyzer"
|
|
@@ -226,13 +233,41 @@ class Codeanalyzer:
|
|
|
226
233
|
f"a working Python interpreter that can create virtual environments."
|
|
227
234
|
)
|
|
228
235
|
|
|
236
|
+
@staticmethod
|
|
237
|
+
def _uv_bin() -> Optional[str]:
|
|
238
|
+
"""Path to a uv binary: the one bundled with the ``uv`` PyPI package (a
|
|
239
|
+
dependency, so normally always present -- including inside a Docker image),
|
|
240
|
+
else a uv on PATH, else ``None`` (callers fall back to pip)."""
|
|
241
|
+
try:
|
|
242
|
+
from uv import find_uv_bin
|
|
243
|
+
|
|
244
|
+
return str(find_uv_bin())
|
|
245
|
+
except Exception:
|
|
246
|
+
return shutil.which("uv")
|
|
247
|
+
|
|
248
|
+
def _install_into_venv(self, venv_python: Path, args: List[str]) -> None:
|
|
249
|
+
"""Install packages into the target venv, preferring uv for speed (parallel
|
|
250
|
+
downloads + a shared global cache) and falling back to the venv's own pip
|
|
251
|
+
when uv is unavailable."""
|
|
252
|
+
uv = self._uv_bin()
|
|
253
|
+
if uv:
|
|
254
|
+
cmd = [uv, "pip", "install", "--python", str(venv_python), *args]
|
|
255
|
+
else:
|
|
256
|
+
cmd = [str(venv_python), "-m", "pip", "install", *args]
|
|
257
|
+
self._cmd_exec_helper(cmd, cwd=self.project_dir, check=True)
|
|
258
|
+
|
|
229
259
|
def __enter__(self) -> "Codeanalyzer":
|
|
230
260
|
# If no virtualenv is provided, try to create one using requirements.txt or pyproject.toml
|
|
231
261
|
venv_path = self.cache_dir / self.project_dir.name / "virtualenv"
|
|
232
262
|
# Ensure the cache directory exists for this project
|
|
233
263
|
venv_path.parent.mkdir(parents=True, exist_ok=True)
|
|
264
|
+
if self.no_venv:
|
|
265
|
+
logger.info(
|
|
266
|
+
"--no-venv: using the ambient Python environment "
|
|
267
|
+
"(skipping virtualenv creation and dependency installation)"
|
|
268
|
+
)
|
|
234
269
|
# Create the virtual environment if it does not exist
|
|
235
|
-
if not venv_path.exists() or self.rebuild_analysis:
|
|
270
|
+
if not self.no_venv and (not venv_path.exists() or self.rebuild_analysis):
|
|
236
271
|
logger.info(f"(Re-)creating virtual environment at {venv_path}")
|
|
237
272
|
self._cmd_exec_helper(
|
|
238
273
|
[str(self._get_base_interpreter()), "-m", "venv", str(venv_path)],
|
|
@@ -249,24 +284,19 @@ class Codeanalyzer:
|
|
|
249
284
|
("test-requirements.txt", ["-r"]),
|
|
250
285
|
]
|
|
251
286
|
|
|
252
|
-
for dep_file,
|
|
287
|
+
for dep_file, _ in dependency_files:
|
|
253
288
|
if (self.project_dir / dep_file).exists():
|
|
254
289
|
logger.info(f"Installing dependencies from {dep_file}")
|
|
255
|
-
self.
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
check=True,
|
|
290
|
+
self._install_into_venv(
|
|
291
|
+
venv_python,
|
|
292
|
+
["--upgrade", "-r", str(self.project_dir / dep_file)],
|
|
259
293
|
)
|
|
260
294
|
|
|
261
295
|
# Handle Pipenv files
|
|
262
296
|
if (self.project_dir / "Pipfile").exists():
|
|
263
297
|
logger.info("Installing dependencies from Pipfile")
|
|
264
298
|
# Note: This would require pipenv to be installed
|
|
265
|
-
self.
|
|
266
|
-
[str(venv_python), "-m", "pip", "install", "pipenv"],
|
|
267
|
-
cwd=self.project_dir,
|
|
268
|
-
check=True,
|
|
269
|
-
)
|
|
299
|
+
self._install_into_venv(venv_python, ["pipenv"])
|
|
270
300
|
self._cmd_exec_helper(
|
|
271
301
|
["pipenv", "install", "--dev"],
|
|
272
302
|
cwd=self.project_dir,
|
|
@@ -289,14 +319,18 @@ class Codeanalyzer:
|
|
|
289
319
|
|
|
290
320
|
if any((self.project_dir / file).exists() for file in package_definition_files):
|
|
291
321
|
logger.info("Installing project in editable mode")
|
|
292
|
-
self.
|
|
293
|
-
[str(venv_python), "-m", "pip", "install", "-e", str(self.project_dir)],
|
|
294
|
-
cwd=self.project_dir,
|
|
295
|
-
check=True,
|
|
296
|
-
)
|
|
322
|
+
self._install_into_venv(venv_python, ["-e", str(self.project_dir)])
|
|
297
323
|
else:
|
|
298
324
|
logger.warning("No package definition files found, skipping editable installation")
|
|
299
325
|
|
|
326
|
+
# Point Jedi at the analysis venv so it resolves the project's third-party
|
|
327
|
+
# imports. This runs on both a fresh build and a lazy reuse of an existing
|
|
328
|
+
# venv -- previously self.virtualenv stayed None, so the install above was
|
|
329
|
+
# never actually used by the symbol-table builder. With --no-venv we leave
|
|
330
|
+
# it None so Jedi resolves against the ambient interpreter instead.
|
|
331
|
+
if not self.no_venv and venv_path.exists():
|
|
332
|
+
self.virtualenv = venv_path
|
|
333
|
+
|
|
300
334
|
if self.using_codeql:
|
|
301
335
|
logger.info(f"(Re-)initializing CodeQL analysis for {self.project_dir}")
|
|
302
336
|
|
|
@@ -358,6 +392,43 @@ class Codeanalyzer:
|
|
|
358
392
|
logger.info(f"Clearing cache directory: {self.cache_dir}")
|
|
359
393
|
shutil.rmtree(self.cache_dir)
|
|
360
394
|
|
|
395
|
+
@staticmethod
|
|
396
|
+
def _compute_external_symbols(symbol_table, call_graph):
|
|
397
|
+
"""Build the external-symbol map: every call-graph endpoint whose signature
|
|
398
|
+
is not a declared class/callable in the symbol table is an external (an
|
|
399
|
+
imported library or builtin member). ``name``/``module`` are derived from
|
|
400
|
+
the signature (best effort: split on the last dot)."""
|
|
401
|
+
declared = set()
|
|
402
|
+
|
|
403
|
+
def walk_callable(c):
|
|
404
|
+
declared.add(c.signature)
|
|
405
|
+
for ic in (c.inner_callables or {}).values():
|
|
406
|
+
walk_callable(ic)
|
|
407
|
+
for cl in (c.inner_classes or {}).values():
|
|
408
|
+
walk_class(cl)
|
|
409
|
+
|
|
410
|
+
def walk_class(cl):
|
|
411
|
+
declared.add(cl.signature)
|
|
412
|
+
for m in (cl.methods or {}).values():
|
|
413
|
+
walk_callable(m)
|
|
414
|
+
for ic in (cl.inner_classes or {}).values():
|
|
415
|
+
walk_class(ic)
|
|
416
|
+
|
|
417
|
+
for mod in symbol_table.values():
|
|
418
|
+
for c in (mod.functions or {}).values():
|
|
419
|
+
walk_callable(c)
|
|
420
|
+
for cl in (mod.classes or {}).values():
|
|
421
|
+
walk_class(cl)
|
|
422
|
+
|
|
423
|
+
externals: Dict[str, PyExternalSymbol] = {}
|
|
424
|
+
for edge in call_graph:
|
|
425
|
+
for sig in (edge.source, edge.target):
|
|
426
|
+
if sig in declared or sig in externals:
|
|
427
|
+
continue
|
|
428
|
+
module, name = sig.rsplit(".", 1) if "." in sig else (sig, sig)
|
|
429
|
+
externals[sig] = PyExternalSymbol(name=name, module=module)
|
|
430
|
+
return externals
|
|
431
|
+
|
|
361
432
|
def analyze(self) -> PyApplication:
|
|
362
433
|
"""Analyze the project and return a PyApplication with symbol table.
|
|
363
434
|
|
|
@@ -397,8 +468,19 @@ class Codeanalyzer:
|
|
|
397
468
|
jedi_edges = jedi_call_graph_edges(symbol_table)
|
|
398
469
|
call_graph = merge_edges(jedi_edges, codeql_edges)
|
|
399
470
|
|
|
471
|
+
# Classify call-graph endpoints that are not declared in the symbol table
|
|
472
|
+
# (imported library / builtin members) once, so the JSON and Neo4j backends
|
|
473
|
+
# share one authoritative external-symbol set.
|
|
474
|
+
external_symbols = self._compute_external_symbols(symbol_table, call_graph)
|
|
475
|
+
|
|
400
476
|
# Recreate pyapplication
|
|
401
|
-
app =
|
|
477
|
+
app = (
|
|
478
|
+
PyApplication.builder()
|
|
479
|
+
.symbol_table(symbol_table)
|
|
480
|
+
.call_graph(call_graph)
|
|
481
|
+
.external_symbols(external_symbols)
|
|
482
|
+
.build()
|
|
483
|
+
)
|
|
402
484
|
|
|
403
485
|
# Save to cache
|
|
404
486
|
self._save_analysis_cache(app, cache_file)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
################################################################################
|
|
2
|
+
# Copyright IBM Corporation 2025
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
################################################################################
|
|
16
|
+
|
|
17
|
+
"""Neo4j output: a pure projection of the :class:`PyApplication` IR to graph rows,
|
|
18
|
+
plus the two writers (cypher snapshot / bolt incremental). Nothing here runs
|
|
19
|
+
unless ``--emit neo4j`` (or ``--emit schema``) is selected.
|
|
20
|
+
"""
|
|
21
|
+
from codeanalyzer.neo4j.bolt import BoltConfig, bolt_writer
|
|
22
|
+
from codeanalyzer.neo4j.catalog import (
|
|
23
|
+
MARKER_LABELS,
|
|
24
|
+
NODE_LABELS,
|
|
25
|
+
REL_TYPES,
|
|
26
|
+
SCHEMA_VERSION,
|
|
27
|
+
build_schema_document,
|
|
28
|
+
)
|
|
29
|
+
from codeanalyzer.neo4j.cypher import render_cypher
|
|
30
|
+
from codeanalyzer.neo4j.project import project
|
|
31
|
+
from codeanalyzer.neo4j.rows import EdgeRow, GraphRows, NodeRow
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"project",
|
|
35
|
+
"render_cypher",
|
|
36
|
+
"bolt_writer",
|
|
37
|
+
"BoltConfig",
|
|
38
|
+
"build_schema_document",
|
|
39
|
+
"SCHEMA_VERSION",
|
|
40
|
+
"NODE_LABELS",
|
|
41
|
+
"REL_TYPES",
|
|
42
|
+
"MARKER_LABELS",
|
|
43
|
+
"GraphRows",
|
|
44
|
+
"NodeRow",
|
|
45
|
+
"EdgeRow",
|
|
46
|
+
]
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
################################################################################
|
|
2
|
+
# Copyright IBM Corporation 2025
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
################################################################################
|
|
16
|
+
|
|
17
|
+
"""The incremental writer: push :class:`GraphRows` into a live Neo4j over Bolt.
|
|
18
|
+
Unlike the snapshot writer, this one reads the DB's current state and updates
|
|
19
|
+
only what changed.
|
|
20
|
+
|
|
21
|
+
Algorithm (the module subgraph is the unit of idempotent replacement):
|
|
22
|
+
1. ensure constraints + indexes.
|
|
23
|
+
2. diff each module's ``content_hash`` against the DB → the set of changed modules.
|
|
24
|
+
3. per changed module, in a transaction: delete the edges it owned (edges out of
|
|
25
|
+
its nodes), detach-delete the declarations it no longer emits, then upsert
|
|
26
|
+
its current nodes.
|
|
27
|
+
4. upsert edges owned by changed modules (+ the shared edges).
|
|
28
|
+
5. on a FULL run only, prune modules whose source file vanished.
|
|
29
|
+
|
|
30
|
+
Nodes are MERGE-upserted, never blindly deleted, so a declaration another
|
|
31
|
+
(unchanged) module still references survives and its incoming edges stay valid.
|
|
32
|
+
``:PyExternal`` / ``:PyPackage`` / ``:PyDecorator`` are shared (no ``_module``) and are
|
|
33
|
+
MERGE-only.
|
|
34
|
+
|
|
35
|
+
The ``neo4j`` driver is imported lazily so it stays an optional dependency and
|
|
36
|
+
off the default (json) output path entirely.
|
|
37
|
+
"""
|
|
38
|
+
from __future__ import annotations
|
|
39
|
+
|
|
40
|
+
from dataclasses import dataclass
|
|
41
|
+
from typing import Dict, List, Optional
|
|
42
|
+
|
|
43
|
+
from codeanalyzer.neo4j.rows import EdgeRow, GraphRows, NodeRow, chunk
|
|
44
|
+
from codeanalyzer.neo4j.schema import CONSTRAINTS, INDEXES
|
|
45
|
+
from codeanalyzer.utils import logger
|
|
46
|
+
|
|
47
|
+
DESCENDANTS = "[:PY_DECLARES|PY_HAS_METHOD|PY_HAS_ATTRIBUTE|PY_DECLARES_VAR|PY_HAS_CALLSITE*1..]"
|
|
48
|
+
BATCH = 1000
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class BoltConfig:
|
|
53
|
+
uri: str
|
|
54
|
+
user: str
|
|
55
|
+
password: str
|
|
56
|
+
database: Optional[str] = None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
60
|
+
try:
|
|
61
|
+
import neo4j # noqa: WPS433 (lazy, optional dependency)
|
|
62
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
63
|
+
raise RuntimeError(
|
|
64
|
+
"The 'neo4j' driver is required for '--emit neo4j --neo4j-uri'. "
|
|
65
|
+
"Install it with: pip install 'codeanalyzer-python[neo4j]'"
|
|
66
|
+
) from exc
|
|
67
|
+
|
|
68
|
+
driver = neo4j.GraphDatabase.driver(cfg.uri, auth=(cfg.user, cfg.password))
|
|
69
|
+
session_kwargs = {"database": cfg.database} if cfg.database else {}
|
|
70
|
+
|
|
71
|
+
def session():
|
|
72
|
+
return driver.session(**session_kwargs)
|
|
73
|
+
|
|
74
|
+
try:
|
|
75
|
+
# 1. schema (DDL runs in its own autocommit transactions).
|
|
76
|
+
with session() as s:
|
|
77
|
+
for stmt in [*CONSTRAINTS, *INDEXES]:
|
|
78
|
+
s.run(stmt)
|
|
79
|
+
|
|
80
|
+
# The application anchor (a shared node) — used to scope the orphan prune
|
|
81
|
+
# so it never touches modules belonging to a different :PyApplication.
|
|
82
|
+
app_name = next(
|
|
83
|
+
(n.value for n in rows.nodes if n.labels and n.labels[0] == "PyApplication"),
|
|
84
|
+
None,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Partition nodes by owning module; shared nodes have no _module.
|
|
88
|
+
by_module: Dict[str, List[NodeRow]] = {}
|
|
89
|
+
shared: List[NodeRow] = []
|
|
90
|
+
module_of: Dict[str, str] = {} # node value → owning module
|
|
91
|
+
for n in rows.nodes:
|
|
92
|
+
m = n.props.get("_module")
|
|
93
|
+
if isinstance(m, str):
|
|
94
|
+
by_module.setdefault(m, []).append(n)
|
|
95
|
+
module_of[n.value] = m
|
|
96
|
+
else:
|
|
97
|
+
shared.append(n)
|
|
98
|
+
|
|
99
|
+
# 2. diff content_hash.
|
|
100
|
+
db_hash: Dict[str, Optional[str]] = {}
|
|
101
|
+
with session() as s:
|
|
102
|
+
res = s.run("MATCH (m:PyModule) RETURN m.file_key AS k, m.content_hash AS h")
|
|
103
|
+
for rec in res:
|
|
104
|
+
db_hash[rec["k"]] = rec["h"]
|
|
105
|
+
changed = set()
|
|
106
|
+
for m, nodes in by_module.items():
|
|
107
|
+
row_hash = _hash_of(nodes, m)
|
|
108
|
+
if m not in db_hash or row_hash is None or row_hash != db_hash.get(m):
|
|
109
|
+
changed.add(m)
|
|
110
|
+
logger.info(
|
|
111
|
+
f"neo4j(bolt): {len(by_module)} modules ({len(changed)} changed), "
|
|
112
|
+
f"{len(shared)} shared nodes, {len(rows.edges)} edges"
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
# 3. shared nodes are always upserted (MERGE-only).
|
|
116
|
+
_upsert_nodes(session, neo4j, shared)
|
|
117
|
+
|
|
118
|
+
# 4. per changed module: purge owned edges + vanished decls, then upsert its nodes.
|
|
119
|
+
for m in changed:
|
|
120
|
+
nodes = by_module[m]
|
|
121
|
+
keys = [n.value for n in nodes]
|
|
122
|
+
with session() as s:
|
|
123
|
+
def _purge(tx, module=m, node_keys=keys):
|
|
124
|
+
tx.run("MATCH (x {_module: $m})-[r]->() DELETE r", m=module)
|
|
125
|
+
tx.run(
|
|
126
|
+
"MATCH (x {_module: $m}) "
|
|
127
|
+
"WHERE NOT coalesce(x.signature, x.id, x.file_key) IN $keys "
|
|
128
|
+
"DETACH DELETE x",
|
|
129
|
+
m=module,
|
|
130
|
+
keys=node_keys,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
s.execute_write(_purge)
|
|
134
|
+
_upsert_nodes(session, neo4j, nodes)
|
|
135
|
+
|
|
136
|
+
# 5. upsert edges owned by a changed module (owner = source node's module) or shared.
|
|
137
|
+
edges = [
|
|
138
|
+
e
|
|
139
|
+
for e in rows.edges
|
|
140
|
+
if module_of.get(e.from_ref.value) is None or module_of.get(e.from_ref.value) in changed
|
|
141
|
+
]
|
|
142
|
+
_upsert_edges(session, neo4j, edges)
|
|
143
|
+
|
|
144
|
+
# 6. orphan prune — only safe on a full run (a targeted run can't tell deleted from untargeted).
|
|
145
|
+
# Scope to THIS application's anchor so a full run for application B never
|
|
146
|
+
# deletes application A's modules from a shared database.
|
|
147
|
+
if full_run and app_name is not None:
|
|
148
|
+
present = list(by_module.keys())
|
|
149
|
+
with session() as s:
|
|
150
|
+
res = s.run(
|
|
151
|
+
"MATCH (:PyApplication {name: $app})-[:PY_HAS_MODULE]->(m:PyModule) "
|
|
152
|
+
"WHERE NOT m.file_key IN $present "
|
|
153
|
+
f"OPTIONAL MATCH (m)-{DESCENDANTS}->(x) DETACH DELETE x, m "
|
|
154
|
+
"RETURN count(m) AS pruned",
|
|
155
|
+
app=app_name,
|
|
156
|
+
present=present,
|
|
157
|
+
)
|
|
158
|
+
pruned = res.single()
|
|
159
|
+
pruned_count = pruned["pruned"] if pruned else 0
|
|
160
|
+
logger.info(f"neo4j(bolt): pruned {pruned_count} vanished module(s)")
|
|
161
|
+
else:
|
|
162
|
+
logger.info(
|
|
163
|
+
"neo4j(bolt): targeted run — orphan pruning skipped (deleted files not removed)"
|
|
164
|
+
)
|
|
165
|
+
finally:
|
|
166
|
+
driver.close()
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# ----------------------------------------------------------------------------------------------
|
|
170
|
+
# Batched upserts
|
|
171
|
+
# ----------------------------------------------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _upsert_nodes(session, neo4j, nodes: List[NodeRow]) -> None:
|
|
175
|
+
groups: Dict[str, List[NodeRow]] = {}
|
|
176
|
+
for n in nodes:
|
|
177
|
+
groups.setdefault(f"{':'.join(n.labels)}|{n.key_prop}", []).append(n)
|
|
178
|
+
|
|
179
|
+
for group in groups.values():
|
|
180
|
+
labels = group[0].labels
|
|
181
|
+
key_prop = group[0].key_prop
|
|
182
|
+
set_labels = f", n:{':'.join(labels[1:])}" if len(labels) > 1 else ""
|
|
183
|
+
cypher = (
|
|
184
|
+
f"UNWIND $rows AS row MERGE (n:{labels[0]} {{{key_prop}: row.k}}) "
|
|
185
|
+
f"SET n += row.p{set_labels}"
|
|
186
|
+
)
|
|
187
|
+
for batch in chunk(group, BATCH):
|
|
188
|
+
payload = [{"k": n.value, "p": _to_params(n.props, neo4j)} for n in batch]
|
|
189
|
+
with session() as s:
|
|
190
|
+
s.run(cypher, rows=payload)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _upsert_edges(session, neo4j, edges: List[EdgeRow]) -> None:
|
|
194
|
+
groups: Dict[str, List[EdgeRow]] = {}
|
|
195
|
+
for e in edges:
|
|
196
|
+
key = f"{e.type}|{e.from_ref.label}.{e.from_ref.key_prop}|{e.to_ref.label}.{e.to_ref.key_prop}"
|
|
197
|
+
groups.setdefault(key, []).append(e)
|
|
198
|
+
|
|
199
|
+
for group in groups.values():
|
|
200
|
+
first = group[0]
|
|
201
|
+
from_ref, to_ref = first.from_ref, first.to_ref
|
|
202
|
+
cypher = (
|
|
203
|
+
f"UNWIND $rows AS row "
|
|
204
|
+
f"MATCH (a:{from_ref.label} {{{from_ref.key_prop}: row.f}}) "
|
|
205
|
+
f"MATCH (b:{to_ref.label} {{{to_ref.key_prop}: row.t}}) "
|
|
206
|
+
f"MERGE (a)-[r:{first.type}]->(b) SET r += row.p"
|
|
207
|
+
)
|
|
208
|
+
for batch in chunk(group, BATCH):
|
|
209
|
+
payload = [
|
|
210
|
+
{"f": e.from_ref.value, "t": e.to_ref.value, "p": _to_params(e.props, neo4j)}
|
|
211
|
+
for e in batch
|
|
212
|
+
]
|
|
213
|
+
with session() as s:
|
|
214
|
+
s.run(cypher, rows=payload)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# ----------------------------------------------------------------------------------------------
|
|
218
|
+
# Helpers
|
|
219
|
+
# ----------------------------------------------------------------------------------------------
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _hash_of(nodes: List[NodeRow], file_key: str) -> Optional[str]:
|
|
223
|
+
for n in nodes:
|
|
224
|
+
if n.labels[0] == "PyModule" and n.value == file_key:
|
|
225
|
+
h = n.props.get("content_hash")
|
|
226
|
+
return h if isinstance(h, str) else None
|
|
227
|
+
return None
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _to_params(props, neo4j) -> dict:
|
|
231
|
+
"""Map props to driver params. The Python driver already distinguishes int
|
|
232
|
+
from float, so unlike the JS driver no integer coercion is needed — this is a
|
|
233
|
+
straight passthrough kept symmetric with the snapshot writer's shape."""
|
|
234
|
+
return dict(props)
|