codeanalyzer-python 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +99 -6
- codeanalyzer/core.py +192 -215
- codeanalyzer/neo4j/bolt.py +13 -2
- codeanalyzer/neo4j/catalog.py +2 -2
- codeanalyzer/neo4j/project.py +18 -10
- codeanalyzer/neo4j/rows.py +10 -5
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +21 -1
- codeanalyzer/schema/__init__.py +2 -0
- codeanalyzer/schema/py_schema.py +16 -1
- codeanalyzer/semantic_analysis/call_graph.py +24 -3
- codeanalyzer/semantic_analysis/pycg/__init__.py +20 -0
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +1054 -0
- codeanalyzer/semantic_analysis/{codeql/__init__.py → pycg/pycg_exceptions.py} +5 -8
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +401 -0
- codeanalyzer/utils/logging.py +5 -2
- codeanalyzer/utils/progress_bar.py +15 -7
- codeanalyzer_python-0.3.0.dist-info/METADATA +550 -0
- codeanalyzer_python-0.3.0.dist-info/RECORD +38 -0
- codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +0 -382
- codeanalyzer/semantic_analysis/codeql/codeql_exceptions.py +0 -12
- codeanalyzer/semantic_analysis/codeql/codeql_loader.py +0 -91
- codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py +0 -185
- codeanalyzer_python-0.2.0.dist-info/METADATA +0 -393
- codeanalyzer_python-0.2.0.dist-info/RECORD +0 -39
- {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.2.0.dist-info → codeanalyzer_python-0.3.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/core.py
CHANGED
|
@@ -6,18 +6,25 @@ import sys
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Any, Dict, Optional, Union, List
|
|
8
8
|
|
|
9
|
+
import time
|
|
10
|
+
|
|
9
11
|
import ray
|
|
10
12
|
from codeanalyzer.utils import logger
|
|
11
|
-
from codeanalyzer.schema import
|
|
13
|
+
from codeanalyzer.schema import (
|
|
14
|
+
PyApplication,
|
|
15
|
+
PyExternalSymbol,
|
|
16
|
+
PyModule,
|
|
17
|
+
model_dump_json,
|
|
18
|
+
model_validate_json,
|
|
19
|
+
)
|
|
12
20
|
from codeanalyzer.schema.py_schema import PyCallEdge
|
|
13
21
|
from codeanalyzer.semantic_analysis.call_graph import (
|
|
22
|
+
filter_external_edges,
|
|
14
23
|
jedi_call_graph_edges,
|
|
15
24
|
merge_edges,
|
|
16
25
|
resolve_unresolved_constructors,
|
|
17
26
|
)
|
|
18
|
-
from codeanalyzer.semantic_analysis.
|
|
19
|
-
from codeanalyzer.semantic_analysis.codeql.codeql_analysis import CodeQL
|
|
20
|
-
from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import CodeQLExceptions
|
|
27
|
+
from codeanalyzer.semantic_analysis.pycg import PyCG, PyCGExceptions
|
|
21
28
|
from codeanalyzer.syntactic_analysis.exceptions import SymbolTableBuilderRayError
|
|
22
29
|
from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
|
|
23
30
|
from codeanalyzer.utils import ProgressBar
|
|
@@ -48,7 +55,7 @@ def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, s
|
|
|
48
55
|
|
|
49
56
|
|
|
50
57
|
class Codeanalyzer:
|
|
51
|
-
"""Core
|
|
58
|
+
"""Core static analysis engine for Python projects.
|
|
52
59
|
|
|
53
60
|
Args:
|
|
54
61
|
options (AnalysisOptions): Analysis configuration options containing all necessary parameters.
|
|
@@ -58,15 +65,13 @@ class Codeanalyzer:
|
|
|
58
65
|
self.options = options
|
|
59
66
|
self.project_dir = Path(options.input).resolve()
|
|
60
67
|
self.skip_tests = options.skip_tests
|
|
61
|
-
self.
|
|
68
|
+
self.analysis_level = options.analysis_level
|
|
62
69
|
self.rebuild_analysis = options.rebuild_analysis
|
|
70
|
+
self.no_venv = options.no_venv
|
|
63
71
|
self.cache_dir = (
|
|
64
72
|
options.cache_dir.resolve() if options.cache_dir is not None else self.project_dir
|
|
65
73
|
) / ".codeanalyzer"
|
|
66
74
|
self.clear_cache = options.clear_cache
|
|
67
|
-
self.db_path: Optional[Path] = None
|
|
68
|
-
self.codeql_bin: Optional[Path] = None
|
|
69
|
-
self.codeql_packs_dir: Optional[Path] = None
|
|
70
75
|
self.virtualenv: Optional[Path] = None
|
|
71
76
|
self.using_ray: bool = options.using_ray
|
|
72
77
|
self.file_name: Optional[Path] = options.file_name
|
|
@@ -78,6 +83,7 @@ class Codeanalyzer:
|
|
|
78
83
|
capture_output: bool = True,
|
|
79
84
|
check: bool = True,
|
|
80
85
|
suppress_output: bool = False,
|
|
86
|
+
log_on_failure: bool = True,
|
|
81
87
|
) -> subprocess.CompletedProcess:
|
|
82
88
|
"""
|
|
83
89
|
Runs a subprocess with real-time output streaming to the logger.
|
|
@@ -87,7 +93,10 @@ class Codeanalyzer:
|
|
|
87
93
|
cwd: Working directory to run the command in.
|
|
88
94
|
capture_output: If True, retains and returns the output.
|
|
89
95
|
check: If True, raises CalledProcessError on non-zero exit.
|
|
90
|
-
suppress_output: If True, silences
|
|
96
|
+
suppress_output: If True, silences per-line debug output.
|
|
97
|
+
log_on_failure: If False, suppresses the error-level log on
|
|
98
|
+
non-zero exit (use when the caller handles the exception and
|
|
99
|
+
will emit its own diagnostic).
|
|
91
100
|
|
|
92
101
|
Returns:
|
|
93
102
|
subprocess.CompletedProcess
|
|
@@ -118,9 +127,10 @@ class Codeanalyzer:
|
|
|
118
127
|
|
|
119
128
|
if check and returncode != 0:
|
|
120
129
|
error_output = "\n".join(output_lines)
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
130
|
+
if log_on_failure:
|
|
131
|
+
logger.error(f"Command failed with exit code {returncode}: {' '.join(cmd)}")
|
|
132
|
+
if error_output:
|
|
133
|
+
logger.error(f"Command output:\n{error_output}")
|
|
124
134
|
raise subprocess.CalledProcessError(returncode, cmd, output=error_output)
|
|
125
135
|
|
|
126
136
|
return subprocess.CompletedProcess(
|
|
@@ -226,13 +236,51 @@ class Codeanalyzer:
|
|
|
226
236
|
f"a working Python interpreter that can create virtual environments."
|
|
227
237
|
)
|
|
228
238
|
|
|
239
|
+
@staticmethod
|
|
240
|
+
def _uv_bin() -> Optional[str]:
|
|
241
|
+
"""Path to the uv binary bundled with the ``uv`` PyPI package (a declared
|
|
242
|
+
dependency, so always present in our install -- including inside a Docker
|
|
243
|
+
image). We deliberately ignore any uv on PATH so the analyzer always uses
|
|
244
|
+
the pinned, vendored uv. Returns ``None`` only if the package is somehow
|
|
245
|
+
missing (callers fall back to pip)."""
|
|
246
|
+
try:
|
|
247
|
+
from uv import find_uv_bin
|
|
248
|
+
return str(find_uv_bin())
|
|
249
|
+
except Exception:
|
|
250
|
+
return None
|
|
251
|
+
|
|
252
|
+
def _install_into_venv(self, venv_python: Path, args: List[str]) -> None:
|
|
253
|
+
"""Install packages into the target venv, preferring uv for speed (parallel
|
|
254
|
+
downloads + a shared global cache) and falling back to the venv's own pip
|
|
255
|
+
when uv is unavailable.
|
|
256
|
+
|
|
257
|
+
Raises ``subprocess.CalledProcessError`` on failure; callers in
|
|
258
|
+
``__enter__`` catch this and warn-and-continue so a single failing
|
|
259
|
+
package (e.g. a C extension that needs system libs) does not abort the
|
|
260
|
+
entire analysis.
|
|
261
|
+
"""
|
|
262
|
+
uv = self._uv_bin()
|
|
263
|
+
if uv:
|
|
264
|
+
cmd = [uv, "pip", "install", "--python", str(venv_python), *args]
|
|
265
|
+
else:
|
|
266
|
+
cmd = [str(venv_python), "-m", "pip", "install", *args]
|
|
267
|
+
self._cmd_exec_helper(
|
|
268
|
+
cmd, cwd=self.project_dir, check=True,
|
|
269
|
+
suppress_output=True, log_on_failure=False,
|
|
270
|
+
)
|
|
271
|
+
|
|
229
272
|
def __enter__(self) -> "Codeanalyzer":
|
|
230
273
|
# If no virtualenv is provided, try to create one using requirements.txt or pyproject.toml
|
|
231
274
|
venv_path = self.cache_dir / self.project_dir.name / "virtualenv"
|
|
232
275
|
# Ensure the cache directory exists for this project
|
|
233
276
|
venv_path.parent.mkdir(parents=True, exist_ok=True)
|
|
277
|
+
if self.no_venv:
|
|
278
|
+
logger.info(
|
|
279
|
+
"--no-venv: using the ambient Python environment "
|
|
280
|
+
"(skipping virtualenv creation and dependency installation)"
|
|
281
|
+
)
|
|
234
282
|
# Create the virtual environment if it does not exist
|
|
235
|
-
if not venv_path.exists() or self.rebuild_analysis:
|
|
283
|
+
if not self.no_venv and (not venv_path.exists() or self.rebuild_analysis):
|
|
236
284
|
logger.info(f"(Re-)creating virtual environment at {venv_path}")
|
|
237
285
|
self._cmd_exec_helper(
|
|
238
286
|
[str(self._get_base_interpreter()), "-m", "venv", str(venv_path)],
|
|
@@ -249,29 +297,35 @@ class Codeanalyzer:
|
|
|
249
297
|
("test-requirements.txt", ["-r"]),
|
|
250
298
|
]
|
|
251
299
|
|
|
252
|
-
for dep_file,
|
|
300
|
+
for dep_file, _ in dependency_files:
|
|
253
301
|
if (self.project_dir / dep_file).exists():
|
|
254
302
|
logger.info(f"Installing dependencies from {dep_file}")
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
303
|
+
try:
|
|
304
|
+
self._install_into_venv(
|
|
305
|
+
venv_python,
|
|
306
|
+
["--upgrade", "-r", str(self.project_dir / dep_file)],
|
|
307
|
+
)
|
|
308
|
+
except subprocess.CalledProcessError as exc:
|
|
309
|
+
logger.warning(
|
|
310
|
+
f"Dependency installation from {dep_file} failed "
|
|
311
|
+
f"(exit {exc.returncode}) — continuing without it. "
|
|
312
|
+
"Jedi type resolution may be incomplete."
|
|
313
|
+
)
|
|
260
314
|
|
|
261
315
|
# Handle Pipenv files
|
|
262
316
|
if (self.project_dir / "Pipfile").exists():
|
|
263
317
|
logger.info("Installing dependencies from Pipfile")
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
318
|
+
try:
|
|
319
|
+
self._install_into_venv(venv_python, ["pipenv"])
|
|
320
|
+
self._cmd_exec_helper(
|
|
321
|
+
["pipenv", "install", "--dev"],
|
|
322
|
+
cwd=self.project_dir,
|
|
323
|
+
check=True,
|
|
324
|
+
)
|
|
325
|
+
except subprocess.CalledProcessError as exc:
|
|
326
|
+
logger.warning(
|
|
327
|
+
f"Pipenv installation failed (exit {exc.returncode}) — continuing without it."
|
|
328
|
+
)
|
|
275
329
|
|
|
276
330
|
# Handle conda environment files
|
|
277
331
|
conda_files = ["conda.yml", "environment.yml"]
|
|
@@ -289,67 +343,23 @@ class Codeanalyzer:
|
|
|
289
343
|
|
|
290
344
|
if any((self.project_dir / file).exists() for file in package_definition_files):
|
|
291
345
|
logger.info("Installing project in editable mode")
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
346
|
+
try:
|
|
347
|
+
self._install_into_venv(venv_python, ["-e", str(self.project_dir)])
|
|
348
|
+
except subprocess.CalledProcessError as exc:
|
|
349
|
+
logger.warning(
|
|
350
|
+
f"Editable install failed (exit {exc.returncode}) — "
|
|
351
|
+
"continuing without it. Jedi type resolution may be incomplete."
|
|
352
|
+
)
|
|
297
353
|
else:
|
|
298
354
|
logger.warning("No package definition files found, skipping editable installation")
|
|
299
355
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
# CLI install ships only the language extractors; the
|
|
308
|
-
# ``codeql/python-all`` library pack must be fetched separately.
|
|
309
|
-
self.codeql_packs_dir = self._ensure_codeql_packs(self.codeql_bin)
|
|
310
|
-
|
|
311
|
-
cache_root = self.cache_dir / "codeql"
|
|
312
|
-
cache_root.mkdir(parents=True, exist_ok=True)
|
|
313
|
-
self.db_path = cache_root / f"{self.project_dir.name}-db"
|
|
314
|
-
self.db_path.mkdir(exist_ok=True)
|
|
315
|
-
|
|
316
|
-
checksum_file = self.db_path / ".checksum"
|
|
317
|
-
current_checksum = self._compute_checksum(self.project_dir)
|
|
318
|
-
|
|
319
|
-
def is_cache_valid() -> bool:
|
|
320
|
-
if not (self.db_path / "db-python").exists():
|
|
321
|
-
return False
|
|
322
|
-
if not checksum_file.exists():
|
|
323
|
-
return False
|
|
324
|
-
return checksum_file.read_text().strip() == current_checksum
|
|
325
|
-
|
|
326
|
-
if self.rebuild_analysis or not is_cache_valid():
|
|
327
|
-
logger.info("Creating new CodeQL database...")
|
|
328
|
-
|
|
329
|
-
cmd = [
|
|
330
|
-
str(self.codeql_bin),
|
|
331
|
-
"database",
|
|
332
|
-
"create",
|
|
333
|
-
str(self.db_path),
|
|
334
|
-
f"--source-root={self.project_dir}",
|
|
335
|
-
"--language=python",
|
|
336
|
-
"--overwrite",
|
|
337
|
-
]
|
|
338
|
-
|
|
339
|
-
proc = subprocess.Popen(
|
|
340
|
-
cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
|
|
341
|
-
)
|
|
342
|
-
_, err = proc.communicate()
|
|
343
|
-
|
|
344
|
-
if proc.returncode != 0:
|
|
345
|
-
raise CodeQLExceptions.CodeQLDatabaseBuildException(
|
|
346
|
-
f"Error building CodeQL database:\n{err.decode()}"
|
|
347
|
-
)
|
|
348
|
-
|
|
349
|
-
checksum_file.write_text(current_checksum)
|
|
350
|
-
|
|
351
|
-
else:
|
|
352
|
-
logger.info(f"Reusing cached CodeQL DB at {self.db_path}")
|
|
356
|
+
# Point Jedi at the analysis venv so it resolves the project's third-party
|
|
357
|
+
# imports. This runs on both a fresh build and a lazy reuse of an existing
|
|
358
|
+
# venv -- previously self.virtualenv stayed None, so the install above was
|
|
359
|
+
# never actually used by the symbol-table builder. With --no-venv we leave
|
|
360
|
+
# it None so Jedi resolves against the ambient interpreter instead.
|
|
361
|
+
if not self.no_venv and venv_path.exists():
|
|
362
|
+
self.virtualenv = venv_path
|
|
353
363
|
|
|
354
364
|
return self
|
|
355
365
|
|
|
@@ -358,6 +368,43 @@ class Codeanalyzer:
|
|
|
358
368
|
logger.info(f"Clearing cache directory: {self.cache_dir}")
|
|
359
369
|
shutil.rmtree(self.cache_dir)
|
|
360
370
|
|
|
371
|
+
@staticmethod
|
|
372
|
+
def _compute_external_symbols(symbol_table, call_graph):
|
|
373
|
+
"""Build the external-symbol map: every call-graph endpoint whose signature
|
|
374
|
+
is not a declared class/callable in the symbol table is an external (an
|
|
375
|
+
imported library or builtin member). ``name``/``module`` are derived from
|
|
376
|
+
the signature (best effort: split on the last dot)."""
|
|
377
|
+
declared = set()
|
|
378
|
+
|
|
379
|
+
def walk_callable(c):
|
|
380
|
+
declared.add(c.signature)
|
|
381
|
+
for ic in (c.inner_callables or {}).values():
|
|
382
|
+
walk_callable(ic)
|
|
383
|
+
for cl in (c.inner_classes or {}).values():
|
|
384
|
+
walk_class(cl)
|
|
385
|
+
|
|
386
|
+
def walk_class(cl):
|
|
387
|
+
declared.add(cl.signature)
|
|
388
|
+
for m in (cl.methods or {}).values():
|
|
389
|
+
walk_callable(m)
|
|
390
|
+
for ic in (cl.inner_classes or {}).values():
|
|
391
|
+
walk_class(ic)
|
|
392
|
+
|
|
393
|
+
for mod in symbol_table.values():
|
|
394
|
+
for c in (mod.functions or {}).values():
|
|
395
|
+
walk_callable(c)
|
|
396
|
+
for cl in (mod.classes or {}).values():
|
|
397
|
+
walk_class(cl)
|
|
398
|
+
|
|
399
|
+
externals: Dict[str, PyExternalSymbol] = {}
|
|
400
|
+
for edge in call_graph:
|
|
401
|
+
for sig in (edge.source, edge.target):
|
|
402
|
+
if sig in declared or sig in externals:
|
|
403
|
+
continue
|
|
404
|
+
module, name = sig.rsplit(".", 1) if "." in sig else (sig, sig)
|
|
405
|
+
externals[sig] = PyExternalSymbol(name=name, module=module)
|
|
406
|
+
return externals
|
|
407
|
+
|
|
361
408
|
def analyze(self) -> PyApplication:
|
|
362
409
|
"""Analyze the project and return a PyApplication with symbol table.
|
|
363
410
|
|
|
@@ -378,27 +425,35 @@ class Codeanalyzer:
|
|
|
378
425
|
# Build symbol table from cached application if available (if no available, the build a new one)
|
|
379
426
|
symbol_table = self._build_symbol_table(cached_pyapplication.symbol_table if cached_pyapplication else {})
|
|
380
427
|
|
|
381
|
-
# Build the call graph in four steps:
|
|
382
|
-
# 1. Run CodeQL (when enabled). Produces resolved edges with
|
|
383
|
-
# ``provenance=["codeql"]`` and augments ``PyCallsite``s
|
|
384
|
-
# in-place — filling ``callee_signature`` for sites Jedi
|
|
385
|
-
# couldn't resolve.
|
|
386
|
-
# 2. Heuristic fallback for constructor calls neither Jedi nor
|
|
387
|
-
# CodeQL could resolve (commonly classes nested inside
|
|
388
|
-
# functions). Walks the symbol table by class short-name +
|
|
389
|
-
# scope and writes ``<class>.__init__`` into the site.
|
|
390
|
-
# 3. Derive Jedi edges from the now-fully-augmented symbol
|
|
391
|
-
# table — these reflect every resolution the symbol table
|
|
392
|
-
# contains, regardless of which pass put it there.
|
|
393
|
-
# 4. Merge with CodeQL edges; provenance unions for edges both
|
|
394
|
-
# backends saw.
|
|
395
|
-
codeql_edges = self._get_call_graph(symbol_table, augment_sites=True)
|
|
396
428
|
resolve_unresolved_constructors(symbol_table)
|
|
429
|
+
|
|
430
|
+
# Level 1: Jedi call graph.
|
|
431
|
+
t0_jedi = time.perf_counter()
|
|
397
432
|
jedi_edges = jedi_call_graph_edges(symbol_table)
|
|
398
|
-
call_graph =
|
|
433
|
+
call_graph = list(jedi_edges)
|
|
434
|
+
logger.info("✅ Jedi: %d edges in %.1fs", len(call_graph), time.perf_counter() - t0_jedi)
|
|
435
|
+
|
|
436
|
+
if self.analysis_level >= 2:
|
|
437
|
+
# Level 2: also add PyCG edges. The Jedi edges double as the
|
|
438
|
+
# coupling graph that drives coupling-aware PyCG sharding.
|
|
439
|
+
pycg_edges = self._get_pycg_call_graph(symbol_table, jedi_edges)
|
|
440
|
+
call_graph = merge_edges(call_graph, pycg_edges)
|
|
441
|
+
|
|
442
|
+
call_graph = filter_external_edges(call_graph, symbol_table)
|
|
443
|
+
|
|
444
|
+
# Classify call-graph endpoints that are not declared in the symbol table
|
|
445
|
+
# (imported library / builtin members) once, so the JSON and Neo4j backends
|
|
446
|
+
# share one authoritative external-symbol set.
|
|
447
|
+
external_symbols = self._compute_external_symbols(symbol_table, call_graph)
|
|
399
448
|
|
|
400
449
|
# Recreate pyapplication
|
|
401
|
-
app =
|
|
450
|
+
app = (
|
|
451
|
+
PyApplication.builder()
|
|
452
|
+
.symbol_table(symbol_table)
|
|
453
|
+
.call_graph(call_graph)
|
|
454
|
+
.external_symbols(external_symbols)
|
|
455
|
+
.build()
|
|
456
|
+
)
|
|
402
457
|
|
|
403
458
|
# Save to cache
|
|
404
459
|
self._save_analysis_cache(app, cache_file)
|
|
@@ -491,7 +546,8 @@ class Codeanalyzer:
|
|
|
491
546
|
Dict[str, PyModule]: A dictionary mapping file paths to PyModule objects.
|
|
492
547
|
"""
|
|
493
548
|
symbol_table: Dict[str, PyModule] = {}
|
|
494
|
-
|
|
549
|
+
t0_st = time.perf_counter()
|
|
550
|
+
|
|
495
551
|
# Handle single file analysis
|
|
496
552
|
if self.file_name is not None:
|
|
497
553
|
single_file = self.project_dir / self.file_name
|
|
@@ -598,123 +654,44 @@ class Codeanalyzer:
|
|
|
598
654
|
if files_from_cache > 0:
|
|
599
655
|
logger.info(f"Reused {files_from_cache} files from cache, processed {files_processed} new/changed files")
|
|
600
656
|
|
|
601
|
-
logger.info(
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
def _ensure_codeql_packs(self, codeql_bin: Path) -> Path:
|
|
605
|
-
"""Materialize a qlpack that depends on ``codeql/python-all``.
|
|
606
|
-
|
|
607
|
-
The CodeQL CLI install ships only the language extractors — query
|
|
608
|
-
library packs (and their transitive dependencies like
|
|
609
|
-
``codeql/concepts``) must be resolved separately. The canonical
|
|
610
|
-
way is to declare the dependency in a ``qlpack.yml`` and run
|
|
611
|
-
``codeql pack install`` in that directory; CodeQL writes a
|
|
612
|
-
``codeql-pack.lock.yml`` and downloads everything needed.
|
|
613
|
-
|
|
614
|
-
We do this once per project under ``<cache_dir>/codeql/qlpack/``
|
|
615
|
-
and return that directory. The query runner then writes its
|
|
616
|
-
temporary ``.ql`` file inside this pack — colocation makes
|
|
617
|
-
``import python`` resolve without any ``--additional-packs`` or
|
|
618
|
-
``--search-path`` gymnastics.
|
|
619
|
-
"""
|
|
620
|
-
pack_dir = self.cache_dir / "codeql" / "qlpack"
|
|
621
|
-
pack_dir.mkdir(parents=True, exist_ok=True)
|
|
622
|
-
qlpack_yml = pack_dir / "qlpack.yml"
|
|
623
|
-
lock_file = pack_dir / "codeql-pack.lock.yml"
|
|
624
|
-
|
|
625
|
-
if not qlpack_yml.exists():
|
|
626
|
-
qlpack_yml.write_text(
|
|
627
|
-
"name: codeanalyzer-deps\n"
|
|
628
|
-
"version: 1.0.0\n"
|
|
629
|
-
"dependencies:\n"
|
|
630
|
-
' codeql/python-all: "*"\n'
|
|
631
|
-
)
|
|
632
|
-
|
|
633
|
-
if lock_file.exists():
|
|
634
|
-
logger.debug(f"CodeQL pack dependencies already installed in {pack_dir}")
|
|
635
|
-
return pack_dir
|
|
636
|
-
|
|
637
|
-
logger.info(f"Installing CodeQL pack dependencies in {pack_dir}.")
|
|
638
|
-
proc = subprocess.Popen(
|
|
639
|
-
[str(codeql_bin), "pack", "install", str(pack_dir)],
|
|
640
|
-
stdout=subprocess.PIPE,
|
|
641
|
-
stderr=subprocess.PIPE,
|
|
642
|
-
)
|
|
643
|
-
_, err = proc.communicate()
|
|
644
|
-
if proc.returncode != 0:
|
|
645
|
-
raise CodeQLExceptions.CodeQLDatabaseBuildException(
|
|
646
|
-
f"Failed to install CodeQL pack dependencies:\n"
|
|
647
|
-
f"{(err or b'').decode(errors='replace')}"
|
|
648
|
-
)
|
|
649
|
-
return pack_dir
|
|
650
|
-
|
|
651
|
-
def _ensure_codeql_bin(self) -> Path:
|
|
652
|
-
"""Locate (or download) the CodeQL CLI binary into the project cache.
|
|
653
|
-
|
|
654
|
-
Resolution order:
|
|
655
|
-
1. An existing binary inside ``<cache_dir>/codeql/bin/`` —
|
|
656
|
-
reused across runs on the same project.
|
|
657
|
-
2. ``codeql`` already on the user's PATH — picked up verbatim.
|
|
658
|
-
3. Otherwise, download into ``<cache_dir>/codeql/bin/``.
|
|
659
|
-
|
|
660
|
-
The project-local cache is preferred over PATH so the version we
|
|
661
|
-
installed earlier wins over whatever the OS ships — keeps behavior
|
|
662
|
-
deterministic when the user has both.
|
|
663
|
-
"""
|
|
664
|
-
bin_root = self.cache_dir / "codeql" / "bin"
|
|
665
|
-
bin_root.mkdir(parents=True, exist_ok=True)
|
|
666
|
-
|
|
667
|
-
existing = next(
|
|
668
|
-
(p for p in bin_root.rglob("codeql") if p.is_file()),
|
|
669
|
-
None,
|
|
657
|
+
logger.info(
|
|
658
|
+
"✅ Symbol table: %d modules in %.1fs",
|
|
659
|
+
len(symbol_table), time.perf_counter() - t0_st,
|
|
670
660
|
)
|
|
671
|
-
|
|
672
|
-
logger.debug(f"Reusing cached CodeQL CLI at {existing}")
|
|
673
|
-
return existing.resolve()
|
|
674
|
-
|
|
675
|
-
on_path = shutil.which("codeql")
|
|
676
|
-
if on_path:
|
|
677
|
-
logger.debug(f"Using CodeQL CLI from PATH at {on_path}")
|
|
678
|
-
return Path(on_path)
|
|
679
|
-
|
|
680
|
-
logger.info(f"CodeQL CLI not found; downloading into {bin_root}.")
|
|
681
|
-
downloaded = CodeQLLoader.download_and_extract_codeql(bin_root)
|
|
682
|
-
if not downloaded.exists() or not os.access(downloaded, os.X_OK):
|
|
683
|
-
raise FileNotFoundError(
|
|
684
|
-
f"CodeQL binary not executable after download: {downloaded}"
|
|
685
|
-
)
|
|
686
|
-
return downloaded
|
|
661
|
+
return symbol_table
|
|
687
662
|
|
|
688
|
-
def
|
|
663
|
+
def _get_pycg_call_graph(
|
|
689
664
|
self,
|
|
690
665
|
symbol_table: Dict[str, PyModule],
|
|
691
|
-
|
|
666
|
+
jedi_edges: List[PyCallEdge],
|
|
692
667
|
) -> List[PyCallEdge]:
|
|
693
|
-
"""Build
|
|
668
|
+
"""Build PyCG-resolved call edges.
|
|
694
669
|
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
670
|
+
Runs PyCG's iterative name-pointer analysis over the whole project
|
|
671
|
+
and returns edges with ``provenance=["pycg"]``. Falls back to an
|
|
672
|
+
empty list and logs a warning on any failure so the caller can
|
|
673
|
+
continue with Jedi-only edges.
|
|
698
674
|
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
CodeQL query is shared (cached on the ``CodeQL`` instance) so
|
|
703
|
-
this costs no extra DB work.
|
|
675
|
+
*jedi_edges* are the level-1 call edges; under the ``jedi`` shard
|
|
676
|
+
strategy they drive coupling-aware partitioning (see
|
|
677
|
+
:func:`shard_planner.plan_shards`).
|
|
704
678
|
"""
|
|
705
|
-
if not self.using_codeql or self.db_path is None:
|
|
706
|
-
return []
|
|
707
679
|
try:
|
|
708
|
-
|
|
680
|
+
pycg = PyCG(
|
|
709
681
|
self.project_dir,
|
|
710
|
-
self.
|
|
711
|
-
|
|
712
|
-
|
|
682
|
+
skip_tests=self.skip_tests,
|
|
683
|
+
shard=self.options.pycg_shard,
|
|
684
|
+
shard_ceiling=self.options.pycg_shard_ceiling,
|
|
685
|
+
shard_timeout=self.options.pycg_shard_timeout,
|
|
686
|
+
shard_strategy=self.options.pycg_shard_strategy,
|
|
687
|
+
max_iter=self.options.pycg_max_iter,
|
|
688
|
+
using_ray=self.using_ray,
|
|
713
689
|
)
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
return
|
|
718
|
-
except
|
|
719
|
-
logger.warning(f"
|
|
690
|
+
return pycg.build_call_graph_edges(symbol_table, jedi_edges=jedi_edges)
|
|
691
|
+
except PyCGExceptions.PyCGImportError as exc:
|
|
692
|
+
logger.warning(f"PyCG not installed — level 2 edges will be Jedi-only: {exc}")
|
|
693
|
+
return []
|
|
694
|
+
except PyCGExceptions.PyCGAnalysisError as exc:
|
|
695
|
+
logger.warning(f"PyCG analysis failed — level 2 edges will be Jedi-only: {exc}")
|
|
696
|
+
logger.debug("PyCG full traceback:", exc_info=True)
|
|
720
697
|
return []
|
codeanalyzer/neo4j/bolt.py
CHANGED
|
@@ -77,6 +77,13 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
|
77
77
|
for stmt in [*CONSTRAINTS, *INDEXES]:
|
|
78
78
|
s.run(stmt)
|
|
79
79
|
|
|
80
|
+
# The application anchor (a shared node) — used to scope the orphan prune
|
|
81
|
+
# so it never touches modules belonging to a different :PyApplication.
|
|
82
|
+
app_name = next(
|
|
83
|
+
(n.value for n in rows.nodes if n.labels and n.labels[0] == "PyApplication"),
|
|
84
|
+
None,
|
|
85
|
+
)
|
|
86
|
+
|
|
80
87
|
# Partition nodes by owning module; shared nodes have no _module.
|
|
81
88
|
by_module: Dict[str, List[NodeRow]] = {}
|
|
82
89
|
shared: List[NodeRow] = []
|
|
@@ -135,13 +142,17 @@ def bolt_writer(rows: GraphRows, cfg: BoltConfig, full_run: bool) -> None:
|
|
|
135
142
|
_upsert_edges(session, neo4j, edges)
|
|
136
143
|
|
|
137
144
|
# 6. orphan prune — only safe on a full run (a targeted run can't tell deleted from untargeted).
|
|
138
|
-
|
|
145
|
+
# Scope to THIS application's anchor so a full run for application B never
|
|
146
|
+
# deletes application A's modules from a shared database.
|
|
147
|
+
if full_run and app_name is not None:
|
|
139
148
|
present = list(by_module.keys())
|
|
140
149
|
with session() as s:
|
|
141
150
|
res = s.run(
|
|
142
|
-
"MATCH (m:PyModule)
|
|
151
|
+
"MATCH (:PyApplication {name: $app})-[:PY_HAS_MODULE]->(m:PyModule) "
|
|
152
|
+
"WHERE NOT m.file_key IN $present "
|
|
143
153
|
f"OPTIONAL MATCH (m)-{DESCENDANTS}->(x) DETACH DELETE x, m "
|
|
144
154
|
"RETURN count(m) AS pruned",
|
|
155
|
+
app=app_name,
|
|
145
156
|
present=present,
|
|
146
157
|
)
|
|
147
158
|
pruned = res.single()
|
codeanalyzer/neo4j/catalog.py
CHANGED
|
@@ -34,7 +34,7 @@ from typing import Dict, List
|
|
|
34
34
|
|
|
35
35
|
from codeanalyzer.neo4j.schema import CONSTRAINTS, INDEXES
|
|
36
36
|
|
|
37
|
-
SCHEMA_VERSION = "1.
|
|
37
|
+
SCHEMA_VERSION = "1.1.0"
|
|
38
38
|
|
|
39
39
|
# PropType ∈ {"string", "integer", "float", "boolean", "string[]", "integer[]"}.
|
|
40
40
|
|
|
@@ -119,7 +119,7 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
119
119
|
"PyExternal",
|
|
120
120
|
"PySymbol",
|
|
121
121
|
"signature",
|
|
122
|
-
{"signature": "string", "name": "string"},
|
|
122
|
+
{"signature": "string", "name": "string", "module": "string"},
|
|
123
123
|
),
|
|
124
124
|
NodeLabel("PyPackage", "PyPackage", "name", {"name": "string"}),
|
|
125
125
|
NodeLabel(
|
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -60,11 +60,12 @@ def project(app: PyApplication, app_name: str) -> GraphRows:
|
|
|
60
60
|
b.edge("PY_HAS_MODULE", app_ref, mod_ref)
|
|
61
61
|
_project_module_body(b, file_key, mod_ref, mod)
|
|
62
62
|
|
|
63
|
-
# The aggregated :PY_CALLS twin. Endpoints
|
|
64
|
-
# :PyExternal ghost nodes
|
|
63
|
+
# The aggregated :PY_CALLS twin. Endpoints listed in app.external_symbols become
|
|
64
|
+
# :PyExternal ghost nodes; the rest are declared :PySymbol nodes already emitted.
|
|
65
|
+
externals = app.external_symbols or {}
|
|
65
66
|
for e in app.call_graph:
|
|
66
|
-
src = _call_endpoint(b, e.source)
|
|
67
|
-
tgt = _call_endpoint(b, e.target)
|
|
67
|
+
src = _call_endpoint(b, e.source, externals)
|
|
68
|
+
tgt = _call_endpoint(b, e.target, externals)
|
|
68
69
|
b.edge("PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.provenance or [])))
|
|
69
70
|
|
|
70
71
|
return b.finish()
|
|
@@ -74,13 +75,20 @@ def _sym(signature: str) -> NodeRef:
|
|
|
74
75
|
return NodeRef("PySymbol", "signature", signature)
|
|
75
76
|
|
|
76
77
|
|
|
77
|
-
def _call_endpoint(b: RowBuilder, signature: str) -> NodeRef:
|
|
78
|
-
"""A call-graph endpoint: a
|
|
79
|
-
|
|
80
|
-
|
|
78
|
+
def _call_endpoint(b: RowBuilder, signature: str, externals: dict) -> NodeRef:
|
|
79
|
+
"""A call-graph endpoint: a declared callable already emitted, or an external
|
|
80
|
+
symbol (imported library / builtin member) materialized as a :PyExternal ghost.
|
|
81
|
+
|
|
82
|
+
Classification is authoritative -- it comes from ``app.external_symbols``, not a
|
|
83
|
+
"present in the graph" heuristic -- so an imported module name (which exists only
|
|
84
|
+
as a :PyPackage) can never shadow the call target. A small fallback still
|
|
85
|
+
materializes an external for any endpoint that is neither declared nor listed."""
|
|
86
|
+
ext = externals.get(signature)
|
|
87
|
+
if ext is None and b.has_key("PySymbol", signature):
|
|
81
88
|
return _sym(signature)
|
|
82
|
-
name = signature.rsplit(".", 1)[-1] if "." in signature else signature
|
|
83
|
-
|
|
89
|
+
name = ext.name if ext is not None else (signature.rsplit(".", 1)[-1] if "." in signature else signature)
|
|
90
|
+
module = ext.module if ext is not None else None
|
|
91
|
+
return b.node(["PySymbol", "PyExternal"], "signature", signature, prune({"name": name, "module": module}))
|
|
84
92
|
|
|
85
93
|
|
|
86
94
|
# ----------------------------------------------------------------------------------------------
|