codeanalyzer-python 0.2.1__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +116 -6
- codeanalyzer/core.py +139 -213
- codeanalyzer/neo4j/__init__.py +1 -1
- codeanalyzer/neo4j/emit.py +2 -2
- codeanalyzer/neo4j/project.py +84 -18
- codeanalyzer/neo4j/schema.py +261 -15
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +20 -1
- codeanalyzer/provenance.py +61 -0
- codeanalyzer/schema/py_schema.py +39 -1
- codeanalyzer/semantic_analysis/call_graph.py +24 -3
- codeanalyzer/semantic_analysis/pycg/__init__.py +20 -0
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +1054 -0
- codeanalyzer/semantic_analysis/{codeql/__init__.py → pycg/pycg_exceptions.py} +5 -8
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +401 -0
- codeanalyzer/syntactic_analysis/import_resolver.py +67 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +74 -10
- codeanalyzer/utils/logging.py +5 -2
- codeanalyzer/utils/progress_bar.py +15 -7
- codeanalyzer_python-0.3.1.dist-info/METADATA +553 -0
- codeanalyzer_python-0.3.1.dist-info/RECORD +39 -0
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/WHEEL +1 -1
- codeanalyzer/neo4j/catalog.py +0 -245
- codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +0 -382
- codeanalyzer/semantic_analysis/codeql/codeql_exceptions.py +0 -12
- codeanalyzer/semantic_analysis/codeql/codeql_loader.py +0 -91
- codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py +0 -185
- codeanalyzer_python-0.2.1.dist-info/METADATA +0 -415
- codeanalyzer_python-0.2.1.dist-info/RECORD +0 -39
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/__main__.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from importlib.metadata import version as _pkg_version, PackageNotFoundError
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
from typing import Optional, Annotated
|
|
3
4
|
|
|
@@ -7,10 +8,35 @@ from codeanalyzer.core import Codeanalyzer
|
|
|
7
8
|
from codeanalyzer.utils import _set_log_level, logger
|
|
8
9
|
from codeanalyzer.config import OutputFormat
|
|
9
10
|
from codeanalyzer.schema import model_dump_json
|
|
10
|
-
from codeanalyzer.options import AnalysisOptions, EmitTarget
|
|
11
|
+
from codeanalyzer.options import AnalysisOptions, EmitTarget, ShardStrategy
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _version_callback(value: bool) -> None:
|
|
15
|
+
"""Print the installed ``codeanalyzer-python`` version and exit.
|
|
16
|
+
|
|
17
|
+
Eager so it fires before the rest of the CLI callback (no -i/--input
|
|
18
|
+
required). Reads the version from package metadata so it always reflects
|
|
19
|
+
what is actually installed rather than a hardcoded string."""
|
|
20
|
+
if not value:
|
|
21
|
+
return
|
|
22
|
+
try:
|
|
23
|
+
installed = _pkg_version("codeanalyzer-python")
|
|
24
|
+
except PackageNotFoundError:
|
|
25
|
+
installed = "unknown"
|
|
26
|
+
typer.echo(f"canpy {installed}")
|
|
27
|
+
raise typer.Exit()
|
|
11
28
|
|
|
12
29
|
|
|
13
30
|
def main(
|
|
31
|
+
version: Annotated[
|
|
32
|
+
Optional[bool],
|
|
33
|
+
typer.Option(
|
|
34
|
+
"--version",
|
|
35
|
+
help="Show the canpy version and exit.",
|
|
36
|
+
callback=_version_callback,
|
|
37
|
+
is_eager=True,
|
|
38
|
+
),
|
|
39
|
+
] = None,
|
|
14
40
|
input: Annotated[
|
|
15
41
|
Optional[Path],
|
|
16
42
|
typer.Option(
|
|
@@ -83,9 +109,16 @@ def main(
|
|
|
83
109
|
help="Neo4j database name (default: server default). [env: NEO4J_DATABASE]",
|
|
84
110
|
),
|
|
85
111
|
] = None,
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
112
|
+
analysis_level: Annotated[
|
|
113
|
+
int,
|
|
114
|
+
typer.Option(
|
|
115
|
+
"-a",
|
|
116
|
+
"--analysis-level",
|
|
117
|
+
help="Analysis depth: 1=symbol table+Jedi call graph, 2=+PyCG call graph.",
|
|
118
|
+
min=1,
|
|
119
|
+
max=2,
|
|
120
|
+
),
|
|
121
|
+
] = 1,
|
|
89
122
|
using_ray: Annotated[
|
|
90
123
|
bool,
|
|
91
124
|
typer.Option("--ray/--no-ray", help="Enable Ray for distributed analysis."),
|
|
@@ -137,6 +170,78 @@ def main(
|
|
|
137
170
|
verbosity: Annotated[
|
|
138
171
|
int, typer.Option("-v", count=True, help="Increase verbosity: -v, -vv, -vvv")
|
|
139
172
|
] = 0,
|
|
173
|
+
pycg_shard: Annotated[
|
|
174
|
+
bool,
|
|
175
|
+
typer.Option(
|
|
176
|
+
"--pycg-shard/--no-pycg-shard",
|
|
177
|
+
help=(
|
|
178
|
+
"Shard PyCG call-graph analysis by Python package (level 2 only). "
|
|
179
|
+
"When the project exceeds the 500-file ceiling, PyCG is run "
|
|
180
|
+
"independently per top-level package with cross-package imports "
|
|
181
|
+
"treated as ghost nodes. Without this flag, projects over the "
|
|
182
|
+
"ceiling fall back to Jedi-only edges."
|
|
183
|
+
),
|
|
184
|
+
),
|
|
185
|
+
] = False,
|
|
186
|
+
pycg_shard_ceiling: Annotated[
|
|
187
|
+
int,
|
|
188
|
+
typer.Option(
|
|
189
|
+
"--pycg-shard-ceiling",
|
|
190
|
+
help=(
|
|
191
|
+
"Maximum files per shard when --pycg-shard is active (default 100). "
|
|
192
|
+
"Shards exceeding this limit are skipped; their call edges are "
|
|
193
|
+
"omitted from the call graph (Jedi edges for those packages are "
|
|
194
|
+
"still included). Lower values are safer for packages with deep "
|
|
195
|
+
"class hierarchies or heavy import graphs."
|
|
196
|
+
),
|
|
197
|
+
min=1,
|
|
198
|
+
),
|
|
199
|
+
] = 100,
|
|
200
|
+
pycg_shard_timeout: Annotated[
|
|
201
|
+
int,
|
|
202
|
+
typer.Option(
|
|
203
|
+
"--pycg-shard-timeout",
|
|
204
|
+
help=(
|
|
205
|
+
"Per-shard wall-clock timeout in seconds when --pycg-shard is "
|
|
206
|
+
"active (default 120). A shard that exceeds this limit is skipped "
|
|
207
|
+
"gracefully. PyCG's fixpoint is bimodal: it either converges "
|
|
208
|
+
"quickly or diverges indefinitely, so the timeout acts as a final "
|
|
209
|
+
"safety net after the file-count ceiling. Set to 0 to disable. "
|
|
210
|
+
"POSIX only (macOS / Linux); ignored on Windows."
|
|
211
|
+
),
|
|
212
|
+
min=0,
|
|
213
|
+
),
|
|
214
|
+
] = 120,
|
|
215
|
+
pycg_shard_strategy: Annotated[
|
|
216
|
+
ShardStrategy,
|
|
217
|
+
typer.Option(
|
|
218
|
+
"--pycg-shard-strategy",
|
|
219
|
+
help=(
|
|
220
|
+
"How --pycg-shard groups files (level 2 only). 'jedi' (default) "
|
|
221
|
+
"partitions the Jedi module-dependency graph (SCC + Louvain) so "
|
|
222
|
+
"tightly-coupled modules co-compute and few call edges are "
|
|
223
|
+
"severed between shards; import cycles are never split. "
|
|
224
|
+
"'package' uses the legacy one-shard-per-package-directory "
|
|
225
|
+
"grouping."
|
|
226
|
+
),
|
|
227
|
+
),
|
|
228
|
+
] = ShardStrategy.JEDI,
|
|
229
|
+
pycg_max_iter: Annotated[
|
|
230
|
+
int,
|
|
231
|
+
typer.Option(
|
|
232
|
+
"--pycg-max-iter",
|
|
233
|
+
help=(
|
|
234
|
+
"Cap on PyCG's fixpoint passes per shard/project (level 2; "
|
|
235
|
+
"default 50). PyCG iterates until its points-to state stops "
|
|
236
|
+
"changing, but its access-path domain has no convergence bound, "
|
|
237
|
+
"so heavy metaclass/mixin code (e.g. an ORM) can loop with each "
|
|
238
|
+
"pass costing seconds. The cap returns a sound-but-incomplete "
|
|
239
|
+
"call graph instead of looping until the timeout kills it. "
|
|
240
|
+
"Set to -1 for PyCG's unbounded run-to-convergence behaviour."
|
|
241
|
+
),
|
|
242
|
+
min=-1,
|
|
243
|
+
),
|
|
244
|
+
] = 50,
|
|
140
245
|
):
|
|
141
246
|
options = AnalysisOptions(
|
|
142
247
|
input=input,
|
|
@@ -148,7 +253,7 @@ def main(
|
|
|
148
253
|
neo4j_user=neo4j_user,
|
|
149
254
|
neo4j_password=neo4j_password,
|
|
150
255
|
neo4j_database=neo4j_database,
|
|
151
|
-
|
|
256
|
+
analysis_level=analysis_level,
|
|
152
257
|
using_ray=using_ray,
|
|
153
258
|
rebuild_analysis=rebuild_analysis,
|
|
154
259
|
skip_tests=skip_tests,
|
|
@@ -157,6 +262,11 @@ def main(
|
|
|
157
262
|
cache_dir=cache_dir,
|
|
158
263
|
clear_cache=clear_cache,
|
|
159
264
|
verbosity=verbosity,
|
|
265
|
+
pycg_shard=pycg_shard,
|
|
266
|
+
pycg_shard_ceiling=pycg_shard_ceiling,
|
|
267
|
+
pycg_shard_timeout=pycg_shard_timeout,
|
|
268
|
+
pycg_shard_strategy=pycg_shard_strategy,
|
|
269
|
+
pycg_max_iter=pycg_max_iter,
|
|
160
270
|
)
|
|
161
271
|
|
|
162
272
|
_set_log_level(options.verbosity)
|
|
@@ -230,7 +340,7 @@ def _write_output(artifacts, output_dir: Path, format: OutputFormat):
|
|
|
230
340
|
app = typer.Typer(
|
|
231
341
|
callback=main,
|
|
232
342
|
name="canpy",
|
|
233
|
-
help="Static Analysis on Python source code using Jedi,
|
|
343
|
+
help="Static Analysis on Python source code using Jedi, PyCG and Tree sitter.",
|
|
234
344
|
invoke_without_command=True,
|
|
235
345
|
no_args_is_help=True,
|
|
236
346
|
add_completion=False,
|
codeanalyzer/core.py
CHANGED
|
@@ -6,6 +6,8 @@ import sys
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Any, Dict, Optional, Union, List
|
|
8
8
|
|
|
9
|
+
import time
|
|
10
|
+
|
|
9
11
|
import ray
|
|
10
12
|
from codeanalyzer.utils import logger
|
|
11
13
|
from codeanalyzer.schema import (
|
|
@@ -17,17 +19,18 @@ from codeanalyzer.schema import (
|
|
|
17
19
|
)
|
|
18
20
|
from codeanalyzer.schema.py_schema import PyCallEdge
|
|
19
21
|
from codeanalyzer.semantic_analysis.call_graph import (
|
|
22
|
+
filter_external_edges,
|
|
20
23
|
jedi_call_graph_edges,
|
|
21
24
|
merge_edges,
|
|
22
25
|
resolve_unresolved_constructors,
|
|
23
26
|
)
|
|
24
|
-
from codeanalyzer.semantic_analysis.
|
|
25
|
-
from codeanalyzer.semantic_analysis.codeql.codeql_analysis import CodeQL
|
|
26
|
-
from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import CodeQLExceptions
|
|
27
|
+
from codeanalyzer.semantic_analysis.pycg import PyCG, PyCGExceptions
|
|
27
28
|
from codeanalyzer.syntactic_analysis.exceptions import SymbolTableBuilderRayError
|
|
29
|
+
from codeanalyzer.syntactic_analysis.import_resolver import resolve_imports
|
|
28
30
|
from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
|
|
29
31
|
from codeanalyzer.utils import ProgressBar
|
|
30
32
|
from codeanalyzer.options import AnalysisOptions
|
|
33
|
+
from codeanalyzer.provenance import analyzer_info, repository_info
|
|
31
34
|
|
|
32
35
|
@ray.remote
|
|
33
36
|
def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, str], virtualenv: Union[Path, str, None]) -> Dict[str, PyModule]:
|
|
@@ -54,7 +57,7 @@ def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, s
|
|
|
54
57
|
|
|
55
58
|
|
|
56
59
|
class Codeanalyzer:
|
|
57
|
-
"""Core
|
|
60
|
+
"""Core static analysis engine for Python projects.
|
|
58
61
|
|
|
59
62
|
Args:
|
|
60
63
|
options (AnalysisOptions): Analysis configuration options containing all necessary parameters.
|
|
@@ -64,16 +67,13 @@ class Codeanalyzer:
|
|
|
64
67
|
self.options = options
|
|
65
68
|
self.project_dir = Path(options.input).resolve()
|
|
66
69
|
self.skip_tests = options.skip_tests
|
|
67
|
-
self.
|
|
70
|
+
self.analysis_level = options.analysis_level
|
|
68
71
|
self.rebuild_analysis = options.rebuild_analysis
|
|
69
72
|
self.no_venv = options.no_venv
|
|
70
73
|
self.cache_dir = (
|
|
71
74
|
options.cache_dir.resolve() if options.cache_dir is not None else self.project_dir
|
|
72
75
|
) / ".codeanalyzer"
|
|
73
76
|
self.clear_cache = options.clear_cache
|
|
74
|
-
self.db_path: Optional[Path] = None
|
|
75
|
-
self.codeql_bin: Optional[Path] = None
|
|
76
|
-
self.codeql_packs_dir: Optional[Path] = None
|
|
77
77
|
self.virtualenv: Optional[Path] = None
|
|
78
78
|
self.using_ray: bool = options.using_ray
|
|
79
79
|
self.file_name: Optional[Path] = options.file_name
|
|
@@ -85,6 +85,7 @@ class Codeanalyzer:
|
|
|
85
85
|
capture_output: bool = True,
|
|
86
86
|
check: bool = True,
|
|
87
87
|
suppress_output: bool = False,
|
|
88
|
+
log_on_failure: bool = True,
|
|
88
89
|
) -> subprocess.CompletedProcess:
|
|
89
90
|
"""
|
|
90
91
|
Runs a subprocess with real-time output streaming to the logger.
|
|
@@ -94,7 +95,10 @@ class Codeanalyzer:
|
|
|
94
95
|
cwd: Working directory to run the command in.
|
|
95
96
|
capture_output: If True, retains and returns the output.
|
|
96
97
|
check: If True, raises CalledProcessError on non-zero exit.
|
|
97
|
-
suppress_output: If True, silences
|
|
98
|
+
suppress_output: If True, silences per-line debug output.
|
|
99
|
+
log_on_failure: If False, suppresses the error-level log on
|
|
100
|
+
non-zero exit (use when the caller handles the exception and
|
|
101
|
+
will emit its own diagnostic).
|
|
98
102
|
|
|
99
103
|
Returns:
|
|
100
104
|
subprocess.CompletedProcess
|
|
@@ -125,9 +129,10 @@ class Codeanalyzer:
|
|
|
125
129
|
|
|
126
130
|
if check and returncode != 0:
|
|
127
131
|
error_output = "\n".join(output_lines)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
132
|
+
if log_on_failure:
|
|
133
|
+
logger.error(f"Command failed with exit code {returncode}: {' '.join(cmd)}")
|
|
134
|
+
if error_output:
|
|
135
|
+
logger.error(f"Command output:\n{error_output}")
|
|
131
136
|
raise subprocess.CalledProcessError(returncode, cmd, output=error_output)
|
|
132
137
|
|
|
133
138
|
return subprocess.CompletedProcess(
|
|
@@ -235,26 +240,36 @@ class Codeanalyzer:
|
|
|
235
240
|
|
|
236
241
|
@staticmethod
|
|
237
242
|
def _uv_bin() -> Optional[str]:
|
|
238
|
-
"""Path to
|
|
239
|
-
dependency, so
|
|
240
|
-
|
|
243
|
+
"""Path to the uv binary bundled with the ``uv`` PyPI package (a declared
|
|
244
|
+
dependency, so always present in our install -- including inside a Docker
|
|
245
|
+
image). We deliberately ignore any uv on PATH so the analyzer always uses
|
|
246
|
+
the pinned, vendored uv. Returns ``None`` only if the package is somehow
|
|
247
|
+
missing (callers fall back to pip)."""
|
|
241
248
|
try:
|
|
242
249
|
from uv import find_uv_bin
|
|
243
|
-
|
|
244
250
|
return str(find_uv_bin())
|
|
245
251
|
except Exception:
|
|
246
|
-
return
|
|
252
|
+
return None
|
|
247
253
|
|
|
248
254
|
def _install_into_venv(self, venv_python: Path, args: List[str]) -> None:
|
|
249
255
|
"""Install packages into the target venv, preferring uv for speed (parallel
|
|
250
256
|
downloads + a shared global cache) and falling back to the venv's own pip
|
|
251
|
-
when uv is unavailable.
|
|
257
|
+
when uv is unavailable.
|
|
258
|
+
|
|
259
|
+
Raises ``subprocess.CalledProcessError`` on failure; callers in
|
|
260
|
+
``__enter__`` catch this and warn-and-continue so a single failing
|
|
261
|
+
package (e.g. a C extension that needs system libs) does not abort the
|
|
262
|
+
entire analysis.
|
|
263
|
+
"""
|
|
252
264
|
uv = self._uv_bin()
|
|
253
265
|
if uv:
|
|
254
266
|
cmd = [uv, "pip", "install", "--python", str(venv_python), *args]
|
|
255
267
|
else:
|
|
256
268
|
cmd = [str(venv_python), "-m", "pip", "install", *args]
|
|
257
|
-
self._cmd_exec_helper(
|
|
269
|
+
self._cmd_exec_helper(
|
|
270
|
+
cmd, cwd=self.project_dir, check=True,
|
|
271
|
+
suppress_output=True, log_on_failure=False,
|
|
272
|
+
)
|
|
258
273
|
|
|
259
274
|
def __enter__(self) -> "Codeanalyzer":
|
|
260
275
|
# If no virtualenv is provided, try to create one using requirements.txt or pyproject.toml
|
|
@@ -287,21 +302,32 @@ class Codeanalyzer:
|
|
|
287
302
|
for dep_file, _ in dependency_files:
|
|
288
303
|
if (self.project_dir / dep_file).exists():
|
|
289
304
|
logger.info(f"Installing dependencies from {dep_file}")
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
305
|
+
try:
|
|
306
|
+
self._install_into_venv(
|
|
307
|
+
venv_python,
|
|
308
|
+
["--upgrade", "-r", str(self.project_dir / dep_file)],
|
|
309
|
+
)
|
|
310
|
+
except subprocess.CalledProcessError as exc:
|
|
311
|
+
logger.warning(
|
|
312
|
+
f"Dependency installation from {dep_file} failed "
|
|
313
|
+
f"(exit {exc.returncode}) — continuing without it. "
|
|
314
|
+
"Jedi type resolution may be incomplete."
|
|
315
|
+
)
|
|
294
316
|
|
|
295
317
|
# Handle Pipenv files
|
|
296
318
|
if (self.project_dir / "Pipfile").exists():
|
|
297
319
|
logger.info("Installing dependencies from Pipfile")
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
320
|
+
try:
|
|
321
|
+
self._install_into_venv(venv_python, ["pipenv"])
|
|
322
|
+
self._cmd_exec_helper(
|
|
323
|
+
["pipenv", "install", "--dev"],
|
|
324
|
+
cwd=self.project_dir,
|
|
325
|
+
check=True,
|
|
326
|
+
)
|
|
327
|
+
except subprocess.CalledProcessError as exc:
|
|
328
|
+
logger.warning(
|
|
329
|
+
f"Pipenv installation failed (exit {exc.returncode}) — continuing without it."
|
|
330
|
+
)
|
|
305
331
|
|
|
306
332
|
# Handle conda environment files
|
|
307
333
|
conda_files = ["conda.yml", "environment.yml"]
|
|
@@ -319,7 +345,13 @@ class Codeanalyzer:
|
|
|
319
345
|
|
|
320
346
|
if any((self.project_dir / file).exists() for file in package_definition_files):
|
|
321
347
|
logger.info("Installing project in editable mode")
|
|
322
|
-
|
|
348
|
+
try:
|
|
349
|
+
self._install_into_venv(venv_python, ["-e", str(self.project_dir)])
|
|
350
|
+
except subprocess.CalledProcessError as exc:
|
|
351
|
+
logger.warning(
|
|
352
|
+
f"Editable install failed (exit {exc.returncode}) — "
|
|
353
|
+
"continuing without it. Jedi type resolution may be incomplete."
|
|
354
|
+
)
|
|
323
355
|
else:
|
|
324
356
|
logger.warning("No package definition files found, skipping editable installation")
|
|
325
357
|
|
|
@@ -331,60 +363,6 @@ class Codeanalyzer:
|
|
|
331
363
|
if not self.no_venv and venv_path.exists():
|
|
332
364
|
self.virtualenv = venv_path
|
|
333
365
|
|
|
334
|
-
if self.using_codeql:
|
|
335
|
-
logger.info(f"(Re-)initializing CodeQL analysis for {self.project_dir}")
|
|
336
|
-
|
|
337
|
-
# Resolve the CLI binary before anything else uses it: DB build
|
|
338
|
-
# below needs it, and so does every subsequent query run.
|
|
339
|
-
self.codeql_bin = self._ensure_codeql_bin()
|
|
340
|
-
# Download the standard query library pack (idempotent). The
|
|
341
|
-
# CLI install ships only the language extractors; the
|
|
342
|
-
# ``codeql/python-all`` library pack must be fetched separately.
|
|
343
|
-
self.codeql_packs_dir = self._ensure_codeql_packs(self.codeql_bin)
|
|
344
|
-
|
|
345
|
-
cache_root = self.cache_dir / "codeql"
|
|
346
|
-
cache_root.mkdir(parents=True, exist_ok=True)
|
|
347
|
-
self.db_path = cache_root / f"{self.project_dir.name}-db"
|
|
348
|
-
self.db_path.mkdir(exist_ok=True)
|
|
349
|
-
|
|
350
|
-
checksum_file = self.db_path / ".checksum"
|
|
351
|
-
current_checksum = self._compute_checksum(self.project_dir)
|
|
352
|
-
|
|
353
|
-
def is_cache_valid() -> bool:
|
|
354
|
-
if not (self.db_path / "db-python").exists():
|
|
355
|
-
return False
|
|
356
|
-
if not checksum_file.exists():
|
|
357
|
-
return False
|
|
358
|
-
return checksum_file.read_text().strip() == current_checksum
|
|
359
|
-
|
|
360
|
-
if self.rebuild_analysis or not is_cache_valid():
|
|
361
|
-
logger.info("Creating new CodeQL database...")
|
|
362
|
-
|
|
363
|
-
cmd = [
|
|
364
|
-
str(self.codeql_bin),
|
|
365
|
-
"database",
|
|
366
|
-
"create",
|
|
367
|
-
str(self.db_path),
|
|
368
|
-
f"--source-root={self.project_dir}",
|
|
369
|
-
"--language=python",
|
|
370
|
-
"--overwrite",
|
|
371
|
-
]
|
|
372
|
-
|
|
373
|
-
proc = subprocess.Popen(
|
|
374
|
-
cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
|
|
375
|
-
)
|
|
376
|
-
_, err = proc.communicate()
|
|
377
|
-
|
|
378
|
-
if proc.returncode != 0:
|
|
379
|
-
raise CodeQLExceptions.CodeQLDatabaseBuildException(
|
|
380
|
-
f"Error building CodeQL database:\n{err.decode()}"
|
|
381
|
-
)
|
|
382
|
-
|
|
383
|
-
checksum_file.write_text(current_checksum)
|
|
384
|
-
|
|
385
|
-
else:
|
|
386
|
-
logger.info(f"Reusing cached CodeQL DB at {self.db_path}")
|
|
387
|
-
|
|
388
366
|
return self
|
|
389
367
|
|
|
390
368
|
def __exit__(self, *args, **kwargs) -> None:
|
|
@@ -435,8 +413,8 @@ class Codeanalyzer:
|
|
|
435
413
|
Uses caching to avoid re-analyzing unchanged files.
|
|
436
414
|
"""
|
|
437
415
|
cache_file = self.cache_dir / "analysis_cache.json"
|
|
438
|
-
|
|
439
|
-
# Try to load existing cached analysis
|
|
416
|
+
|
|
417
|
+
# Try to load existing cached analysis
|
|
440
418
|
cached_pyapplication = None
|
|
441
419
|
if not self.rebuild_analysis and cache_file.exists():
|
|
442
420
|
try:
|
|
@@ -446,27 +424,30 @@ class Codeanalyzer:
|
|
|
446
424
|
logger.warning(f"Failed to load cache: {e}. Rebuilding analysis.")
|
|
447
425
|
cached_pyapplication = None
|
|
448
426
|
|
|
427
|
+
if cached_pyapplication is not None and not self._cache_analyzer_matches(
|
|
428
|
+
cached_pyapplication, analyzer_info(self.analysis_level).version
|
|
429
|
+
):
|
|
430
|
+
logger.info("Analysis cache written by a different analyzer version; rebuilding.")
|
|
431
|
+
cached_pyapplication = None
|
|
432
|
+
|
|
449
433
|
# Build symbol table from cached application if available (if no available, the build a new one)
|
|
450
434
|
symbol_table = self._build_symbol_table(cached_pyapplication.symbol_table if cached_pyapplication else {})
|
|
451
435
|
|
|
452
|
-
# Build the call graph in four steps:
|
|
453
|
-
# 1. Run CodeQL (when enabled). Produces resolved edges with
|
|
454
|
-
# ``provenance=["codeql"]`` and augments ``PyCallsite``s
|
|
455
|
-
# in-place — filling ``callee_signature`` for sites Jedi
|
|
456
|
-
# couldn't resolve.
|
|
457
|
-
# 2. Heuristic fallback for constructor calls neither Jedi nor
|
|
458
|
-
# CodeQL could resolve (commonly classes nested inside
|
|
459
|
-
# functions). Walks the symbol table by class short-name +
|
|
460
|
-
# scope and writes ``<class>.__init__`` into the site.
|
|
461
|
-
# 3. Derive Jedi edges from the now-fully-augmented symbol
|
|
462
|
-
# table — these reflect every resolution the symbol table
|
|
463
|
-
# contains, regardless of which pass put it there.
|
|
464
|
-
# 4. Merge with CodeQL edges; provenance unions for edges both
|
|
465
|
-
# backends saw.
|
|
466
|
-
codeql_edges = self._get_call_graph(symbol_table, augment_sites=True)
|
|
467
436
|
resolve_unresolved_constructors(symbol_table)
|
|
437
|
+
|
|
438
|
+
# Level 1: Jedi call graph.
|
|
439
|
+
t0_jedi = time.perf_counter()
|
|
468
440
|
jedi_edges = jedi_call_graph_edges(symbol_table)
|
|
469
|
-
call_graph =
|
|
441
|
+
call_graph = list(jedi_edges)
|
|
442
|
+
logger.info("✅ Jedi: %d edges in %.1fs", len(call_graph), time.perf_counter() - t0_jedi)
|
|
443
|
+
|
|
444
|
+
if self.analysis_level >= 2:
|
|
445
|
+
# Level 2: also add PyCG edges. The Jedi edges double as the
|
|
446
|
+
# coupling graph that drives coupling-aware PyCG sharding.
|
|
447
|
+
pycg_edges = self._get_pycg_call_graph(symbol_table, jedi_edges)
|
|
448
|
+
call_graph = merge_edges(call_graph, pycg_edges)
|
|
449
|
+
|
|
450
|
+
call_graph = filter_external_edges(call_graph, symbol_table)
|
|
470
451
|
|
|
471
452
|
# Classify call-graph endpoints that are not declared in the symbol table
|
|
472
453
|
# (imported library / builtin members) once, so the JSON and Neo4j backends
|
|
@@ -481,12 +462,35 @@ class Codeanalyzer:
|
|
|
481
462
|
.external_symbols(external_symbols)
|
|
482
463
|
.build()
|
|
483
464
|
)
|
|
484
|
-
|
|
465
|
+
|
|
466
|
+
# Every run re-resolves import spellings against the analyzed module
|
|
467
|
+
# set -- pure and cheap; cached modules from older caches default to
|
|
468
|
+
# resolved_module=None and get stamped here (issue #82).
|
|
469
|
+
resolve_imports(app, self.project_dir)
|
|
470
|
+
|
|
471
|
+
# Single choke point for provenance: every produced app (fresh symbol
|
|
472
|
+
# table or reused-from-cache) passes through here before being cached
|
|
473
|
+
# or returned, so analyzer/repository always reflect *this* run/checkout
|
|
474
|
+
# even when the symbol table itself came from the on-disk cache.
|
|
475
|
+
app.analyzer = analyzer_info(self.analysis_level)
|
|
476
|
+
app.repository = repository_info(self.project_dir)
|
|
477
|
+
|
|
485
478
|
# Save to cache
|
|
486
479
|
self._save_analysis_cache(app, cache_file)
|
|
487
480
|
|
|
488
481
|
return app
|
|
489
482
|
|
|
483
|
+
@staticmethod
|
|
484
|
+
def _cache_analyzer_matches(cached_app: Optional[PyApplication], current_version: str) -> bool:
|
|
485
|
+
"""A cache written by another analyzer version (or before versions were
|
|
486
|
+
recorded) may lack fields the current models populate — pydantic fills
|
|
487
|
+
silent defaults, which would masquerade as analyzed absence."""
|
|
488
|
+
return (
|
|
489
|
+
cached_app is not None
|
|
490
|
+
and cached_app.analyzer is not None
|
|
491
|
+
and cached_app.analyzer.version == current_version
|
|
492
|
+
)
|
|
493
|
+
|
|
490
494
|
def _load_pyapplication_from_cache(self, cache_file: Path) -> PyApplication:
|
|
491
495
|
"""Load cached analysis from file.
|
|
492
496
|
|
|
@@ -573,7 +577,8 @@ class Codeanalyzer:
|
|
|
573
577
|
Dict[str, PyModule]: A dictionary mapping file paths to PyModule objects.
|
|
574
578
|
"""
|
|
575
579
|
symbol_table: Dict[str, PyModule] = {}
|
|
576
|
-
|
|
580
|
+
t0_st = time.perf_counter()
|
|
581
|
+
|
|
577
582
|
# Handle single file analysis
|
|
578
583
|
if self.file_name is not None:
|
|
579
584
|
single_file = self.project_dir / self.file_name
|
|
@@ -680,123 +685,44 @@ class Codeanalyzer:
|
|
|
680
685
|
if files_from_cache > 0:
|
|
681
686
|
logger.info(f"Reused {files_from_cache} files from cache, processed {files_processed} new/changed files")
|
|
682
687
|
|
|
683
|
-
logger.info(
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
def _ensure_codeql_packs(self, codeql_bin: Path) -> Path:
|
|
687
|
-
"""Materialize a qlpack that depends on ``codeql/python-all``.
|
|
688
|
-
|
|
689
|
-
The CodeQL CLI install ships only the language extractors — query
|
|
690
|
-
library packs (and their transitive dependencies like
|
|
691
|
-
``codeql/concepts``) must be resolved separately. The canonical
|
|
692
|
-
way is to declare the dependency in a ``qlpack.yml`` and run
|
|
693
|
-
``codeql pack install`` in that directory; CodeQL writes a
|
|
694
|
-
``codeql-pack.lock.yml`` and downloads everything needed.
|
|
695
|
-
|
|
696
|
-
We do this once per project under ``<cache_dir>/codeql/qlpack/``
|
|
697
|
-
and return that directory. The query runner then writes its
|
|
698
|
-
temporary ``.ql`` file inside this pack — colocation makes
|
|
699
|
-
``import python`` resolve without any ``--additional-packs`` or
|
|
700
|
-
``--search-path`` gymnastics.
|
|
701
|
-
"""
|
|
702
|
-
pack_dir = self.cache_dir / "codeql" / "qlpack"
|
|
703
|
-
pack_dir.mkdir(parents=True, exist_ok=True)
|
|
704
|
-
qlpack_yml = pack_dir / "qlpack.yml"
|
|
705
|
-
lock_file = pack_dir / "codeql-pack.lock.yml"
|
|
706
|
-
|
|
707
|
-
if not qlpack_yml.exists():
|
|
708
|
-
qlpack_yml.write_text(
|
|
709
|
-
"name: codeanalyzer-deps\n"
|
|
710
|
-
"version: 1.0.0\n"
|
|
711
|
-
"dependencies:\n"
|
|
712
|
-
' codeql/python-all: "*"\n'
|
|
713
|
-
)
|
|
714
|
-
|
|
715
|
-
if lock_file.exists():
|
|
716
|
-
logger.debug(f"CodeQL pack dependencies already installed in {pack_dir}")
|
|
717
|
-
return pack_dir
|
|
718
|
-
|
|
719
|
-
logger.info(f"Installing CodeQL pack dependencies in {pack_dir}.")
|
|
720
|
-
proc = subprocess.Popen(
|
|
721
|
-
[str(codeql_bin), "pack", "install", str(pack_dir)],
|
|
722
|
-
stdout=subprocess.PIPE,
|
|
723
|
-
stderr=subprocess.PIPE,
|
|
724
|
-
)
|
|
725
|
-
_, err = proc.communicate()
|
|
726
|
-
if proc.returncode != 0:
|
|
727
|
-
raise CodeQLExceptions.CodeQLDatabaseBuildException(
|
|
728
|
-
f"Failed to install CodeQL pack dependencies:\n"
|
|
729
|
-
f"{(err or b'').decode(errors='replace')}"
|
|
730
|
-
)
|
|
731
|
-
return pack_dir
|
|
732
|
-
|
|
733
|
-
def _ensure_codeql_bin(self) -> Path:
|
|
734
|
-
"""Locate (or download) the CodeQL CLI binary into the project cache.
|
|
735
|
-
|
|
736
|
-
Resolution order:
|
|
737
|
-
1. An existing binary inside ``<cache_dir>/codeql/bin/`` —
|
|
738
|
-
reused across runs on the same project.
|
|
739
|
-
2. ``codeql`` already on the user's PATH — picked up verbatim.
|
|
740
|
-
3. Otherwise, download into ``<cache_dir>/codeql/bin/``.
|
|
741
|
-
|
|
742
|
-
The project-local cache is preferred over PATH so the version we
|
|
743
|
-
installed earlier wins over whatever the OS ships — keeps behavior
|
|
744
|
-
deterministic when the user has both.
|
|
745
|
-
"""
|
|
746
|
-
bin_root = self.cache_dir / "codeql" / "bin"
|
|
747
|
-
bin_root.mkdir(parents=True, exist_ok=True)
|
|
748
|
-
|
|
749
|
-
existing = next(
|
|
750
|
-
(p for p in bin_root.rglob("codeql") if p.is_file()),
|
|
751
|
-
None,
|
|
688
|
+
logger.info(
|
|
689
|
+
"✅ Symbol table: %d modules in %.1fs",
|
|
690
|
+
len(symbol_table), time.perf_counter() - t0_st,
|
|
752
691
|
)
|
|
753
|
-
|
|
754
|
-
logger.debug(f"Reusing cached CodeQL CLI at {existing}")
|
|
755
|
-
return existing.resolve()
|
|
756
|
-
|
|
757
|
-
on_path = shutil.which("codeql")
|
|
758
|
-
if on_path:
|
|
759
|
-
logger.debug(f"Using CodeQL CLI from PATH at {on_path}")
|
|
760
|
-
return Path(on_path)
|
|
761
|
-
|
|
762
|
-
logger.info(f"CodeQL CLI not found; downloading into {bin_root}.")
|
|
763
|
-
downloaded = CodeQLLoader.download_and_extract_codeql(bin_root)
|
|
764
|
-
if not downloaded.exists() or not os.access(downloaded, os.X_OK):
|
|
765
|
-
raise FileNotFoundError(
|
|
766
|
-
f"CodeQL binary not executable after download: {downloaded}"
|
|
767
|
-
)
|
|
768
|
-
return downloaded
|
|
692
|
+
return symbol_table
|
|
769
693
|
|
|
770
|
-
def
|
|
694
|
+
def _get_pycg_call_graph(
|
|
771
695
|
self,
|
|
772
696
|
symbol_table: Dict[str, PyModule],
|
|
773
|
-
|
|
697
|
+
jedi_edges: List[PyCallEdge],
|
|
774
698
|
) -> List[PyCallEdge]:
|
|
775
|
-
"""Build
|
|
699
|
+
"""Build PyCG-resolved call edges.
|
|
776
700
|
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
701
|
+
Runs PyCG's iterative name-pointer analysis over the whole project
|
|
702
|
+
and returns edges with ``provenance=["pycg"]``. Falls back to an
|
|
703
|
+
empty list and logs a warning on any failure so the caller can
|
|
704
|
+
continue with Jedi-only edges.
|
|
780
705
|
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
CodeQL query is shared (cached on the ``CodeQL`` instance) so
|
|
785
|
-
this costs no extra DB work.
|
|
706
|
+
*jedi_edges* are the level-1 call edges; under the ``jedi`` shard
|
|
707
|
+
strategy they drive coupling-aware partitioning (see
|
|
708
|
+
:func:`shard_planner.plan_shards`).
|
|
786
709
|
"""
|
|
787
|
-
if not self.using_codeql or self.db_path is None:
|
|
788
|
-
return []
|
|
789
710
|
try:
|
|
790
|
-
|
|
711
|
+
pycg = PyCG(
|
|
791
712
|
self.project_dir,
|
|
792
|
-
self.
|
|
793
|
-
|
|
794
|
-
|
|
713
|
+
skip_tests=self.skip_tests,
|
|
714
|
+
shard=self.options.pycg_shard,
|
|
715
|
+
shard_ceiling=self.options.pycg_shard_ceiling,
|
|
716
|
+
shard_timeout=self.options.pycg_shard_timeout,
|
|
717
|
+
shard_strategy=self.options.pycg_shard_strategy,
|
|
718
|
+
max_iter=self.options.pycg_max_iter,
|
|
719
|
+
using_ray=self.using_ray,
|
|
795
720
|
)
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
return
|
|
800
|
-
except
|
|
801
|
-
logger.warning(f"
|
|
721
|
+
return pycg.build_call_graph_edges(symbol_table, jedi_edges=jedi_edges)
|
|
722
|
+
except PyCGExceptions.PyCGImportError as exc:
|
|
723
|
+
logger.warning(f"PyCG not installed — level 2 edges will be Jedi-only: {exc}")
|
|
724
|
+
return []
|
|
725
|
+
except PyCGExceptions.PyCGAnalysisError as exc:
|
|
726
|
+
logger.warning(f"PyCG analysis failed — level 2 edges will be Jedi-only: {exc}")
|
|
727
|
+
logger.debug("PyCG full traceback:", exc_info=True)
|
|
802
728
|
return []
|