codeanalyzer-python 0.2.1__py3-none-any.whl → 0.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. codeanalyzer/__main__.py +116 -6
  2. codeanalyzer/core.py +139 -213
  3. codeanalyzer/neo4j/__init__.py +1 -1
  4. codeanalyzer/neo4j/emit.py +2 -2
  5. codeanalyzer/neo4j/project.py +84 -18
  6. codeanalyzer/neo4j/schema.py +261 -15
  7. codeanalyzer/options/__init__.py +2 -2
  8. codeanalyzer/options/options.py +20 -1
  9. codeanalyzer/provenance.py +61 -0
  10. codeanalyzer/schema/py_schema.py +39 -1
  11. codeanalyzer/semantic_analysis/call_graph.py +24 -3
  12. codeanalyzer/semantic_analysis/pycg/__init__.py +20 -0
  13. codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +1054 -0
  14. codeanalyzer/semantic_analysis/{codeql/__init__.py → pycg/pycg_exceptions.py} +5 -8
  15. codeanalyzer/semantic_analysis/pycg/shard_planner.py +401 -0
  16. codeanalyzer/syntactic_analysis/import_resolver.py +67 -0
  17. codeanalyzer/syntactic_analysis/symbol_table_builder.py +74 -10
  18. codeanalyzer/utils/logging.py +5 -2
  19. codeanalyzer/utils/progress_bar.py +15 -7
  20. codeanalyzer_python-0.3.1.dist-info/METADATA +553 -0
  21. codeanalyzer_python-0.3.1.dist-info/RECORD +39 -0
  22. {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/WHEEL +1 -1
  23. codeanalyzer/neo4j/catalog.py +0 -245
  24. codeanalyzer/semantic_analysis/codeql/codeql_analysis.py +0 -382
  25. codeanalyzer/semantic_analysis/codeql/codeql_exceptions.py +0 -12
  26. codeanalyzer/semantic_analysis/codeql/codeql_loader.py +0 -91
  27. codeanalyzer/semantic_analysis/codeql/codeql_query_runner.py +0 -185
  28. codeanalyzer_python-0.2.1.dist-info/METADATA +0 -415
  29. codeanalyzer_python-0.2.1.dist-info/RECORD +0 -39
  30. {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/entry_points.txt +0 -0
  31. {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/licenses/LICENSE +0 -0
  32. {codeanalyzer_python-0.2.1.dist-info → codeanalyzer_python-0.3.1.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/__main__.py CHANGED
@@ -1,3 +1,4 @@
1
+ from importlib.metadata import version as _pkg_version, PackageNotFoundError
1
2
  from pathlib import Path
2
3
  from typing import Optional, Annotated
3
4
 
@@ -7,10 +8,35 @@ from codeanalyzer.core import Codeanalyzer
7
8
  from codeanalyzer.utils import _set_log_level, logger
8
9
  from codeanalyzer.config import OutputFormat
9
10
  from codeanalyzer.schema import model_dump_json
10
- from codeanalyzer.options import AnalysisOptions, EmitTarget
11
+ from codeanalyzer.options import AnalysisOptions, EmitTarget, ShardStrategy
12
+
13
+
14
+ def _version_callback(value: bool) -> None:
15
+ """Print the installed ``codeanalyzer-python`` version and exit.
16
+
17
+ Eager so it fires before the rest of the CLI callback (no -i/--input
18
+ required). Reads the version from package metadata so it always reflects
19
+ what is actually installed rather than a hardcoded string."""
20
+ if not value:
21
+ return
22
+ try:
23
+ installed = _pkg_version("codeanalyzer-python")
24
+ except PackageNotFoundError:
25
+ installed = "unknown"
26
+ typer.echo(f"canpy {installed}")
27
+ raise typer.Exit()
11
28
 
12
29
 
13
30
  def main(
31
+ version: Annotated[
32
+ Optional[bool],
33
+ typer.Option(
34
+ "--version",
35
+ help="Show the canpy version and exit.",
36
+ callback=_version_callback,
37
+ is_eager=True,
38
+ ),
39
+ ] = None,
14
40
  input: Annotated[
15
41
  Optional[Path],
16
42
  typer.Option(
@@ -83,9 +109,16 @@ def main(
83
109
  help="Neo4j database name (default: server default). [env: NEO4J_DATABASE]",
84
110
  ),
85
111
  ] = None,
86
- using_codeql: Annotated[
87
- bool, typer.Option("--codeql/--no-codeql", help="Enable CodeQL-based analysis.")
88
- ] = False,
112
+ analysis_level: Annotated[
113
+ int,
114
+ typer.Option(
115
+ "-a",
116
+ "--analysis-level",
117
+ help="Analysis depth: 1=symbol table+Jedi call graph, 2=+PyCG call graph.",
118
+ min=1,
119
+ max=2,
120
+ ),
121
+ ] = 1,
89
122
  using_ray: Annotated[
90
123
  bool,
91
124
  typer.Option("--ray/--no-ray", help="Enable Ray for distributed analysis."),
@@ -137,6 +170,78 @@ def main(
137
170
  verbosity: Annotated[
138
171
  int, typer.Option("-v", count=True, help="Increase verbosity: -v, -vv, -vvv")
139
172
  ] = 0,
173
+ pycg_shard: Annotated[
174
+ bool,
175
+ typer.Option(
176
+ "--pycg-shard/--no-pycg-shard",
177
+ help=(
178
+ "Shard PyCG call-graph analysis by Python package (level 2 only). "
179
+ "When the project exceeds the 500-file ceiling, PyCG is run "
180
+ "independently per top-level package with cross-package imports "
181
+ "treated as ghost nodes. Without this flag, projects over the "
182
+ "ceiling fall back to Jedi-only edges."
183
+ ),
184
+ ),
185
+ ] = False,
186
+ pycg_shard_ceiling: Annotated[
187
+ int,
188
+ typer.Option(
189
+ "--pycg-shard-ceiling",
190
+ help=(
191
+ "Maximum files per shard when --pycg-shard is active (default 100). "
192
+ "Shards exceeding this limit are skipped; their call edges are "
193
+ "omitted from the call graph (Jedi edges for those packages are "
194
+ "still included). Lower values are safer for packages with deep "
195
+ "class hierarchies or heavy import graphs."
196
+ ),
197
+ min=1,
198
+ ),
199
+ ] = 100,
200
+ pycg_shard_timeout: Annotated[
201
+ int,
202
+ typer.Option(
203
+ "--pycg-shard-timeout",
204
+ help=(
205
+ "Per-shard wall-clock timeout in seconds when --pycg-shard is "
206
+ "active (default 120). A shard that exceeds this limit is skipped "
207
+ "gracefully. PyCG's fixpoint is bimodal: it either converges "
208
+ "quickly or diverges indefinitely, so the timeout acts as a final "
209
+ "safety net after the file-count ceiling. Set to 0 to disable. "
210
+ "POSIX only (macOS / Linux); ignored on Windows."
211
+ ),
212
+ min=0,
213
+ ),
214
+ ] = 120,
215
+ pycg_shard_strategy: Annotated[
216
+ ShardStrategy,
217
+ typer.Option(
218
+ "--pycg-shard-strategy",
219
+ help=(
220
+ "How --pycg-shard groups files (level 2 only). 'jedi' (default) "
221
+ "partitions the Jedi module-dependency graph (SCC + Louvain) so "
222
+ "tightly-coupled modules co-compute and few call edges are "
223
+ "severed between shards; import cycles are never split. "
224
+ "'package' uses the legacy one-shard-per-package-directory "
225
+ "grouping."
226
+ ),
227
+ ),
228
+ ] = ShardStrategy.JEDI,
229
+ pycg_max_iter: Annotated[
230
+ int,
231
+ typer.Option(
232
+ "--pycg-max-iter",
233
+ help=(
234
+ "Cap on PyCG's fixpoint passes per shard/project (level 2; "
235
+ "default 50). PyCG iterates until its points-to state stops "
236
+ "changing, but its access-path domain has no convergence bound, "
237
+ "so heavy metaclass/mixin code (e.g. an ORM) can loop with each "
238
+ "pass costing seconds. The cap returns a sound-but-incomplete "
239
+ "call graph instead of looping until the timeout kills it. "
240
+ "Set to -1 for PyCG's unbounded run-to-convergence behaviour."
241
+ ),
242
+ min=-1,
243
+ ),
244
+ ] = 50,
140
245
  ):
141
246
  options = AnalysisOptions(
142
247
  input=input,
@@ -148,7 +253,7 @@ def main(
148
253
  neo4j_user=neo4j_user,
149
254
  neo4j_password=neo4j_password,
150
255
  neo4j_database=neo4j_database,
151
- using_codeql=using_codeql,
256
+ analysis_level=analysis_level,
152
257
  using_ray=using_ray,
153
258
  rebuild_analysis=rebuild_analysis,
154
259
  skip_tests=skip_tests,
@@ -157,6 +262,11 @@ def main(
157
262
  cache_dir=cache_dir,
158
263
  clear_cache=clear_cache,
159
264
  verbosity=verbosity,
265
+ pycg_shard=pycg_shard,
266
+ pycg_shard_ceiling=pycg_shard_ceiling,
267
+ pycg_shard_timeout=pycg_shard_timeout,
268
+ pycg_shard_strategy=pycg_shard_strategy,
269
+ pycg_max_iter=pycg_max_iter,
160
270
  )
161
271
 
162
272
  _set_log_level(options.verbosity)
@@ -230,7 +340,7 @@ def _write_output(artifacts, output_dir: Path, format: OutputFormat):
230
340
  app = typer.Typer(
231
341
  callback=main,
232
342
  name="canpy",
233
- help="Static Analysis on Python source code using Jedi, CodeQL and Tree sitter.",
343
+ help="Static Analysis on Python source code using Jedi, PyCG and Tree sitter.",
234
344
  invoke_without_command=True,
235
345
  no_args_is_help=True,
236
346
  add_completion=False,
codeanalyzer/core.py CHANGED
@@ -6,6 +6,8 @@ import sys
6
6
  from pathlib import Path
7
7
  from typing import Any, Dict, Optional, Union, List
8
8
 
9
+ import time
10
+
9
11
  import ray
10
12
  from codeanalyzer.utils import logger
11
13
  from codeanalyzer.schema import (
@@ -17,17 +19,18 @@ from codeanalyzer.schema import (
17
19
  )
18
20
  from codeanalyzer.schema.py_schema import PyCallEdge
19
21
  from codeanalyzer.semantic_analysis.call_graph import (
22
+ filter_external_edges,
20
23
  jedi_call_graph_edges,
21
24
  merge_edges,
22
25
  resolve_unresolved_constructors,
23
26
  )
24
- from codeanalyzer.semantic_analysis.codeql import CodeQLLoader
25
- from codeanalyzer.semantic_analysis.codeql.codeql_analysis import CodeQL
26
- from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import CodeQLExceptions
27
+ from codeanalyzer.semantic_analysis.pycg import PyCG, PyCGExceptions
27
28
  from codeanalyzer.syntactic_analysis.exceptions import SymbolTableBuilderRayError
29
+ from codeanalyzer.syntactic_analysis.import_resolver import resolve_imports
28
30
  from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
29
31
  from codeanalyzer.utils import ProgressBar
30
32
  from codeanalyzer.options import AnalysisOptions
33
+ from codeanalyzer.provenance import analyzer_info, repository_info
31
34
 
32
35
  @ray.remote
33
36
  def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, str], virtualenv: Union[Path, str, None]) -> Dict[str, PyModule]:
@@ -54,7 +57,7 @@ def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, s
54
57
 
55
58
 
56
59
  class Codeanalyzer:
57
- """Core functionality for CodeQL analysis.
60
+ """Core static analysis engine for Python projects.
58
61
 
59
62
  Args:
60
63
  options (AnalysisOptions): Analysis configuration options containing all necessary parameters.
@@ -64,16 +67,13 @@ class Codeanalyzer:
64
67
  self.options = options
65
68
  self.project_dir = Path(options.input).resolve()
66
69
  self.skip_tests = options.skip_tests
67
- self.using_codeql = options.using_codeql
70
+ self.analysis_level = options.analysis_level
68
71
  self.rebuild_analysis = options.rebuild_analysis
69
72
  self.no_venv = options.no_venv
70
73
  self.cache_dir = (
71
74
  options.cache_dir.resolve() if options.cache_dir is not None else self.project_dir
72
75
  ) / ".codeanalyzer"
73
76
  self.clear_cache = options.clear_cache
74
- self.db_path: Optional[Path] = None
75
- self.codeql_bin: Optional[Path] = None
76
- self.codeql_packs_dir: Optional[Path] = None
77
77
  self.virtualenv: Optional[Path] = None
78
78
  self.using_ray: bool = options.using_ray
79
79
  self.file_name: Optional[Path] = options.file_name
@@ -85,6 +85,7 @@ class Codeanalyzer:
85
85
  capture_output: bool = True,
86
86
  check: bool = True,
87
87
  suppress_output: bool = False,
88
+ log_on_failure: bool = True,
88
89
  ) -> subprocess.CompletedProcess:
89
90
  """
90
91
  Runs a subprocess with real-time output streaming to the logger.
@@ -94,7 +95,10 @@ class Codeanalyzer:
94
95
  cwd: Working directory to run the command in.
95
96
  capture_output: If True, retains and returns the output.
96
97
  check: If True, raises CalledProcessError on non-zero exit.
97
- suppress_output: If True, silences log output.
98
+ suppress_output: If True, silences per-line debug output.
99
+ log_on_failure: If False, suppresses the error-level log on
100
+ non-zero exit (use when the caller handles the exception and
101
+ will emit its own diagnostic).
98
102
 
99
103
  Returns:
100
104
  subprocess.CompletedProcess
@@ -125,9 +129,10 @@ class Codeanalyzer:
125
129
 
126
130
  if check and returncode != 0:
127
131
  error_output = "\n".join(output_lines)
128
- logger.error(f"Command failed with exit code {returncode}: {' '.join(cmd)}")
129
- if error_output:
130
- logger.error(f"Command output:\n{error_output}")
132
+ if log_on_failure:
133
+ logger.error(f"Command failed with exit code {returncode}: {' '.join(cmd)}")
134
+ if error_output:
135
+ logger.error(f"Command output:\n{error_output}")
131
136
  raise subprocess.CalledProcessError(returncode, cmd, output=error_output)
132
137
 
133
138
  return subprocess.CompletedProcess(
@@ -235,26 +240,36 @@ class Codeanalyzer:
235
240
 
236
241
  @staticmethod
237
242
  def _uv_bin() -> Optional[str]:
238
- """Path to a uv binary: the one bundled with the ``uv`` PyPI package (a
239
- dependency, so normally always present -- including inside a Docker image),
240
- else a uv on PATH, else ``None`` (callers fall back to pip)."""
243
+ """Path to the uv binary bundled with the ``uv`` PyPI package (a declared
244
+ dependency, so always present in our install -- including inside a Docker
245
+ image). We deliberately ignore any uv on PATH so the analyzer always uses
246
+ the pinned, vendored uv. Returns ``None`` only if the package is somehow
247
+ missing (callers fall back to pip)."""
241
248
  try:
242
249
  from uv import find_uv_bin
243
-
244
250
  return str(find_uv_bin())
245
251
  except Exception:
246
- return shutil.which("uv")
252
+ return None
247
253
 
248
254
  def _install_into_venv(self, venv_python: Path, args: List[str]) -> None:
249
255
  """Install packages into the target venv, preferring uv for speed (parallel
250
256
  downloads + a shared global cache) and falling back to the venv's own pip
251
- when uv is unavailable."""
257
+ when uv is unavailable.
258
+
259
+ Raises ``subprocess.CalledProcessError`` on failure; callers in
260
+ ``__enter__`` catch this and warn-and-continue so a single failing
261
+ package (e.g. a C extension that needs system libs) does not abort the
262
+ entire analysis.
263
+ """
252
264
  uv = self._uv_bin()
253
265
  if uv:
254
266
  cmd = [uv, "pip", "install", "--python", str(venv_python), *args]
255
267
  else:
256
268
  cmd = [str(venv_python), "-m", "pip", "install", *args]
257
- self._cmd_exec_helper(cmd, cwd=self.project_dir, check=True)
269
+ self._cmd_exec_helper(
270
+ cmd, cwd=self.project_dir, check=True,
271
+ suppress_output=True, log_on_failure=False,
272
+ )
258
273
 
259
274
  def __enter__(self) -> "Codeanalyzer":
260
275
  # If no virtualenv is provided, try to create one using requirements.txt or pyproject.toml
@@ -287,21 +302,32 @@ class Codeanalyzer:
287
302
  for dep_file, _ in dependency_files:
288
303
  if (self.project_dir / dep_file).exists():
289
304
  logger.info(f"Installing dependencies from {dep_file}")
290
- self._install_into_venv(
291
- venv_python,
292
- ["--upgrade", "-r", str(self.project_dir / dep_file)],
293
- )
305
+ try:
306
+ self._install_into_venv(
307
+ venv_python,
308
+ ["--upgrade", "-r", str(self.project_dir / dep_file)],
309
+ )
310
+ except subprocess.CalledProcessError as exc:
311
+ logger.warning(
312
+ f"Dependency installation from {dep_file} failed "
313
+ f"(exit {exc.returncode}) — continuing without it. "
314
+ "Jedi type resolution may be incomplete."
315
+ )
294
316
 
295
317
  # Handle Pipenv files
296
318
  if (self.project_dir / "Pipfile").exists():
297
319
  logger.info("Installing dependencies from Pipfile")
298
- # Note: This would require pipenv to be installed
299
- self._install_into_venv(venv_python, ["pipenv"])
300
- self._cmd_exec_helper(
301
- ["pipenv", "install", "--dev"],
302
- cwd=self.project_dir,
303
- check=True,
304
- )
320
+ try:
321
+ self._install_into_venv(venv_python, ["pipenv"])
322
+ self._cmd_exec_helper(
323
+ ["pipenv", "install", "--dev"],
324
+ cwd=self.project_dir,
325
+ check=True,
326
+ )
327
+ except subprocess.CalledProcessError as exc:
328
+ logger.warning(
329
+ f"Pipenv installation failed (exit {exc.returncode}) — continuing without it."
330
+ )
305
331
 
306
332
  # Handle conda environment files
307
333
  conda_files = ["conda.yml", "environment.yml"]
@@ -319,7 +345,13 @@ class Codeanalyzer:
319
345
 
320
346
  if any((self.project_dir / file).exists() for file in package_definition_files):
321
347
  logger.info("Installing project in editable mode")
322
- self._install_into_venv(venv_python, ["-e", str(self.project_dir)])
348
+ try:
349
+ self._install_into_venv(venv_python, ["-e", str(self.project_dir)])
350
+ except subprocess.CalledProcessError as exc:
351
+ logger.warning(
352
+ f"Editable install failed (exit {exc.returncode}) — "
353
+ "continuing without it. Jedi type resolution may be incomplete."
354
+ )
323
355
  else:
324
356
  logger.warning("No package definition files found, skipping editable installation")
325
357
 
@@ -331,60 +363,6 @@ class Codeanalyzer:
331
363
  if not self.no_venv and venv_path.exists():
332
364
  self.virtualenv = venv_path
333
365
 
334
- if self.using_codeql:
335
- logger.info(f"(Re-)initializing CodeQL analysis for {self.project_dir}")
336
-
337
- # Resolve the CLI binary before anything else uses it: DB build
338
- # below needs it, and so does every subsequent query run.
339
- self.codeql_bin = self._ensure_codeql_bin()
340
- # Download the standard query library pack (idempotent). The
341
- # CLI install ships only the language extractors; the
342
- # ``codeql/python-all`` library pack must be fetched separately.
343
- self.codeql_packs_dir = self._ensure_codeql_packs(self.codeql_bin)
344
-
345
- cache_root = self.cache_dir / "codeql"
346
- cache_root.mkdir(parents=True, exist_ok=True)
347
- self.db_path = cache_root / f"{self.project_dir.name}-db"
348
- self.db_path.mkdir(exist_ok=True)
349
-
350
- checksum_file = self.db_path / ".checksum"
351
- current_checksum = self._compute_checksum(self.project_dir)
352
-
353
- def is_cache_valid() -> bool:
354
- if not (self.db_path / "db-python").exists():
355
- return False
356
- if not checksum_file.exists():
357
- return False
358
- return checksum_file.read_text().strip() == current_checksum
359
-
360
- if self.rebuild_analysis or not is_cache_valid():
361
- logger.info("Creating new CodeQL database...")
362
-
363
- cmd = [
364
- str(self.codeql_bin),
365
- "database",
366
- "create",
367
- str(self.db_path),
368
- f"--source-root={self.project_dir}",
369
- "--language=python",
370
- "--overwrite",
371
- ]
372
-
373
- proc = subprocess.Popen(
374
- cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
375
- )
376
- _, err = proc.communicate()
377
-
378
- if proc.returncode != 0:
379
- raise CodeQLExceptions.CodeQLDatabaseBuildException(
380
- f"Error building CodeQL database:\n{err.decode()}"
381
- )
382
-
383
- checksum_file.write_text(current_checksum)
384
-
385
- else:
386
- logger.info(f"Reusing cached CodeQL DB at {self.db_path}")
387
-
388
366
  return self
389
367
 
390
368
  def __exit__(self, *args, **kwargs) -> None:
@@ -435,8 +413,8 @@ class Codeanalyzer:
435
413
  Uses caching to avoid re-analyzing unchanged files.
436
414
  """
437
415
  cache_file = self.cache_dir / "analysis_cache.json"
438
-
439
- # Try to load existing cached analysis
416
+
417
+ # Try to load existing cached analysis
440
418
  cached_pyapplication = None
441
419
  if not self.rebuild_analysis and cache_file.exists():
442
420
  try:
@@ -446,27 +424,30 @@ class Codeanalyzer:
446
424
  logger.warning(f"Failed to load cache: {e}. Rebuilding analysis.")
447
425
  cached_pyapplication = None
448
426
 
427
+ if cached_pyapplication is not None and not self._cache_analyzer_matches(
428
+ cached_pyapplication, analyzer_info(self.analysis_level).version
429
+ ):
430
+ logger.info("Analysis cache written by a different analyzer version; rebuilding.")
431
+ cached_pyapplication = None
432
+
449
433
  # Build symbol table from cached application if available (if no available, the build a new one)
450
434
  symbol_table = self._build_symbol_table(cached_pyapplication.symbol_table if cached_pyapplication else {})
451
435
 
452
- # Build the call graph in four steps:
453
- # 1. Run CodeQL (when enabled). Produces resolved edges with
454
- # ``provenance=["codeql"]`` and augments ``PyCallsite``s
455
- # in-place — filling ``callee_signature`` for sites Jedi
456
- # couldn't resolve.
457
- # 2. Heuristic fallback for constructor calls neither Jedi nor
458
- # CodeQL could resolve (commonly classes nested inside
459
- # functions). Walks the symbol table by class short-name +
460
- # scope and writes ``<class>.__init__`` into the site.
461
- # 3. Derive Jedi edges from the now-fully-augmented symbol
462
- # table — these reflect every resolution the symbol table
463
- # contains, regardless of which pass put it there.
464
- # 4. Merge with CodeQL edges; provenance unions for edges both
465
- # backends saw.
466
- codeql_edges = self._get_call_graph(symbol_table, augment_sites=True)
467
436
  resolve_unresolved_constructors(symbol_table)
437
+
438
+ # Level 1: Jedi call graph.
439
+ t0_jedi = time.perf_counter()
468
440
  jedi_edges = jedi_call_graph_edges(symbol_table)
469
- call_graph = merge_edges(jedi_edges, codeql_edges)
441
+ call_graph = list(jedi_edges)
442
+ logger.info("✅ Jedi: %d edges in %.1fs", len(call_graph), time.perf_counter() - t0_jedi)
443
+
444
+ if self.analysis_level >= 2:
445
+ # Level 2: also add PyCG edges. The Jedi edges double as the
446
+ # coupling graph that drives coupling-aware PyCG sharding.
447
+ pycg_edges = self._get_pycg_call_graph(symbol_table, jedi_edges)
448
+ call_graph = merge_edges(call_graph, pycg_edges)
449
+
450
+ call_graph = filter_external_edges(call_graph, symbol_table)
470
451
 
471
452
  # Classify call-graph endpoints that are not declared in the symbol table
472
453
  # (imported library / builtin members) once, so the JSON and Neo4j backends
@@ -481,12 +462,35 @@ class Codeanalyzer:
481
462
  .external_symbols(external_symbols)
482
463
  .build()
483
464
  )
484
-
465
+
466
+ # Every run re-resolves import spellings against the analyzed module
467
+ # set -- pure and cheap; cached modules from older caches default to
468
+ # resolved_module=None and get stamped here (issue #82).
469
+ resolve_imports(app, self.project_dir)
470
+
471
+ # Single choke point for provenance: every produced app (fresh symbol
472
+ # table or reused-from-cache) passes through here before being cached
473
+ # or returned, so analyzer/repository always reflect *this* run/checkout
474
+ # even when the symbol table itself came from the on-disk cache.
475
+ app.analyzer = analyzer_info(self.analysis_level)
476
+ app.repository = repository_info(self.project_dir)
477
+
485
478
  # Save to cache
486
479
  self._save_analysis_cache(app, cache_file)
487
480
 
488
481
  return app
489
482
 
483
+ @staticmethod
484
+ def _cache_analyzer_matches(cached_app: Optional[PyApplication], current_version: str) -> bool:
485
+ """A cache written by another analyzer version (or before versions were
486
+ recorded) may lack fields the current models populate — pydantic fills
487
+ silent defaults, which would masquerade as analyzed absence."""
488
+ return (
489
+ cached_app is not None
490
+ and cached_app.analyzer is not None
491
+ and cached_app.analyzer.version == current_version
492
+ )
493
+
490
494
  def _load_pyapplication_from_cache(self, cache_file: Path) -> PyApplication:
491
495
  """Load cached analysis from file.
492
496
 
@@ -573,7 +577,8 @@ class Codeanalyzer:
573
577
  Dict[str, PyModule]: A dictionary mapping file paths to PyModule objects.
574
578
  """
575
579
  symbol_table: Dict[str, PyModule] = {}
576
-
580
+ t0_st = time.perf_counter()
581
+
577
582
  # Handle single file analysis
578
583
  if self.file_name is not None:
579
584
  single_file = self.project_dir / self.file_name
@@ -680,123 +685,44 @@ class Codeanalyzer:
680
685
  if files_from_cache > 0:
681
686
  logger.info(f"Reused {files_from_cache} files from cache, processed {files_processed} new/changed files")
682
687
 
683
- logger.info("✅ Symbol table generation complete.")
684
- return symbol_table
685
-
686
- def _ensure_codeql_packs(self, codeql_bin: Path) -> Path:
687
- """Materialize a qlpack that depends on ``codeql/python-all``.
688
-
689
- The CodeQL CLI install ships only the language extractors — query
690
- library packs (and their transitive dependencies like
691
- ``codeql/concepts``) must be resolved separately. The canonical
692
- way is to declare the dependency in a ``qlpack.yml`` and run
693
- ``codeql pack install`` in that directory; CodeQL writes a
694
- ``codeql-pack.lock.yml`` and downloads everything needed.
695
-
696
- We do this once per project under ``<cache_dir>/codeql/qlpack/``
697
- and return that directory. The query runner then writes its
698
- temporary ``.ql`` file inside this pack — colocation makes
699
- ``import python`` resolve without any ``--additional-packs`` or
700
- ``--search-path`` gymnastics.
701
- """
702
- pack_dir = self.cache_dir / "codeql" / "qlpack"
703
- pack_dir.mkdir(parents=True, exist_ok=True)
704
- qlpack_yml = pack_dir / "qlpack.yml"
705
- lock_file = pack_dir / "codeql-pack.lock.yml"
706
-
707
- if not qlpack_yml.exists():
708
- qlpack_yml.write_text(
709
- "name: codeanalyzer-deps\n"
710
- "version: 1.0.0\n"
711
- "dependencies:\n"
712
- ' codeql/python-all: "*"\n'
713
- )
714
-
715
- if lock_file.exists():
716
- logger.debug(f"CodeQL pack dependencies already installed in {pack_dir}")
717
- return pack_dir
718
-
719
- logger.info(f"Installing CodeQL pack dependencies in {pack_dir}.")
720
- proc = subprocess.Popen(
721
- [str(codeql_bin), "pack", "install", str(pack_dir)],
722
- stdout=subprocess.PIPE,
723
- stderr=subprocess.PIPE,
724
- )
725
- _, err = proc.communicate()
726
- if proc.returncode != 0:
727
- raise CodeQLExceptions.CodeQLDatabaseBuildException(
728
- f"Failed to install CodeQL pack dependencies:\n"
729
- f"{(err or b'').decode(errors='replace')}"
730
- )
731
- return pack_dir
732
-
733
- def _ensure_codeql_bin(self) -> Path:
734
- """Locate (or download) the CodeQL CLI binary into the project cache.
735
-
736
- Resolution order:
737
- 1. An existing binary inside ``<cache_dir>/codeql/bin/`` —
738
- reused across runs on the same project.
739
- 2. ``codeql`` already on the user's PATH — picked up verbatim.
740
- 3. Otherwise, download into ``<cache_dir>/codeql/bin/``.
741
-
742
- The project-local cache is preferred over PATH so the version we
743
- installed earlier wins over whatever the OS ships — keeps behavior
744
- deterministic when the user has both.
745
- """
746
- bin_root = self.cache_dir / "codeql" / "bin"
747
- bin_root.mkdir(parents=True, exist_ok=True)
748
-
749
- existing = next(
750
- (p for p in bin_root.rglob("codeql") if p.is_file()),
751
- None,
688
+ logger.info(
689
+ "✅ Symbol table: %d modules in %.1fs",
690
+ len(symbol_table), time.perf_counter() - t0_st,
752
691
  )
753
- if existing and os.access(existing, os.X_OK):
754
- logger.debug(f"Reusing cached CodeQL CLI at {existing}")
755
- return existing.resolve()
756
-
757
- on_path = shutil.which("codeql")
758
- if on_path:
759
- logger.debug(f"Using CodeQL CLI from PATH at {on_path}")
760
- return Path(on_path)
761
-
762
- logger.info(f"CodeQL CLI not found; downloading into {bin_root}.")
763
- downloaded = CodeQLLoader.download_and_extract_codeql(bin_root)
764
- if not downloaded.exists() or not os.access(downloaded, os.X_OK):
765
- raise FileNotFoundError(
766
- f"CodeQL binary not executable after download: {downloaded}"
767
- )
768
- return downloaded
692
+ return symbol_table
769
693
 
770
- def _get_call_graph(
694
+ def _get_pycg_call_graph(
771
695
  self,
772
696
  symbol_table: Dict[str, PyModule],
773
- augment_sites: bool = False,
697
+ jedi_edges: List[PyCallEdge],
774
698
  ) -> List[PyCallEdge]:
775
- """Build CodeQL-resolved call edges and optionally augment sites.
699
+ """Build PyCG-resolved call edges.
776
700
 
777
- Returns an empty list when CodeQL isn't enabled or the database
778
- isn't available. Edges carry ``provenance=["codeql"]`` merge
779
- with Jedi-derived edges via ``call_graph.merge_edges``.
701
+ Runs PyCG's iterative name-pointer analysis over the whole project
702
+ and returns edges with ``provenance=["pycg"]``. Falls back to an
703
+ empty list and logs a warning on any failure so the caller can
704
+ continue with Jedi-only edges.
780
705
 
781
- When ``augment_sites`` is True, also mutates
782
- ``PyCallable.call_sites`` in the symbol table to backfill
783
- ``callee_signature`` for sites Jedi couldn't resolve. The single
784
- CodeQL query is shared (cached on the ``CodeQL`` instance) so
785
- this costs no extra DB work.
706
+ *jedi_edges* are the level-1 call edges; under the ``jedi`` shard
707
+ strategy they drive coupling-aware partitioning (see
708
+ :func:`shard_planner.plan_shards`).
786
709
  """
787
- if not self.using_codeql or self.db_path is None:
788
- return []
789
710
  try:
790
- cq = CodeQL(
711
+ pycg = PyCG(
791
712
  self.project_dir,
792
- self.db_path,
793
- codeql_bin=self.codeql_bin,
794
- codeql_packs_dir=self.codeql_packs_dir,
713
+ skip_tests=self.skip_tests,
714
+ shard=self.options.pycg_shard,
715
+ shard_ceiling=self.options.pycg_shard_ceiling,
716
+ shard_timeout=self.options.pycg_shard_timeout,
717
+ shard_strategy=self.options.pycg_shard_strategy,
718
+ max_iter=self.options.pycg_max_iter,
719
+ using_ray=self.using_ray,
795
720
  )
796
- edges = cq.build_call_graph_edges(symbol_table)
797
- if augment_sites:
798
- cq.augment_call_sites(symbol_table)
799
- return edges
800
- except Exception as exc:
801
- logger.warning(f"CodeQL call-graph extraction failed: {exc}")
721
+ return pycg.build_call_graph_edges(symbol_table, jedi_edges=jedi_edges)
722
+ except PyCGExceptions.PyCGImportError as exc:
723
+ logger.warning(f"PyCG not installed — level 2 edges will be Jedi-only: {exc}")
724
+ return []
725
+ except PyCGExceptions.PyCGAnalysisError as exc:
726
+ logger.warning(f"PyCG analysis failed — level 2 edges will be Jedi-only: {exc}")
727
+ logger.debug("PyCG full traceback:", exc_info=True)
802
728
  return []