codeanalyzer-python 1.1.1__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +95 -123
- codeanalyzer/core.py +21 -45
- codeanalyzer/dataflow/access_paths.py +26 -4
- codeanalyzer/dataflow/builder.py +7 -0
- codeanalyzer/dataflow/identity.py +1 -1
- codeanalyzer/dataflow/pdg.py +7 -2
- codeanalyzer/dataflow/scc.py +1 -1
- codeanalyzer/entrypoints/__init__.py +3 -0
- codeanalyzer/entrypoints/detect.py +124 -0
- codeanalyzer/entrypoints/matching.py +182 -0
- codeanalyzer/entrypoints/pipeline.py +131 -0
- codeanalyzer/entrypoints/rules.py +159 -0
- codeanalyzer/entrypoints/rules.yml +88 -0
- codeanalyzer/neo4j/bolt.py +1 -1
- codeanalyzer/neo4j/project.py +85 -60
- codeanalyzer/neo4j/schema.py +35 -34
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +2 -26
- codeanalyzer/schema/__init__.py +48 -0
- codeanalyzer/schema/l1_body.py +11 -1
- codeanalyzer/schema/l2_callees.py +29 -13
- codeanalyzer/schema/py_schema.py +95 -103
- codeanalyzer/semantic_analysis/call_graph.py +20 -4
- codeanalyzer/semantic_analysis/defuse_linker.py +1499 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +88 -3
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.2.0.dist-info}/METADATA +36 -161
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.2.0.dist-info}/RECORD +31 -30
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.2.0.dist-info}/WHEEL +1 -1
- codeanalyzer/config/__init__.py +0 -3
- codeanalyzer/config/config.py +0 -8
- codeanalyzer/semantic_analysis/pycg/__init__.py +0 -20
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +0 -1115
- codeanalyzer/semantic_analysis/pycg/pycg_exceptions.py +0 -23
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +0 -401
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.2.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.2.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.2.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/__main__.py
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import os
|
|
2
3
|
import sys
|
|
3
4
|
from importlib.metadata import version as _pkg_version, PackageNotFoundError
|
|
4
5
|
from pathlib import Path
|
|
5
|
-
from typing import Optional, Annotated
|
|
6
|
+
from typing import List, Optional, Annotated
|
|
6
7
|
|
|
7
8
|
import typer
|
|
8
9
|
|
|
@@ -10,7 +11,7 @@ import typer
|
|
|
10
11
|
def _pin_hash_seed() -> None:
|
|
11
12
|
"""Re-exec once with ``PYTHONHASHSEED=0`` unless the caller pinned one.
|
|
12
13
|
|
|
13
|
-
|
|
14
|
+
Jedi's inference (and any hash-ordered iteration in the pipeline) walks sets
|
|
14
15
|
keyed on module/access-path strings, so an unpinned per-interpreter hash
|
|
15
16
|
seed makes the emitted L2+ call graph vary run to run (issue #99). The
|
|
16
17
|
seed cannot be set after interpreter start, hence the exec. Export
|
|
@@ -37,9 +38,8 @@ def _pin_hash_seed() -> None:
|
|
|
37
38
|
|
|
38
39
|
from codeanalyzer.core import Codeanalyzer
|
|
39
40
|
from codeanalyzer.utils import _set_log_level, logger
|
|
40
|
-
from codeanalyzer.
|
|
41
|
-
from codeanalyzer.
|
|
42
|
-
from codeanalyzer.options import AnalysisOptions, EmitTarget, ShardStrategy
|
|
41
|
+
from codeanalyzer.schema import model_dump, model_dump_json, strip_internal_only
|
|
42
|
+
from codeanalyzer.options import AnalysisOptions, EmitTarget
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
def _version_callback(value: bool) -> None:
|
|
@@ -80,15 +80,6 @@ def main(
|
|
|
80
80
|
Optional[Path],
|
|
81
81
|
typer.Option("-o", "--output", help="Output directory for artifacts."),
|
|
82
82
|
] = None,
|
|
83
|
-
format: Annotated[
|
|
84
|
-
OutputFormat,
|
|
85
|
-
typer.Option(
|
|
86
|
-
"-f",
|
|
87
|
-
"--format",
|
|
88
|
-
help="Output format for --emit json: json or msgpack.",
|
|
89
|
-
case_sensitive=False,
|
|
90
|
-
),
|
|
91
|
-
] = OutputFormat.JSON,
|
|
92
83
|
emit: Annotated[
|
|
93
84
|
EmitTarget,
|
|
94
85
|
typer.Option(
|
|
@@ -141,26 +132,31 @@ def main(
|
|
|
141
132
|
),
|
|
142
133
|
] = None,
|
|
143
134
|
analysis_level: Annotated[
|
|
144
|
-
int,
|
|
135
|
+
Optional[int],
|
|
145
136
|
typer.Option(
|
|
146
137
|
"-a",
|
|
147
138
|
"--analysis-level",
|
|
148
|
-
help="Analysis depth: 1=symbol table+Jedi call graph, 2=+
|
|
139
|
+
help="Analysis depth: 1=symbol table+Jedi call graph, 2=+defuse-linker call "
|
|
149
140
|
"graph, 3=+native intraprocedural dataflow (CFG/PDG), "
|
|
150
|
-
"4=+interprocedural SDG (param/summary edges, alias-aware DDG)."
|
|
141
|
+
"4=+interprocedural SDG (param/summary edges, alias-aware DDG). "
|
|
142
|
+
"[default: 1; incompatible with --emit neo4j, which is always "
|
|
143
|
+
"full-depth]",
|
|
151
144
|
min=1,
|
|
152
145
|
max=4,
|
|
146
|
+
show_default="1",
|
|
153
147
|
),
|
|
154
|
-
] =
|
|
148
|
+
] = None,
|
|
155
149
|
graphs: Annotated[
|
|
156
|
-
str,
|
|
150
|
+
Optional[str],
|
|
157
151
|
typer.Option(
|
|
158
152
|
"--graphs",
|
|
159
153
|
help="Level 3+ only: comma-separated program-graph sections to emit "
|
|
160
154
|
"(cfg, dfg, pdg, sdg). Default: cfg,dfg,pdg. `dfg` emits the PDG's data "
|
|
161
|
-
"edges only; `sdg` requires -a 4."
|
|
155
|
+
"edges only; `sdg` requires -a 4. Incompatible with --emit neo4j "
|
|
156
|
+
"(always full-depth).",
|
|
157
|
+
show_default="cfg,dfg,pdg",
|
|
162
158
|
),
|
|
163
|
-
] =
|
|
159
|
+
] = None,
|
|
164
160
|
graph_field_depth: Annotated[
|
|
165
161
|
int,
|
|
166
162
|
typer.Option(
|
|
@@ -222,87 +218,54 @@ def main(
|
|
|
222
218
|
verbosity: Annotated[
|
|
223
219
|
int, typer.Option("-v", count=True, help="Increase verbosity: -v, -vv, -vvv")
|
|
224
220
|
] = 0,
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
typer.Option(
|
|
228
|
-
"--pycg-shard/--no-pycg-shard",
|
|
229
|
-
help=(
|
|
230
|
-
"Shard PyCG call-graph analysis by Python package (level 2 only). "
|
|
231
|
-
"When the project exceeds the 500-file ceiling, PyCG is run "
|
|
232
|
-
"independently per top-level package with cross-package imports "
|
|
233
|
-
"treated as ghost nodes. Without this flag, projects over the "
|
|
234
|
-
"ceiling fall back to Jedi-only edges."
|
|
235
|
-
),
|
|
236
|
-
),
|
|
237
|
-
] = False,
|
|
238
|
-
pycg_shard_ceiling: Annotated[
|
|
239
|
-
int,
|
|
240
|
-
typer.Option(
|
|
241
|
-
"--pycg-shard-ceiling",
|
|
242
|
-
help=(
|
|
243
|
-
"Maximum files per shard when --pycg-shard is active (default 100). "
|
|
244
|
-
"Shards exceeding this limit are skipped; their call edges are "
|
|
245
|
-
"omitted from the call graph (Jedi edges for those packages are "
|
|
246
|
-
"still included). Lower values are safer for packages with deep "
|
|
247
|
-
"class hierarchies or heavy import graphs."
|
|
248
|
-
),
|
|
249
|
-
min=1,
|
|
250
|
-
),
|
|
251
|
-
] = 100,
|
|
252
|
-
pycg_shard_timeout: Annotated[
|
|
253
|
-
int,
|
|
254
|
-
typer.Option(
|
|
255
|
-
"--pycg-shard-timeout",
|
|
256
|
-
help=(
|
|
257
|
-
"Per-shard wall-clock timeout in seconds when --pycg-shard is "
|
|
258
|
-
"active (default 120). A shard that exceeds this limit is skipped "
|
|
259
|
-
"gracefully. PyCG's fixpoint is bimodal: it either converges "
|
|
260
|
-
"quickly or diverges indefinitely, so the timeout acts as a final "
|
|
261
|
-
"safety net after the file-count ceiling. Set to 0 to disable. "
|
|
262
|
-
"POSIX only (macOS / Linux); ignored on Windows."
|
|
263
|
-
),
|
|
264
|
-
min=0,
|
|
265
|
-
),
|
|
266
|
-
] = 120,
|
|
267
|
-
pycg_shard_strategy: Annotated[
|
|
268
|
-
ShardStrategy,
|
|
269
|
-
typer.Option(
|
|
270
|
-
"--pycg-shard-strategy",
|
|
271
|
-
help=(
|
|
272
|
-
"How --pycg-shard groups files (level 2 only). 'jedi' (default) "
|
|
273
|
-
"partitions the Jedi module-dependency graph (SCC + Louvain) so "
|
|
274
|
-
"tightly-coupled modules co-compute and few call edges are "
|
|
275
|
-
"severed between shards; import cycles are never split. "
|
|
276
|
-
"'package' uses the legacy one-shard-per-package-directory "
|
|
277
|
-
"grouping."
|
|
278
|
-
),
|
|
279
|
-
),
|
|
280
|
-
] = ShardStrategy.JEDI,
|
|
281
|
-
pycg_max_iter: Annotated[
|
|
282
|
-
int,
|
|
221
|
+
entrypoint_rules: Annotated[
|
|
222
|
+
Optional[List[Path]],
|
|
283
223
|
typer.Option(
|
|
284
|
-
"--
|
|
285
|
-
help=(
|
|
286
|
-
|
|
287
|
-
"default 50). PyCG iterates until its points-to state stops "
|
|
288
|
-
"changing, but its access-path domain has no convergence bound, "
|
|
289
|
-
"so heavy metaclass/mixin code (e.g. an ORM) can loop with each "
|
|
290
|
-
"pass costing seconds. The cap returns a sound-but-incomplete "
|
|
291
|
-
"call graph instead of looping until the timeout kills it. "
|
|
292
|
-
"Set to -1 for PyCG's unbounded run-to-convergence behaviour."
|
|
293
|
-
),
|
|
294
|
-
min=-1,
|
|
224
|
+
"--entrypoint-rules",
|
|
225
|
+
help="Extra entrypoint rules file (YAML). Repeatable; merges with "
|
|
226
|
+
"the shipped rules. A malformed file is an error.",
|
|
295
227
|
),
|
|
296
|
-
] =
|
|
228
|
+
] = None,
|
|
297
229
|
):
|
|
298
230
|
# Determinism: pin the interpreter hash seed before any analysis (no-op
|
|
299
231
|
# when PYTHONHASHSEED is already set; --version exits before this).
|
|
300
232
|
_pin_hash_seed()
|
|
301
233
|
|
|
302
234
|
# Flag validation (strict: unrecognized values error out, never fall back).
|
|
303
|
-
|
|
235
|
+
# -a and --graphs use None sentinels so an explicitly-passed flag is
|
|
236
|
+
# distinguishable from the default (#119).
|
|
237
|
+
explicit_level = analysis_level is not None
|
|
238
|
+
explicit_graphs = graphs is not None
|
|
239
|
+
|
|
240
|
+
# Neo4j is always full-depth (#119): the graph carries every level's
|
|
241
|
+
# facts, so depth/section selectors cannot be combined with it — reject
|
|
242
|
+
# explicitly-passed flags and force level 4 with every graph section.
|
|
304
243
|
from codeanalyzer.dataflow.builder import VALID_GRAPHS
|
|
305
244
|
|
|
245
|
+
if emit == EmitTarget.NEO4J:
|
|
246
|
+
explicit = [
|
|
247
|
+
flag
|
|
248
|
+
for flag, was_explicit in (
|
|
249
|
+
("-a/--analysis-level", explicit_level),
|
|
250
|
+
("--graphs", explicit_graphs),
|
|
251
|
+
)
|
|
252
|
+
if was_explicit
|
|
253
|
+
]
|
|
254
|
+
if explicit:
|
|
255
|
+
logger.error(
|
|
256
|
+
"--emit neo4j is always full-depth (level 4, all graph "
|
|
257
|
+
f"sections); {' and '.join(explicit)} cannot be combined with it."
|
|
258
|
+
)
|
|
259
|
+
raise typer.Exit(code=2)
|
|
260
|
+
analysis_level = 4
|
|
261
|
+
graphs = ",".join(VALID_GRAPHS)
|
|
262
|
+
|
|
263
|
+
if analysis_level is None:
|
|
264
|
+
analysis_level = 1
|
|
265
|
+
if graphs is None:
|
|
266
|
+
graphs = "cfg,dfg,pdg"
|
|
267
|
+
selected_graphs = [g.strip() for g in graphs.split(",") if g.strip()]
|
|
268
|
+
|
|
306
269
|
unknown_graphs = [g for g in selected_graphs if g not in VALID_GRAPHS]
|
|
307
270
|
if unknown_graphs:
|
|
308
271
|
logger.error(
|
|
@@ -316,7 +279,7 @@ def main(
|
|
|
316
279
|
if "sdg" in selected_graphs and analysis_level < 4:
|
|
317
280
|
logger.error("--graphs sdg requires -a 4 (interprocedural SDG).")
|
|
318
281
|
raise typer.Exit(code=2)
|
|
319
|
-
if analysis_level < 3 and
|
|
282
|
+
if analysis_level < 3 and explicit_graphs:
|
|
320
283
|
logger.error("--graphs is a level-3 option; pass -a 3 to emit program graphs.")
|
|
321
284
|
raise typer.Exit(code=2)
|
|
322
285
|
if analysis_level < 3 and graph_field_depth != 3:
|
|
@@ -326,7 +289,7 @@ def main(
|
|
|
326
289
|
options = AnalysisOptions(
|
|
327
290
|
input=input,
|
|
328
291
|
output=output,
|
|
329
|
-
|
|
292
|
+
|
|
330
293
|
emit=emit,
|
|
331
294
|
app_name=app_name,
|
|
332
295
|
neo4j_uri=neo4j_uri,
|
|
@@ -344,15 +307,25 @@ def main(
|
|
|
344
307
|
cache_dir=cache_dir,
|
|
345
308
|
clear_cache=clear_cache,
|
|
346
309
|
verbosity=verbosity,
|
|
347
|
-
|
|
348
|
-
pycg_shard_ceiling=pycg_shard_ceiling,
|
|
349
|
-
pycg_shard_timeout=pycg_shard_timeout,
|
|
350
|
-
pycg_shard_strategy=pycg_shard_strategy,
|
|
351
|
-
pycg_max_iter=pycg_max_iter,
|
|
310
|
+
entrypoint_rules=tuple(entrypoint_rules or ()),
|
|
352
311
|
)
|
|
353
312
|
|
|
354
313
|
_set_log_level(options.verbosity)
|
|
355
314
|
|
|
315
|
+
# Entrypoint rules are configuration, validated before any analysis work
|
|
316
|
+
# starts (#122 review) -- a typo must fail in milliseconds, not after the
|
|
317
|
+
# symbol table, venv build, Jedi and the defuse linker have all run. `detect_entrypoints`
|
|
318
|
+
# loads the rules again at its own call site; that second load is cheap
|
|
319
|
+
# and keeps the entrypoints pipeline self-contained.
|
|
320
|
+
if options.entrypoint_rules:
|
|
321
|
+
from codeanalyzer.entrypoints.rules import RulesError, load_rules
|
|
322
|
+
|
|
323
|
+
try:
|
|
324
|
+
load_rules(options.entrypoint_rules)
|
|
325
|
+
except RulesError as exc:
|
|
326
|
+
logger.error(f"Invalid --entrypoint-rules: {exc}")
|
|
327
|
+
raise typer.Exit(code=1)
|
|
328
|
+
|
|
356
329
|
# The schema contract is a static artifact — no project analysis required.
|
|
357
330
|
if options.emit == EmitTarget.SCHEMA:
|
|
358
331
|
from codeanalyzer.neo4j.emit import emit_schema
|
|
@@ -392,37 +365,36 @@ def main(
|
|
|
392
365
|
|
|
393
366
|
emit_neo4j(artifacts, options)
|
|
394
367
|
elif options.output is None:
|
|
395
|
-
print(
|
|
368
|
+
print(
|
|
369
|
+
json.dumps(
|
|
370
|
+
strip_internal_only(
|
|
371
|
+
model_dump(artifacts, mode="json", exclude_none=True)
|
|
372
|
+
)
|
|
373
|
+
)
|
|
374
|
+
)
|
|
396
375
|
else:
|
|
397
376
|
options.output.mkdir(parents=True, exist_ok=True)
|
|
398
|
-
_write_output(artifacts, options.output
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
def _write_output(artifacts, output_dir: Path
|
|
402
|
-
"""Write
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
msgpack_data = artifacts.to_msgpack_bytes()
|
|
414
|
-
with output_file.open("wb") as f:
|
|
415
|
-
f.write(msgpack_data)
|
|
416
|
-
logger.info(f"Analysis saved to {output_file}")
|
|
417
|
-
logger.info(
|
|
418
|
-
f"Compression ratio: {artifacts.get_compression_ratio():.1%} of JSON size"
|
|
419
|
-
)
|
|
377
|
+
_write_output(artifacts, options.output)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _write_output(artifacts, output_dir: Path):
|
|
381
|
+
"""Write analysis.json (the single wire format since #118)."""
|
|
382
|
+
output_file = output_dir / "analysis.json"
|
|
383
|
+
# Use Pydantic's model_dump_json() for compact output
|
|
384
|
+
# Strip internal-only fields here rather than with a field-level Pydantic
|
|
385
|
+
# `exclude`: the analysis cache shares the serializer and must keep them.
|
|
386
|
+
json_str = json.dumps(
|
|
387
|
+
strip_internal_only(model_dump(artifacts, mode="json", exclude_none=True))
|
|
388
|
+
)
|
|
389
|
+
with output_file.open("w") as f:
|
|
390
|
+
f.write(json_str)
|
|
391
|
+
logger.info(f"Analysis saved to {output_file}")
|
|
420
392
|
|
|
421
393
|
|
|
422
394
|
app = typer.Typer(
|
|
423
395
|
callback=main,
|
|
424
396
|
name="canpy",
|
|
425
|
-
help="Static Analysis on Python source code using Jedi
|
|
397
|
+
help="Static Analysis on Python source code using Jedi and Tree sitter.",
|
|
426
398
|
invoke_without_command=True,
|
|
427
399
|
no_args_is_help=True,
|
|
428
400
|
add_completion=False,
|
codeanalyzer/core.py
CHANGED
|
@@ -29,7 +29,7 @@ from codeanalyzer.semantic_analysis.call_graph import (
|
|
|
29
29
|
merge_edges,
|
|
30
30
|
resolve_unresolved_constructors,
|
|
31
31
|
)
|
|
32
|
-
from codeanalyzer.semantic_analysis.
|
|
32
|
+
from codeanalyzer.semantic_analysis.defuse_linker import defuse_linker_edges
|
|
33
33
|
from codeanalyzer.syntactic_analysis.exceptions import SymbolTableBuilderRayError
|
|
34
34
|
from codeanalyzer.syntactic_analysis.import_resolver import resolve_imports
|
|
35
35
|
from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
|
|
@@ -41,7 +41,7 @@ def _ensure_ray() -> None:
|
|
|
41
41
|
"""Initialize Ray with the driver's pinned hash seed in the workers.
|
|
42
42
|
|
|
43
43
|
An implicit auto-init would not carry PYTHONHASHSEED into worker
|
|
44
|
-
interpreters, so
|
|
44
|
+
interpreters, so Jedi inference in Ray workers would run with random
|
|
45
45
|
set-iteration order and the emitted edges vary run to run (issue #99)."""
|
|
46
46
|
if not ray.is_initialized():
|
|
47
47
|
ray.init(
|
|
@@ -592,14 +592,20 @@ class Codeanalyzer:
|
|
|
592
592
|
logger.info("✅ Jedi: %d edges in %.1fs", len(call_graph), time.perf_counter() - t0_jedi)
|
|
593
593
|
|
|
594
594
|
if self.analysis_level >= 2:
|
|
595
|
-
# Level 2:
|
|
596
|
-
#
|
|
597
|
-
|
|
598
|
-
|
|
595
|
+
# Level 2: the defuse linker backfills call sites Jedi could not
|
|
596
|
+
# resolve, from local def-use chains and module-scope bindings
|
|
597
|
+
# (docs/design/specs/2026-08-25-defuse-linker-call-graph-design.md).
|
|
598
|
+
t0_linker = time.perf_counter()
|
|
599
|
+
defuse_edges, defuse_resolutions = defuse_linker_edges(symbol_table)
|
|
600
|
+
call_graph = merge_edges(call_graph, defuse_edges)
|
|
601
|
+
logger.info(
|
|
602
|
+
"✅ defuse linker: %d edges in %.1fs",
|
|
603
|
+
len(defuse_edges), time.perf_counter() - t0_linker,
|
|
604
|
+
)
|
|
599
605
|
|
|
600
606
|
call_graph = filter_external_edges(call_graph, symbol_table)
|
|
601
|
-
# Canonical edge order: backend iteration order (
|
|
602
|
-
#
|
|
607
|
+
# Canonical edge order: backend iteration order (Counter insertion,
|
|
608
|
+
# dict iteration) is not a contract — sort so identical edge SETS always
|
|
603
609
|
# serialize identically (issue #99 determinism gate), and so the
|
|
604
610
|
# external-symbol homing below assigns ids in a stable order.
|
|
605
611
|
call_graph.sort(key=lambda e: (e.src, e.dst))
|
|
@@ -633,9 +639,15 @@ class Codeanalyzer:
|
|
|
633
639
|
app.external_symbols = self._home_external_symbols(app, app.id, sig_to_id)
|
|
634
640
|
populate_l1_body(app)
|
|
635
641
|
if self.analysis_level >= 2:
|
|
636
|
-
backfill_callees(app, sig_to_id)
|
|
642
|
+
backfill_callees(app, sig_to_id, resolutions=defuse_resolutions)
|
|
637
643
|
reidentify_call_graph(app, sig_to_id)
|
|
638
644
|
|
|
645
|
+
# Entrypoints: a post-pass over the built L1 tree (#27). Runs at every
|
|
646
|
+
# level -- entrypoints are L1 data and must not vary with -a.
|
|
647
|
+
from codeanalyzer.entrypoints import detect_entrypoints
|
|
648
|
+
|
|
649
|
+
detect_entrypoints(app, self.project_dir, self.options.entrypoint_rules)
|
|
650
|
+
|
|
639
651
|
# L3: intraprocedural dataflow (CFG/CDG/DDG) emitted onto the v2 tree.
|
|
640
652
|
if self.analysis_level >= 3:
|
|
641
653
|
from codeanalyzer.dataflow.builder import (
|
|
@@ -936,39 +948,3 @@ class Codeanalyzer:
|
|
|
936
948
|
len(symbol_table), time.perf_counter() - t0_st,
|
|
937
949
|
)
|
|
938
950
|
return symbol_table
|
|
939
|
-
|
|
940
|
-
def _get_pycg_call_graph(
|
|
941
|
-
self,
|
|
942
|
-
symbol_table: Dict[str, PyModule],
|
|
943
|
-
jedi_edges: List[PyCallEdge],
|
|
944
|
-
) -> List[PyCallEdge]:
|
|
945
|
-
"""Build PyCG-resolved call edges.
|
|
946
|
-
|
|
947
|
-
Runs PyCG's iterative name-pointer analysis over the whole project
|
|
948
|
-
and returns edges with ``prov=["pycg"]``. Falls back to an
|
|
949
|
-
empty list and logs a warning on any failure so the caller can
|
|
950
|
-
continue with Jedi-only edges.
|
|
951
|
-
|
|
952
|
-
*jedi_edges* are the level-1 call edges; under the ``jedi`` shard
|
|
953
|
-
strategy they drive coupling-aware partitioning (see
|
|
954
|
-
:func:`shard_planner.plan_shards`).
|
|
955
|
-
"""
|
|
956
|
-
try:
|
|
957
|
-
pycg = PyCG(
|
|
958
|
-
self.project_dir,
|
|
959
|
-
skip_tests=self.skip_tests,
|
|
960
|
-
shard=self.options.pycg_shard,
|
|
961
|
-
shard_ceiling=self.options.pycg_shard_ceiling,
|
|
962
|
-
shard_timeout=self.options.pycg_shard_timeout,
|
|
963
|
-
shard_strategy=self.options.pycg_shard_strategy,
|
|
964
|
-
max_iter=self.options.pycg_max_iter,
|
|
965
|
-
using_ray=self.using_ray,
|
|
966
|
-
)
|
|
967
|
-
return pycg.build_call_graph_edges(symbol_table, jedi_edges=jedi_edges)
|
|
968
|
-
except PyCGExceptions.PyCGImportError as exc:
|
|
969
|
-
logger.warning(f"PyCG not installed — level 2 edges will be Jedi-only: {exc}")
|
|
970
|
-
return []
|
|
971
|
-
except PyCGExceptions.PyCGAnalysisError as exc:
|
|
972
|
-
logger.warning(f"PyCG analysis failed — level 2 edges will be Jedi-only: {exc}")
|
|
973
|
-
logger.debug("PyCG full traceback:", exc_info=True)
|
|
974
|
-
return []
|
|
@@ -245,15 +245,37 @@ def _names_loaded(node: ast.AST) -> Set[str]:
|
|
|
245
245
|
return out
|
|
246
246
|
|
|
247
247
|
|
|
248
|
-
|
|
248
|
+
#: Jedi resolves every spelling of the builtin -- ``@staticmethod``,
|
|
249
|
+
#: ``@builtins.staticmethod``, ``from builtins import staticmethod as sm`` --
|
|
250
|
+
#: to this one name (#135).
|
|
251
|
+
_STATICMETHOD_QUALIFIED = "builtins.staticmethod"
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def build_scope(
|
|
255
|
+
func: ast.AST,
|
|
256
|
+
enclosing_locals: Set[str],
|
|
257
|
+
decorator_names: Optional[Set[str]] = None,
|
|
258
|
+
) -> FunctionScope:
|
|
249
259
|
"""Classify every base name the callable touches. ``enclosing_locals`` is
|
|
250
260
|
the union of locals/params of all enclosing callables (for capture vs
|
|
251
|
-
global disambiguation).
|
|
261
|
+
global disambiguation).
|
|
262
|
+
|
|
263
|
+
``decorator_names`` are the callable's Jedi-resolved decorator
|
|
264
|
+
``qualified_name``s. When supplied, staticmethod detection is by identity,
|
|
265
|
+
so a dotted or aliased spelling is recognised (#135). When omitted -- a
|
|
266
|
+
caller with no resolved records -- it falls back to matching the written
|
|
267
|
+
source, which only recognises the bare ``@staticmethod``.
|
|
268
|
+
"""
|
|
252
269
|
params = _param_names(func)
|
|
253
270
|
scope = FunctionScope(params=params)
|
|
254
271
|
if params and isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
255
|
-
|
|
256
|
-
|
|
272
|
+
if decorator_names is None:
|
|
273
|
+
is_static = "staticmethod" in {
|
|
274
|
+
ast.unparse(d) for d in func.decorator_list
|
|
275
|
+
}
|
|
276
|
+
else:
|
|
277
|
+
is_static = _STATICMETHOD_QUALIFIED in decorator_names
|
|
278
|
+
if params[0] in ("self", "cls") and not is_static:
|
|
257
279
|
scope.self_name = params[0]
|
|
258
280
|
scope.globals_ = _declared(func, ast.Global)
|
|
259
281
|
nonlocals = _declared(func, ast.Nonlocal)
|
codeanalyzer/dataflow/builder.py
CHANGED
|
@@ -186,6 +186,13 @@ def build_function_pdgs(
|
|
|
186
186
|
oracle=oracle,
|
|
187
187
|
k=k,
|
|
188
188
|
global_qualifier=module.module_name,
|
|
189
|
+
# Resolved decorator names, so staticmethod detection works for a
|
|
190
|
+
# dotted or aliased spelling and not just bare `@staticmethod` (#135).
|
|
191
|
+
decorator_names={
|
|
192
|
+
d.qualified_name
|
|
193
|
+
for d in (pycallable.decorators or [])
|
|
194
|
+
if d.qualified_name
|
|
195
|
+
},
|
|
189
196
|
)
|
|
190
197
|
infos[pycallable.signature] = FunctionInfo(
|
|
191
198
|
signature=pycallable.signature, pdg=pdg, oracle=oracle
|
|
@@ -10,7 +10,7 @@ Two forms per node:
|
|
|
10
10
|
an L1 body node and its coinciding CFG node land on the same key and L1 ⊆ L3
|
|
11
11
|
holds.
|
|
12
12
|
* **global** — ``"<callable can:// id>@<local>"``, the fully addressable id for
|
|
13
|
-
cross-callable references and the Neo4j
|
|
13
|
+
cross-callable references and the Neo4j PyBodyNode keys (a later task).
|
|
14
14
|
"""
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
from collections import defaultdict
|
codeanalyzer/dataflow/pdg.py
CHANGED
|
@@ -66,10 +66,15 @@ def build_pdg(
|
|
|
66
66
|
oracle: TypeBasedAliasOracle,
|
|
67
67
|
k: int = 3,
|
|
68
68
|
global_qualifier: Optional[str] = None,
|
|
69
|
+
decorator_names: Optional[Set[str]] = None,
|
|
69
70
|
) -> FunctionPDG:
|
|
70
|
-
"""CFG → dominance → def-use → PDG for one callable.
|
|
71
|
+
"""CFG → dominance → def-use → PDG for one callable.
|
|
72
|
+
|
|
73
|
+
``decorator_names`` are the callable's resolved decorator qualified names,
|
|
74
|
+
used for staticmethod detection by identity rather than spelling (#135).
|
|
75
|
+
"""
|
|
71
76
|
cfg = build_cfg(func)
|
|
72
|
-
scope = build_scope(func, enclosing_locals)
|
|
77
|
+
scope = build_scope(func, enclosing_locals, decorator_names=decorator_names)
|
|
73
78
|
facts = statement_facts(cfg, func, scope, k, global_qualifier)
|
|
74
79
|
|
|
75
80
|
edges: List[PDGEdge] = [
|
codeanalyzer/dataflow/scc.py
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"""Stage 5b of the level-3 dataflow ladder: SCC condensation of the call graph.
|
|
18
18
|
|
|
19
19
|
The call graph is a frozen oracle (level-1 Jedi edges, provenance-merged with
|
|
20
|
-
level-2
|
|
20
|
+
level-2 resolvers); Tarjan condenses it into strongly connected
|
|
21
21
|
components, and the condensation DAG in reverse topological order is the
|
|
22
22
|
bottom-up processing schedule for summary composition — callees before
|
|
23
23
|
callers, one monotone fixpoint per SCC (mutual recursion).
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Stage 0: which frameworks is this project actually using? (#27)
|
|
2
|
+
|
|
3
|
+
Gates every later stage, so a project without Celery never pays for Celery
|
|
4
|
+
rules and cannot false-positive on a locally-defined ``shared_task``. A
|
|
5
|
+
package counts as present if first-party source imports it OR the dependency
|
|
6
|
+
manifest names it -- either is sufficient, since an import may be dynamic.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Optional, Set
|
|
13
|
+
|
|
14
|
+
from codeanalyzer.entrypoints.rules import RuleSet
|
|
15
|
+
from codeanalyzer.schema.py_schema import PyApplication
|
|
16
|
+
|
|
17
|
+
_REQ = re.compile(r"^\s*['\"]?([A-Za-z0-9_.\-]+)")
|
|
18
|
+
_DEPS_START = re.compile(r"dependencies\s*=\s*\[")
|
|
19
|
+
_TABLE_HEADER = re.compile(r"(?m)^[ \t]*\[")
|
|
20
|
+
_PKG = re.compile(r"['\"]([A-Za-z0-9][A-Za-z0-9_.\-]*)")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def detected_frameworks(app: PyApplication, project_dir: Path, rules: RuleSet) -> Set[str]:
|
|
24
|
+
# `present` (imports, manifest names) and `detect:` values are both
|
|
25
|
+
# lowercased before comparison -- manifest names were already lowercased
|
|
26
|
+
# (PyPI/pip is case-insensitive) but imports and `detect:` were not, so
|
|
27
|
+
# a `detect: [Flask]` user rule silently never matched a `flask` import.
|
|
28
|
+
present = _imported_packages(app) | _manifest_packages(project_dir)
|
|
29
|
+
return {
|
|
30
|
+
name
|
|
31
|
+
for name, fw in rules.frameworks.items()
|
|
32
|
+
if any(pkg.lower() in present for pkg in (fw.detect or [name]))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _imported_packages(app: PyApplication) -> Set[str]:
|
|
37
|
+
out: Set[str] = set()
|
|
38
|
+
for mod in app.symbol_table.values():
|
|
39
|
+
for imp in mod.imports or []:
|
|
40
|
+
# `from flask import Flask` puts the package in `module`, not `name`.
|
|
41
|
+
# Prefer `module`; fall back to `name` for a bare `import flask`.
|
|
42
|
+
spelling = (getattr(imp, "module", "") or getattr(imp, "name", "") or "")
|
|
43
|
+
spelling = spelling.lstrip(".")
|
|
44
|
+
if spelling:
|
|
45
|
+
out.add(spelling.split(".", 1)[0].lower())
|
|
46
|
+
return out
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _manifest_packages(project_dir: Path) -> Set[str]:
|
|
50
|
+
out: Set[str] = set()
|
|
51
|
+
pyproject = project_dir / "pyproject.toml"
|
|
52
|
+
if pyproject.exists():
|
|
53
|
+
# PEP 621 `[project] dependencies = [...]` -- single- or multi-line,
|
|
54
|
+
# possibly containing nested `[...]` extras (`celery[redis]`).
|
|
55
|
+
span = _deps_array_span(_strip_comments(pyproject.read_text()))
|
|
56
|
+
if span is not None:
|
|
57
|
+
for pm in _PKG.finditer(span):
|
|
58
|
+
out.add(pm.group(1).split("[", 1)[0].lower())
|
|
59
|
+
requirements = project_dir / "requirements.txt"
|
|
60
|
+
if requirements.exists():
|
|
61
|
+
for line in requirements.read_text().splitlines():
|
|
62
|
+
m = _REQ.match(line)
|
|
63
|
+
if m:
|
|
64
|
+
out.add(m.group(1).split("[", 1)[0].lower())
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _strip_comments(text: str) -> str:
|
|
69
|
+
"""Drop everything from an unquoted ``#`` to end of line.
|
|
70
|
+
|
|
71
|
+
# ponytail: quote tracking resets each line, so a `#` inside a
|
|
72
|
+
# triple-quoted string spanning lines could be mis-stripped. TOML
|
|
73
|
+
# dependency arrays don't use those in practice; revisit if they do.
|
|
74
|
+
"""
|
|
75
|
+
out_lines = []
|
|
76
|
+
for line in text.splitlines():
|
|
77
|
+
in_str = None
|
|
78
|
+
cut = len(line)
|
|
79
|
+
for i, ch in enumerate(line):
|
|
80
|
+
if in_str:
|
|
81
|
+
if ch == in_str:
|
|
82
|
+
in_str = None
|
|
83
|
+
elif ch in ("'", '"'):
|
|
84
|
+
in_str = ch
|
|
85
|
+
elif ch == "#":
|
|
86
|
+
cut = i
|
|
87
|
+
break
|
|
88
|
+
out_lines.append(line[:cut])
|
|
89
|
+
return "\n".join(out_lines)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _deps_array_span(text: str) -> Optional[str]:
|
|
93
|
+
"""Return the contents between the `dependencies = [` and its matching
|
|
94
|
+
`]`, counting bracket depth so a nested `[...]` (extras, e.g.
|
|
95
|
+
`celery[redis]`) doesn't close the span early.
|
|
96
|
+
|
|
97
|
+
Bounded by the next TOML table header (a `[` starting a line): if the
|
|
98
|
+
array never closes before then, it's unterminated (truncated/corrupt
|
|
99
|
+
file) and this returns None rather than harvesting quoted strings out
|
|
100
|
+
of whatever table follows.
|
|
101
|
+
"""
|
|
102
|
+
m = _DEPS_START.search(text)
|
|
103
|
+
if not m:
|
|
104
|
+
return None
|
|
105
|
+
boundary = _TABLE_HEADER.search(text, m.end())
|
|
106
|
+
limit = boundary.start() if boundary else len(text)
|
|
107
|
+
depth = 1
|
|
108
|
+
in_str = None
|
|
109
|
+
i = m.end()
|
|
110
|
+
while i < limit and depth > 0:
|
|
111
|
+
ch = text[i]
|
|
112
|
+
if in_str:
|
|
113
|
+
if ch == in_str:
|
|
114
|
+
in_str = None
|
|
115
|
+
elif ch in ("'", '"'):
|
|
116
|
+
in_str = ch
|
|
117
|
+
elif ch == "[":
|
|
118
|
+
depth += 1
|
|
119
|
+
elif ch == "]":
|
|
120
|
+
depth -= 1
|
|
121
|
+
i += 1
|
|
122
|
+
if depth != 0:
|
|
123
|
+
return None
|
|
124
|
+
return text[m.end() : i - 1]
|