codeanalyzer-python 1.1.1__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +119 -118
- codeanalyzer/artifacts/__init__.py +20 -0
- codeanalyzer/artifacts/config_keys.py +588 -0
- codeanalyzer/artifacts/config_use.py +597 -0
- codeanalyzer/artifacts/config_use_rules.yml +58 -0
- codeanalyzer/artifacts/dependencies.py +237 -0
- codeanalyzer/artifacts/discovery.py +167 -0
- codeanalyzer/artifacts/parsers.py +248 -0
- codeanalyzer/core.py +112 -45
- codeanalyzer/dataflow/access_paths.py +26 -4
- codeanalyzer/dataflow/builder.py +22 -1
- codeanalyzer/dataflow/identity.py +1 -1
- codeanalyzer/dataflow/pdg.py +7 -2
- codeanalyzer/dataflow/scc.py +1 -1
- codeanalyzer/entrypoints/__init__.py +3 -0
- codeanalyzer/entrypoints/detect.py +124 -0
- codeanalyzer/entrypoints/matching.py +182 -0
- codeanalyzer/entrypoints/pipeline.py +131 -0
- codeanalyzer/entrypoints/rules.py +159 -0
- codeanalyzer/entrypoints/rules.yml +88 -0
- codeanalyzer/neo4j/bolt.py +1 -1
- codeanalyzer/neo4j/project.py +277 -60
- codeanalyzer/neo4j/schema.py +92 -34
- codeanalyzer/options/__init__.py +2 -2
- codeanalyzer/options/options.py +7 -26
- codeanalyzer/schema/__init__.py +48 -0
- codeanalyzer/schema/ids.py +21 -0
- codeanalyzer/schema/l1_body.py +11 -1
- codeanalyzer/schema/l2_callees.py +29 -13
- codeanalyzer/schema/py_schema.py +213 -103
- codeanalyzer/semantic_analysis/call_graph.py +20 -4
- codeanalyzer/semantic_analysis/defuse_linker.py +1499 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +99 -3
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.3.0.dist-info}/METADATA +143 -164
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.3.0.dist-info}/RECORD +39 -31
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.3.0.dist-info}/WHEEL +1 -1
- codeanalyzer/config/__init__.py +0 -3
- codeanalyzer/config/config.py +0 -8
- codeanalyzer/semantic_analysis/pycg/__init__.py +0 -20
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +0 -1115
- codeanalyzer/semantic_analysis/pycg/pycg_exceptions.py +0 -23
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +0 -401
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.3.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.1.1.dist-info → codeanalyzer_python-1.3.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/core.py
CHANGED
|
@@ -29,7 +29,7 @@ from codeanalyzer.semantic_analysis.call_graph import (
|
|
|
29
29
|
merge_edges,
|
|
30
30
|
resolve_unresolved_constructors,
|
|
31
31
|
)
|
|
32
|
-
from codeanalyzer.semantic_analysis.
|
|
32
|
+
from codeanalyzer.semantic_analysis.defuse_linker import defuse_linker_edges
|
|
33
33
|
from codeanalyzer.syntactic_analysis.exceptions import SymbolTableBuilderRayError
|
|
34
34
|
from codeanalyzer.syntactic_analysis.import_resolver import resolve_imports
|
|
35
35
|
from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
|
|
@@ -37,11 +37,26 @@ from codeanalyzer.utils import ProgressBar
|
|
|
37
37
|
from codeanalyzer.options import AnalysisOptions
|
|
38
38
|
from codeanalyzer.provenance import analyzer_info, repository_info
|
|
39
39
|
|
|
40
|
+
def _artifact_full_text(project_dir: Path, path: str, art) -> str:
|
|
41
|
+
"""Mirrors ``artifacts.dependencies._full_text`` verbatim (not imported
|
|
42
|
+
-- that name is module-private to ``dependencies.py``): config-key
|
|
43
|
+
extraction (#152) must never depend on the stored ``source`` -- capped
|
|
44
|
+
by ``text_max_bytes`` and emptied by ``capture_text=False`` (payload-size
|
|
45
|
+
controls, not extraction controls). Read the real file fresh instead;
|
|
46
|
+
fall back to ``art.source`` only if it's gone (e.g. a synthetic artifact
|
|
47
|
+
in a unit test, or the file vanished mid-run). Keep the two in sync if
|
|
48
|
+
this logic changes."""
|
|
49
|
+
try:
|
|
50
|
+
return (project_dir / path).read_bytes().decode("utf-8")
|
|
51
|
+
except (OSError, UnicodeDecodeError):
|
|
52
|
+
return art.source
|
|
53
|
+
|
|
54
|
+
|
|
40
55
|
def _ensure_ray() -> None:
|
|
41
56
|
"""Initialize Ray with the driver's pinned hash seed in the workers.
|
|
42
57
|
|
|
43
58
|
An implicit auto-init would not carry PYTHONHASHSEED into worker
|
|
44
|
-
interpreters, so
|
|
59
|
+
interpreters, so Jedi inference in Ray workers would run with random
|
|
45
60
|
set-iteration order and the emitted edges vary run to run (issue #99)."""
|
|
46
61
|
if not ray.is_initialized():
|
|
47
62
|
ray.init(
|
|
@@ -592,14 +607,20 @@ class Codeanalyzer:
|
|
|
592
607
|
logger.info("✅ Jedi: %d edges in %.1fs", len(call_graph), time.perf_counter() - t0_jedi)
|
|
593
608
|
|
|
594
609
|
if self.analysis_level >= 2:
|
|
595
|
-
# Level 2:
|
|
596
|
-
#
|
|
597
|
-
|
|
598
|
-
|
|
610
|
+
# Level 2: the defuse linker backfills call sites Jedi could not
|
|
611
|
+
# resolve, from local def-use chains and module-scope bindings
|
|
612
|
+
# (docs/design/specs/2026-08-25-defuse-linker-call-graph-design.md).
|
|
613
|
+
t0_linker = time.perf_counter()
|
|
614
|
+
defuse_edges, defuse_resolutions = defuse_linker_edges(symbol_table)
|
|
615
|
+
call_graph = merge_edges(call_graph, defuse_edges)
|
|
616
|
+
logger.info(
|
|
617
|
+
"✅ defuse linker: %d edges in %.1fs",
|
|
618
|
+
len(defuse_edges), time.perf_counter() - t0_linker,
|
|
619
|
+
)
|
|
599
620
|
|
|
600
621
|
call_graph = filter_external_edges(call_graph, symbol_table)
|
|
601
|
-
# Canonical edge order: backend iteration order (
|
|
602
|
-
#
|
|
622
|
+
# Canonical edge order: backend iteration order (Counter insertion,
|
|
623
|
+
# dict iteration) is not a contract — sort so identical edge SETS always
|
|
603
624
|
# serialize identically (issue #99 determinism gate), and so the
|
|
604
625
|
# external-symbol homing below assigns ids in a stable order.
|
|
605
626
|
call_graph.sort(key=lambda e: (e.src, e.dst))
|
|
@@ -633,9 +654,68 @@ class Codeanalyzer:
|
|
|
633
654
|
app.external_symbols = self._home_external_symbols(app, app.id, sig_to_id)
|
|
634
655
|
populate_l1_body(app)
|
|
635
656
|
if self.analysis_level >= 2:
|
|
636
|
-
backfill_callees(app, sig_to_id)
|
|
657
|
+
backfill_callees(app, sig_to_id, resolutions=defuse_resolutions)
|
|
637
658
|
reidentify_call_graph(app, sig_to_id)
|
|
638
659
|
|
|
660
|
+
# Entrypoints: a post-pass over the built L1 tree (#27). Runs at every
|
|
661
|
+
# level -- entrypoints are L1 data and must not vary with -a.
|
|
662
|
+
from codeanalyzer.entrypoints import detect_entrypoints
|
|
663
|
+
|
|
664
|
+
detect_entrypoints(app, self.project_dir, self.options.entrypoint_rules)
|
|
665
|
+
|
|
666
|
+
# Artifacts + dependencies: L1 data, every level, never varies with -a
|
|
667
|
+
# (spec 2026-08-27). Deterministic by default; venv probing is opt-in.
|
|
668
|
+
from codeanalyzer.artifacts import (
|
|
669
|
+
build_dependency_view, discover_artifacts, extract_config_keys,
|
|
670
|
+
is_config_eligible,
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
app.artifacts = discover_artifacts(
|
|
674
|
+
self.project_dir, app_name,
|
|
675
|
+
capture_text=self.options.artifact_text,
|
|
676
|
+
text_max_bytes=self.options.artifact_text_max_bytes,
|
|
677
|
+
)
|
|
678
|
+
app.dependencies, app.unresolved_imports = build_dependency_view(
|
|
679
|
+
app.artifacts,
|
|
680
|
+
app.symbol_table,
|
|
681
|
+
self.project_dir,
|
|
682
|
+
self.virtualenv if self.options.resolve_installed else None,
|
|
683
|
+
self.options.resolve_installed,
|
|
684
|
+
)
|
|
685
|
+
|
|
686
|
+
# Config keys (#152): L1 data, layered onto the same artifacts, every
|
|
687
|
+
# level. Namespace-eligible artifacts only (env-family by basename,
|
|
688
|
+
# else by format). `extraction` combines with any prior
|
|
689
|
+
# dependency-manifest pass on the same artifact: a parse failure here
|
|
690
|
+
# always downgrades to "partial" (never drops the artifact); a clean
|
|
691
|
+
# parse upgrades an untouched "none" to "full", but never overwrites
|
|
692
|
+
# an existing "partial" (e.g. from a failed dependency-manifest parse
|
|
693
|
+
# on the same artifact) -- a successful pass here must not silently
|
|
694
|
+
# erase an unrelated failure already recorded on the artifact.
|
|
695
|
+
for path in sorted(app.artifacts):
|
|
696
|
+
art = app.artifacts[path]
|
|
697
|
+
if not is_config_eligible(art):
|
|
698
|
+
continue
|
|
699
|
+
full_text = _artifact_full_text(self.project_dir, path, art)
|
|
700
|
+
keys, ok = extract_config_keys(art, full_text, self.options.artifact_text)
|
|
701
|
+
art.config_keys = keys
|
|
702
|
+
if not ok:
|
|
703
|
+
art.extraction = "partial"
|
|
704
|
+
elif art.extraction == "none":
|
|
705
|
+
art.extraction = "full"
|
|
706
|
+
|
|
707
|
+
# config_use (#162) detection: needs callees resolved (backfill_
|
|
708
|
+
# callees above, gated the same >= 2) and config_keys just extracted
|
|
709
|
+
# above, so this runs after both rather than immediately following
|
|
710
|
+
# the callee backfill call itself. Resolution is deferred past the
|
|
711
|
+
# L3/L4 blocks below (the dataflow tiers need DDG and call_graph/
|
|
712
|
+
# caller-argument substrate those blocks populate) and gated by
|
|
713
|
+
# level there, literal tier included.
|
|
714
|
+
if self.analysis_level >= 2:
|
|
715
|
+
from codeanalyzer.artifacts import detect_config_reads
|
|
716
|
+
|
|
717
|
+
config_reads = detect_config_reads(app)
|
|
718
|
+
|
|
639
719
|
# L3: intraprocedural dataflow (CFG/CDG/DDG) emitted onto the v2 tree.
|
|
640
720
|
if self.analysis_level >= 3:
|
|
641
721
|
from codeanalyzer.dataflow.builder import (
|
|
@@ -676,6 +756,29 @@ class Codeanalyzer:
|
|
|
676
756
|
# ``infos`` are the syntactic (L3) PDGs from the >=3 block above.
|
|
677
757
|
emit_ddg_pointsto_delta(app, infos, ir, sig_to_id)
|
|
678
758
|
|
|
759
|
+
# config_use (#162) resolution: literal tier always runs at >= 2;
|
|
760
|
+
# the dataflow tiers widen the set as substrate comes online -- intra
|
|
761
|
+
# needs the DDG the L3 block above just emitted, interproc needs
|
|
762
|
+
# call_graph + caller PyCallArgument values (already present since
|
|
763
|
+
# L2) plus, for its one-hop caller-side closure, the caller's own
|
|
764
|
+
# DDG -- populated for every callable by the same L3 block, which
|
|
765
|
+
# always runs before this when analysis_level >= 4. A read resolved
|
|
766
|
+
# at a lower tier is never recomputed, so `-a 2 ⊆ -a 3 ⊆ -a 4` holds
|
|
767
|
+
# by construction.
|
|
768
|
+
if self.analysis_level >= 2:
|
|
769
|
+
from codeanalyzer.artifacts import (
|
|
770
|
+
dataflow_intra_tier, dataflow_interproc_tier, resolve_uses,
|
|
771
|
+
)
|
|
772
|
+
|
|
773
|
+
tier_fns = []
|
|
774
|
+
if self.analysis_level >= 3:
|
|
775
|
+
tier_fns.append(dataflow_intra_tier)
|
|
776
|
+
if self.analysis_level >= 4:
|
|
777
|
+
tier_fns.append(dataflow_interproc_tier)
|
|
778
|
+
app.config_uses, app.config_reads_unresolved = resolve_uses(
|
|
779
|
+
config_reads, app, tier_fns=tier_fns
|
|
780
|
+
)
|
|
781
|
+
|
|
679
782
|
# Build the v2 envelope, then persist it (the cache stores the full
|
|
680
783
|
# ``Analysis`` envelope so a reused cache round-trips schema_version).
|
|
681
784
|
# k_limit is an L3+ envelope key: below the dataflow levels it stays
|
|
@@ -936,39 +1039,3 @@ class Codeanalyzer:
|
|
|
936
1039
|
len(symbol_table), time.perf_counter() - t0_st,
|
|
937
1040
|
)
|
|
938
1041
|
return symbol_table
|
|
939
|
-
|
|
940
|
-
def _get_pycg_call_graph(
|
|
941
|
-
self,
|
|
942
|
-
symbol_table: Dict[str, PyModule],
|
|
943
|
-
jedi_edges: List[PyCallEdge],
|
|
944
|
-
) -> List[PyCallEdge]:
|
|
945
|
-
"""Build PyCG-resolved call edges.
|
|
946
|
-
|
|
947
|
-
Runs PyCG's iterative name-pointer analysis over the whole project
|
|
948
|
-
and returns edges with ``prov=["pycg"]``. Falls back to an
|
|
949
|
-
empty list and logs a warning on any failure so the caller can
|
|
950
|
-
continue with Jedi-only edges.
|
|
951
|
-
|
|
952
|
-
*jedi_edges* are the level-1 call edges; under the ``jedi`` shard
|
|
953
|
-
strategy they drive coupling-aware partitioning (see
|
|
954
|
-
:func:`shard_planner.plan_shards`).
|
|
955
|
-
"""
|
|
956
|
-
try:
|
|
957
|
-
pycg = PyCG(
|
|
958
|
-
self.project_dir,
|
|
959
|
-
skip_tests=self.skip_tests,
|
|
960
|
-
shard=self.options.pycg_shard,
|
|
961
|
-
shard_ceiling=self.options.pycg_shard_ceiling,
|
|
962
|
-
shard_timeout=self.options.pycg_shard_timeout,
|
|
963
|
-
shard_strategy=self.options.pycg_shard_strategy,
|
|
964
|
-
max_iter=self.options.pycg_max_iter,
|
|
965
|
-
using_ray=self.using_ray,
|
|
966
|
-
)
|
|
967
|
-
return pycg.build_call_graph_edges(symbol_table, jedi_edges=jedi_edges)
|
|
968
|
-
except PyCGExceptions.PyCGImportError as exc:
|
|
969
|
-
logger.warning(f"PyCG not installed — level 2 edges will be Jedi-only: {exc}")
|
|
970
|
-
return []
|
|
971
|
-
except PyCGExceptions.PyCGAnalysisError as exc:
|
|
972
|
-
logger.warning(f"PyCG analysis failed — level 2 edges will be Jedi-only: {exc}")
|
|
973
|
-
logger.debug("PyCG full traceback:", exc_info=True)
|
|
974
|
-
return []
|
|
@@ -245,15 +245,37 @@ def _names_loaded(node: ast.AST) -> Set[str]:
|
|
|
245
245
|
return out
|
|
246
246
|
|
|
247
247
|
|
|
248
|
-
|
|
248
|
+
#: Jedi resolves every spelling of the builtin -- ``@staticmethod``,
|
|
249
|
+
#: ``@builtins.staticmethod``, ``from builtins import staticmethod as sm`` --
|
|
250
|
+
#: to this one name (#135).
|
|
251
|
+
_STATICMETHOD_QUALIFIED = "builtins.staticmethod"
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def build_scope(
|
|
255
|
+
func: ast.AST,
|
|
256
|
+
enclosing_locals: Set[str],
|
|
257
|
+
decorator_names: Optional[Set[str]] = None,
|
|
258
|
+
) -> FunctionScope:
|
|
249
259
|
"""Classify every base name the callable touches. ``enclosing_locals`` is
|
|
250
260
|
the union of locals/params of all enclosing callables (for capture vs
|
|
251
|
-
global disambiguation).
|
|
261
|
+
global disambiguation).
|
|
262
|
+
|
|
263
|
+
``decorator_names`` are the callable's Jedi-resolved decorator
|
|
264
|
+
``qualified_name``s. When supplied, staticmethod detection is by identity,
|
|
265
|
+
so a dotted or aliased spelling is recognised (#135). When omitted -- a
|
|
266
|
+
caller with no resolved records -- it falls back to matching the written
|
|
267
|
+
source, which only recognises the bare ``@staticmethod``.
|
|
268
|
+
"""
|
|
252
269
|
params = _param_names(func)
|
|
253
270
|
scope = FunctionScope(params=params)
|
|
254
271
|
if params and isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
255
|
-
|
|
256
|
-
|
|
272
|
+
if decorator_names is None:
|
|
273
|
+
is_static = "staticmethod" in {
|
|
274
|
+
ast.unparse(d) for d in func.decorator_list
|
|
275
|
+
}
|
|
276
|
+
else:
|
|
277
|
+
is_static = _STATICMETHOD_QUALIFIED in decorator_names
|
|
278
|
+
if params[0] in ("self", "cls") and not is_static:
|
|
257
279
|
scope.self_name = params[0]
|
|
258
280
|
scope.globals_ = _declared(func, ast.Global)
|
|
259
281
|
nonlocals = _declared(func, ast.Nonlocal)
|
codeanalyzer/dataflow/builder.py
CHANGED
|
@@ -186,6 +186,13 @@ def build_function_pdgs(
|
|
|
186
186
|
oracle=oracle,
|
|
187
187
|
k=k,
|
|
188
188
|
global_qualifier=module.module_name,
|
|
189
|
+
# Resolved decorator names, so staticmethod detection works for a
|
|
190
|
+
# dotted or aliased spelling and not just bare `@staticmethod` (#135).
|
|
191
|
+
decorator_names={
|
|
192
|
+
d.qualified_name
|
|
193
|
+
for d in (pycallable.decorators or [])
|
|
194
|
+
if d.qualified_name
|
|
195
|
+
},
|
|
189
196
|
)
|
|
190
197
|
infos[pycallable.signature] = FunctionInfo(
|
|
191
198
|
signature=pycallable.signature, pdg=pdg, oracle=oracle
|
|
@@ -346,6 +353,10 @@ def build_program_graphs(
|
|
|
346
353
|
"""
|
|
347
354
|
class_idx = _class_index(app)
|
|
348
355
|
callable_idx = _callable_index(app)
|
|
356
|
+
# Reverse of callable_idx: a callsite's already-L2-backfilled BodyNode.callee
|
|
357
|
+
# (a can:// id, folding in the defuse linker's resolution -- not just Jedi's
|
|
358
|
+
# own callee_signature side channel) resolved back to a dotted signature.
|
|
359
|
+
id_to_sig = {c.id: sig for sig, c in callable_idx.items()}
|
|
349
360
|
|
|
350
361
|
infos, func_asts = build_function_pdgs(app, k, oracle_factory=oracle_factory)
|
|
351
362
|
|
|
@@ -366,7 +377,17 @@ def build_program_graphs(
|
|
|
366
377
|
calls_by_line.setdefault(call.lineno, (node.id, call))
|
|
367
378
|
|
|
368
379
|
for site in pycallable.call_sites or []:
|
|
369
|
-
|
|
380
|
+
# Prefer the callsite's body-backfilled callee over Jedi's own
|
|
381
|
+
# callee_signature side channel: under cross-test parso/Jedi cache
|
|
382
|
+
# pressure that inference can silently degrade (full-suite-only
|
|
383
|
+
# loss of SDG summary/global stitching -- test_dataflow_sdg.py),
|
|
384
|
+
# while BodyNode.callee is the deterministic L2 path (Jedi filtered
|
|
385
|
+
# the same way, PLUS the defuse linker's fallback -- l2_callees.py).
|
|
386
|
+
# Only a resolved INTERNAL target counts (id_to_sig misses on an
|
|
387
|
+
# external/unresolved callee); falls through to callee_signature
|
|
388
|
+
# exactly as before whenever the body doesn't have an answer.
|
|
389
|
+
body_node = pycallable.body.get(f"{site.start_line}:{site.start_column}")
|
|
390
|
+
target = (id_to_sig.get(body_node.callee) if body_node else None) or site.callee_signature
|
|
370
391
|
if not target:
|
|
371
392
|
continue
|
|
372
393
|
if target in class_idx and target not in infos:
|
|
@@ -10,7 +10,7 @@ Two forms per node:
|
|
|
10
10
|
an L1 body node and its coinciding CFG node land on the same key and L1 ⊆ L3
|
|
11
11
|
holds.
|
|
12
12
|
* **global** — ``"<callable can:// id>@<local>"``, the fully addressable id for
|
|
13
|
-
cross-callable references and the Neo4j
|
|
13
|
+
cross-callable references and the Neo4j PyBodyNode keys (a later task).
|
|
14
14
|
"""
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
from collections import defaultdict
|
codeanalyzer/dataflow/pdg.py
CHANGED
|
@@ -66,10 +66,15 @@ def build_pdg(
|
|
|
66
66
|
oracle: TypeBasedAliasOracle,
|
|
67
67
|
k: int = 3,
|
|
68
68
|
global_qualifier: Optional[str] = None,
|
|
69
|
+
decorator_names: Optional[Set[str]] = None,
|
|
69
70
|
) -> FunctionPDG:
|
|
70
|
-
"""CFG → dominance → def-use → PDG for one callable.
|
|
71
|
+
"""CFG → dominance → def-use → PDG for one callable.
|
|
72
|
+
|
|
73
|
+
``decorator_names`` are the callable's resolved decorator qualified names,
|
|
74
|
+
used for staticmethod detection by identity rather than spelling (#135).
|
|
75
|
+
"""
|
|
71
76
|
cfg = build_cfg(func)
|
|
72
|
-
scope = build_scope(func, enclosing_locals)
|
|
77
|
+
scope = build_scope(func, enclosing_locals, decorator_names=decorator_names)
|
|
73
78
|
facts = statement_facts(cfg, func, scope, k, global_qualifier)
|
|
74
79
|
|
|
75
80
|
edges: List[PDGEdge] = [
|
codeanalyzer/dataflow/scc.py
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"""Stage 5b of the level-3 dataflow ladder: SCC condensation of the call graph.
|
|
18
18
|
|
|
19
19
|
The call graph is a frozen oracle (level-1 Jedi edges, provenance-merged with
|
|
20
|
-
level-2
|
|
20
|
+
level-2 resolvers); Tarjan condenses it into strongly connected
|
|
21
21
|
components, and the condensation DAG in reverse topological order is the
|
|
22
22
|
bottom-up processing schedule for summary composition — callees before
|
|
23
23
|
callers, one monotone fixpoint per SCC (mutual recursion).
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Stage 0: which frameworks is this project actually using? (#27)
|
|
2
|
+
|
|
3
|
+
Gates every later stage, so a project without Celery never pays for Celery
|
|
4
|
+
rules and cannot false-positive on a locally-defined ``shared_task``. A
|
|
5
|
+
package counts as present if first-party source imports it OR the dependency
|
|
6
|
+
manifest names it -- either is sufficient, since an import may be dynamic.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Optional, Set
|
|
13
|
+
|
|
14
|
+
from codeanalyzer.entrypoints.rules import RuleSet
|
|
15
|
+
from codeanalyzer.schema.py_schema import PyApplication
|
|
16
|
+
|
|
17
|
+
_REQ = re.compile(r"^\s*['\"]?([A-Za-z0-9_.\-]+)")
|
|
18
|
+
_DEPS_START = re.compile(r"dependencies\s*=\s*\[")
|
|
19
|
+
_TABLE_HEADER = re.compile(r"(?m)^[ \t]*\[")
|
|
20
|
+
_PKG = re.compile(r"['\"]([A-Za-z0-9][A-Za-z0-9_.\-]*)")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def detected_frameworks(app: PyApplication, project_dir: Path, rules: RuleSet) -> Set[str]:
|
|
24
|
+
# `present` (imports, manifest names) and `detect:` values are both
|
|
25
|
+
# lowercased before comparison -- manifest names were already lowercased
|
|
26
|
+
# (PyPI/pip is case-insensitive) but imports and `detect:` were not, so
|
|
27
|
+
# a `detect: [Flask]` user rule silently never matched a `flask` import.
|
|
28
|
+
present = _imported_packages(app) | _manifest_packages(project_dir)
|
|
29
|
+
return {
|
|
30
|
+
name
|
|
31
|
+
for name, fw in rules.frameworks.items()
|
|
32
|
+
if any(pkg.lower() in present for pkg in (fw.detect or [name]))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _imported_packages(app: PyApplication) -> Set[str]:
|
|
37
|
+
out: Set[str] = set()
|
|
38
|
+
for mod in app.symbol_table.values():
|
|
39
|
+
for imp in mod.imports or []:
|
|
40
|
+
# `from flask import Flask` puts the package in `module`, not `name`.
|
|
41
|
+
# Prefer `module`; fall back to `name` for a bare `import flask`.
|
|
42
|
+
spelling = (getattr(imp, "module", "") or getattr(imp, "name", "") or "")
|
|
43
|
+
spelling = spelling.lstrip(".")
|
|
44
|
+
if spelling:
|
|
45
|
+
out.add(spelling.split(".", 1)[0].lower())
|
|
46
|
+
return out
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _manifest_packages(project_dir: Path) -> Set[str]:
|
|
50
|
+
out: Set[str] = set()
|
|
51
|
+
pyproject = project_dir / "pyproject.toml"
|
|
52
|
+
if pyproject.exists():
|
|
53
|
+
# PEP 621 `[project] dependencies = [...]` -- single- or multi-line,
|
|
54
|
+
# possibly containing nested `[...]` extras (`celery[redis]`).
|
|
55
|
+
span = _deps_array_span(_strip_comments(pyproject.read_text()))
|
|
56
|
+
if span is not None:
|
|
57
|
+
for pm in _PKG.finditer(span):
|
|
58
|
+
out.add(pm.group(1).split("[", 1)[0].lower())
|
|
59
|
+
requirements = project_dir / "requirements.txt"
|
|
60
|
+
if requirements.exists():
|
|
61
|
+
for line in requirements.read_text().splitlines():
|
|
62
|
+
m = _REQ.match(line)
|
|
63
|
+
if m:
|
|
64
|
+
out.add(m.group(1).split("[", 1)[0].lower())
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _strip_comments(text: str) -> str:
|
|
69
|
+
"""Drop everything from an unquoted ``#`` to end of line.
|
|
70
|
+
|
|
71
|
+
# ponytail: quote tracking resets each line, so a `#` inside a
|
|
72
|
+
# triple-quoted string spanning lines could be mis-stripped. TOML
|
|
73
|
+
# dependency arrays don't use those in practice; revisit if they do.
|
|
74
|
+
"""
|
|
75
|
+
out_lines = []
|
|
76
|
+
for line in text.splitlines():
|
|
77
|
+
in_str = None
|
|
78
|
+
cut = len(line)
|
|
79
|
+
for i, ch in enumerate(line):
|
|
80
|
+
if in_str:
|
|
81
|
+
if ch == in_str:
|
|
82
|
+
in_str = None
|
|
83
|
+
elif ch in ("'", '"'):
|
|
84
|
+
in_str = ch
|
|
85
|
+
elif ch == "#":
|
|
86
|
+
cut = i
|
|
87
|
+
break
|
|
88
|
+
out_lines.append(line[:cut])
|
|
89
|
+
return "\n".join(out_lines)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _deps_array_span(text: str) -> Optional[str]:
|
|
93
|
+
"""Return the contents between the `dependencies = [` and its matching
|
|
94
|
+
`]`, counting bracket depth so a nested `[...]` (extras, e.g.
|
|
95
|
+
`celery[redis]`) doesn't close the span early.
|
|
96
|
+
|
|
97
|
+
Bounded by the next TOML table header (a `[` starting a line): if the
|
|
98
|
+
array never closes before then, it's unterminated (truncated/corrupt
|
|
99
|
+
file) and this returns None rather than harvesting quoted strings out
|
|
100
|
+
of whatever table follows.
|
|
101
|
+
"""
|
|
102
|
+
m = _DEPS_START.search(text)
|
|
103
|
+
if not m:
|
|
104
|
+
return None
|
|
105
|
+
boundary = _TABLE_HEADER.search(text, m.end())
|
|
106
|
+
limit = boundary.start() if boundary else len(text)
|
|
107
|
+
depth = 1
|
|
108
|
+
in_str = None
|
|
109
|
+
i = m.end()
|
|
110
|
+
while i < limit and depth > 0:
|
|
111
|
+
ch = text[i]
|
|
112
|
+
if in_str:
|
|
113
|
+
if ch == in_str:
|
|
114
|
+
in_str = None
|
|
115
|
+
elif ch in ("'", '"'):
|
|
116
|
+
in_str = ch
|
|
117
|
+
elif ch == "[":
|
|
118
|
+
depth += 1
|
|
119
|
+
elif ch == "]":
|
|
120
|
+
depth -= 1
|
|
121
|
+
i += 1
|
|
122
|
+
if depth != 0:
|
|
123
|
+
return None
|
|
124
|
+
return text[m.end() : i - 1]
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""Stage 3: match rules against decorators and base classes (#27).
|
|
2
|
+
|
|
3
|
+
Matching is on ``PyDecorator.qualified_name`` -- never the written spelling --
|
|
4
|
+
so ``@route`` under ``from flask import route`` hits the same rule as
|
|
5
|
+
``@app.route``. An unresolved decorator (``qualified_name is None``) never
|
|
6
|
+
matches: under-approximate rather than guess.
|
|
7
|
+
|
|
8
|
+
Pattern grammar: ``{a,b}`` alternation (not nested) and a trailing/embedded
|
|
9
|
+
``*`` that matches module MEMBERS only -- it does not cross a ``.``, so
|
|
10
|
+
``rest_framework.viewsets.*`` matches ``ModelViewSet`` but not
|
|
11
|
+
``viewsets.mixins.ListModelMixin``. Everything else is literal.
|
|
12
|
+
``validate_pattern`` rejects anything outside this grammar (unbalanced or
|
|
13
|
+
nested ``{``) so a typo in a rules file is a load-time ``RulesError``
|
|
14
|
+
(enforced by ``rules.py``), never a crash mid-analysis.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import ast
|
|
19
|
+
import re
|
|
20
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Tuple
|
|
21
|
+
|
|
22
|
+
from codeanalyzer.schema.py_schema import PyEntrypoint
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from codeanalyzer.entrypoints.rules import BaseRule, DecoratorRule
|
|
26
|
+
|
|
27
|
+
# Dispatch names that are HTTP verbs. DRF's ViewSet dispatch names
|
|
28
|
+
# (list, retrieve, create, ...) are NOT verbs and must not be emitted as such.
|
|
29
|
+
_HTTP_VERBS = {"get", "post", "put", "patch", "delete", "head", "options"}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class PatternError(ValueError):
|
|
33
|
+
"""A ``match`` pattern outside the ``{a,b}`` / ``*`` grammar `_compile` handles."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def validate_pattern(pattern: str) -> None:
|
|
37
|
+
"""Raise ``PatternError`` for unbalanced or nested ``{``."""
|
|
38
|
+
depth = 0
|
|
39
|
+
for ch in pattern:
|
|
40
|
+
if ch == "{":
|
|
41
|
+
depth += 1
|
|
42
|
+
if depth > 1:
|
|
43
|
+
raise PatternError(f"nested '{{' is not supported: {pattern!r}")
|
|
44
|
+
elif ch == "}":
|
|
45
|
+
depth -= 1
|
|
46
|
+
if depth < 0:
|
|
47
|
+
raise PatternError(f"unmatched '}}': {pattern!r}")
|
|
48
|
+
if depth != 0:
|
|
49
|
+
raise PatternError(f"unbalanced '{{': {pattern!r}")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def match_pattern(pattern: str, qualified_name: Optional[str]) -> bool:
|
|
53
|
+
"""``{a,b}`` alternation and trailing ``*``; everything else is literal."""
|
|
54
|
+
if not qualified_name:
|
|
55
|
+
return False
|
|
56
|
+
return re.fullmatch(_compile(pattern), qualified_name) is not None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _compile(pattern: str) -> str:
|
|
60
|
+
validate_pattern(pattern)
|
|
61
|
+
out, i = [], 0
|
|
62
|
+
while i < len(pattern):
|
|
63
|
+
ch = pattern[i]
|
|
64
|
+
if ch == "{":
|
|
65
|
+
j = pattern.index("}", i)
|
|
66
|
+
alts = pattern[i + 1 : j].split(",")
|
|
67
|
+
out.append("(?:" + "|".join(re.escape(a.strip()) for a in alts) + ")")
|
|
68
|
+
i = j + 1
|
|
69
|
+
elif ch == "*":
|
|
70
|
+
out.append(r"[^.\s]*")
|
|
71
|
+
i += 1
|
|
72
|
+
else:
|
|
73
|
+
out.append(re.escape(ch))
|
|
74
|
+
i += 1
|
|
75
|
+
return "".join(out)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _literal(text: Optional[str]) -> Any:
|
|
79
|
+
"""Best-effort: decorator arguments are unparsed source fragments."""
|
|
80
|
+
if text is None:
|
|
81
|
+
return None
|
|
82
|
+
try:
|
|
83
|
+
return ast.literal_eval(text)
|
|
84
|
+
except (ValueError, SyntaxError):
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _route_of(dec, spec: Optional[Dict[str, Any]]) -> Optional[str]:
|
|
89
|
+
if not spec or spec.get("from") != "positional":
|
|
90
|
+
return None
|
|
91
|
+
args = dec.positional_arguments or []
|
|
92
|
+
idx = int(spec.get("index", 0))
|
|
93
|
+
if idx >= len(args):
|
|
94
|
+
return None
|
|
95
|
+
value = _literal(args[idx])
|
|
96
|
+
return value if isinstance(value, str) else None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _methods_of(dec, spec: Optional[Dict[str, Any]]) -> List[str]:
|
|
100
|
+
if not spec:
|
|
101
|
+
return []
|
|
102
|
+
source = spec.get("from")
|
|
103
|
+
if source == "match_suffix":
|
|
104
|
+
verb = (dec.qualified_name or "").rsplit(".", 1)[-1]
|
|
105
|
+
return [verb.upper()]
|
|
106
|
+
if source == "keyword":
|
|
107
|
+
raw = (dec.keyword_arguments or {}).get(spec.get("name", ""))
|
|
108
|
+
value = _literal(raw)
|
|
109
|
+
if isinstance(value, (list, tuple)):
|
|
110
|
+
return [str(v).upper() for v in value]
|
|
111
|
+
return [str(v).upper() for v in (spec.get("default") or [])]
|
|
112
|
+
return []
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def entrypoints_from_decorators(
|
|
116
|
+
node, framework: str, rules: Iterable["DecoratorRule"]
|
|
117
|
+
) -> List[PyEntrypoint]:
|
|
118
|
+
out: List[PyEntrypoint] = []
|
|
119
|
+
for dec in getattr(node, "decorators", []) or []:
|
|
120
|
+
for rule in rules:
|
|
121
|
+
if not match_pattern(rule.match, dec.qualified_name):
|
|
122
|
+
continue
|
|
123
|
+
out.append(
|
|
124
|
+
PyEntrypoint(
|
|
125
|
+
framework=framework,
|
|
126
|
+
confidence=rule.confidence,
|
|
127
|
+
rule=rule.id,
|
|
128
|
+
ruleset=rule.origin,
|
|
129
|
+
evidence=dec.qualified_name,
|
|
130
|
+
route=_route_of(dec, rule.route),
|
|
131
|
+
http_methods=_methods_of(dec, rule.methods),
|
|
132
|
+
)
|
|
133
|
+
)
|
|
134
|
+
return out
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def entrypoints_from_bases(
|
|
138
|
+
cls,
|
|
139
|
+
framework: str,
|
|
140
|
+
rules: Iterable["BaseRule"],
|
|
141
|
+
resolve: Callable[[str], Optional[str]],
|
|
142
|
+
) -> Tuple[List[PyEntrypoint], Dict[str, List[PyEntrypoint]]]:
|
|
143
|
+
"""Records for a routed class and for the methods the framework dispatches.
|
|
144
|
+
|
|
145
|
+
``resolve`` maps a written base-class name to its resolved qualified name
|
|
146
|
+
(identity when already qualified). Dispatch names are intersected with the
|
|
147
|
+
methods the class actually defines, so a ``ListView`` with only ``get``
|
|
148
|
+
gains no phantom ``post`` entrypoint.
|
|
149
|
+
"""
|
|
150
|
+
class_eps: List[PyEntrypoint] = []
|
|
151
|
+
method_eps: Dict[str, List[PyEntrypoint]] = {}
|
|
152
|
+
|
|
153
|
+
for rule in rules:
|
|
154
|
+
if not any(
|
|
155
|
+
match_pattern(rule.match, resolve(b) or b) for b in (cls.base_classes or [])
|
|
156
|
+
):
|
|
157
|
+
continue
|
|
158
|
+
class_eps.append(
|
|
159
|
+
PyEntrypoint(
|
|
160
|
+
framework=framework,
|
|
161
|
+
confidence=rule.confidence,
|
|
162
|
+
rule=rule.id,
|
|
163
|
+
ruleset=rule.origin,
|
|
164
|
+
evidence=cls.signature,
|
|
165
|
+
)
|
|
166
|
+
)
|
|
167
|
+
defined = set((cls.callables or {}).keys())
|
|
168
|
+
for name in rule.dispatch:
|
|
169
|
+
if name not in defined:
|
|
170
|
+
continue
|
|
171
|
+
method_eps.setdefault(name, []).append(
|
|
172
|
+
PyEntrypoint(
|
|
173
|
+
framework=framework,
|
|
174
|
+
confidence=rule.confidence,
|
|
175
|
+
rule=f"{rule.id}.dispatch",
|
|
176
|
+
ruleset=rule.origin,
|
|
177
|
+
evidence=cls.signature,
|
|
178
|
+
http_methods=[name.upper()] if name in _HTTP_VERBS else [],
|
|
179
|
+
via=cls.id or None,
|
|
180
|
+
)
|
|
181
|
+
)
|
|
182
|
+
return class_eps, method_eps
|