codeanalyzer-python 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +77 -4
- codeanalyzer/core.py +174 -72
- codeanalyzer/dataflow/__init__.py +35 -0
- codeanalyzer/dataflow/access_paths.py +563 -0
- codeanalyzer/dataflow/alias.py +93 -0
- codeanalyzer/dataflow/builder.py +688 -0
- codeanalyzer/dataflow/cfg.py +605 -0
- codeanalyzer/dataflow/defuse.py +113 -0
- codeanalyzer/dataflow/dominance.py +140 -0
- codeanalyzer/dataflow/identity.py +91 -0
- codeanalyzer/dataflow/pdg.py +100 -0
- codeanalyzer/dataflow/scalpel_oracle.py +269 -0
- codeanalyzer/dataflow/scc.py +91 -0
- codeanalyzer/dataflow/sdg.py +424 -0
- codeanalyzer/dataflow/slicing.py +93 -0
- codeanalyzer/dataflow/summaries.py +217 -0
- codeanalyzer/dataflow/syntactic.py +26 -0
- codeanalyzer/neo4j/__init__.py +1 -1
- codeanalyzer/neo4j/bolt.py +19 -4
- codeanalyzer/neo4j/cypher.py +9 -3
- codeanalyzer/neo4j/emit.py +10 -5
- codeanalyzer/neo4j/project.py +307 -60
- codeanalyzer/neo4j/rows.py +18 -15
- codeanalyzer/neo4j/schema.py +297 -15
- codeanalyzer/options/options.py +4 -0
- codeanalyzer/provenance.py +61 -0
- codeanalyzer/schema/__init__.py +19 -0
- codeanalyzer/schema/assign_ids.py +37 -0
- codeanalyzer/schema/call_graph_ids.py +12 -0
- codeanalyzer/schema/ids.py +23 -0
- codeanalyzer/schema/l1_body.py +29 -0
- codeanalyzer/schema/l2_callees.py +36 -0
- codeanalyzer/schema/py_schema.py +175 -26
- codeanalyzer/semantic_analysis/call_graph.py +24 -27
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +10 -10
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +7 -7
- codeanalyzer/syntactic_analysis/import_resolver.py +67 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +103 -20
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/METADATA +233 -43
- codeanalyzer_python-1.0.0.dist-info/RECORD +59 -0
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/WHEEL +1 -1
- codeanalyzer/neo4j/catalog.py +0 -245
- codeanalyzer_python-0.3.0.dist-info/RECORD +0 -38
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.3.0.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/schema/py_schema.py
CHANGED
|
@@ -20,9 +20,8 @@ This module defines the data models used to represent Python code structures
|
|
|
20
20
|
for static analysis purposes.
|
|
21
21
|
"""
|
|
22
22
|
from __future__ import annotations
|
|
23
|
-
import inspect
|
|
24
23
|
from pathlib import Path
|
|
25
|
-
from typing import Any, Dict, List, Optional
|
|
24
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
26
25
|
import gzip
|
|
27
26
|
|
|
28
27
|
from pydantic import BaseModel
|
|
@@ -120,12 +119,23 @@ def builder(cls):
|
|
|
120
119
|
# Get type hints and default values for the fields in the model.
|
|
121
120
|
# For example, {file_path: Path, module_name: str, imports: List[PyImport], ...}
|
|
122
121
|
annotations = cls.__annotations__
|
|
123
|
-
# Get default values for the fields in the model.
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
122
|
+
# Get default values for the fields in the model. `inspect.signature` is
|
|
123
|
+
# unreliable for models carrying forward references (e.g. PyCallable's
|
|
124
|
+
# self-referential ``callables``): Pydantic falls back to a generic
|
|
125
|
+
# ``(**data)`` signature that drops the per-field defaults, so the builder
|
|
126
|
+
# would seed those fields with ``None`` and fail validation. Read the declared
|
|
127
|
+
# defaults straight off the model instead. Required fields are intentionally
|
|
128
|
+
# omitted (seeded ``None``) — the builder chain must set them.
|
|
129
|
+
defaults = {}
|
|
130
|
+
model_fields = getattr(cls, "model_fields", None) # Pydantic v2
|
|
131
|
+
if model_fields:
|
|
132
|
+
for name, field in model_fields.items():
|
|
133
|
+
if not field.is_required():
|
|
134
|
+
defaults[name] = field.get_default(call_default_factory=True)
|
|
135
|
+
else: # Pydantic v1
|
|
136
|
+
for name, field in getattr(cls, "__fields__", {}).items():
|
|
137
|
+
if not field.required:
|
|
138
|
+
defaults[name] = field.get_default()
|
|
129
139
|
# Create a namespace for the builder class.
|
|
130
140
|
namespace = {}
|
|
131
141
|
|
|
@@ -168,6 +178,71 @@ def builder(cls):
|
|
|
168
178
|
return cls
|
|
169
179
|
|
|
170
180
|
|
|
181
|
+
def byte_offsets(source: str, start_line: int, start_col: int,
|
|
182
|
+
end_line: int, end_col: int) -> Tuple[int, int]:
|
|
183
|
+
"""Convert (1-based line, 0-based col) ast positions to utf-8 byte offsets
|
|
184
|
+
into `source`. `col` is a character offset within the line (ast semantics);
|
|
185
|
+
we re-encode the line prefix to bytes so multibyte chars are handled."""
|
|
186
|
+
lines = source.splitlines(keepends=True)
|
|
187
|
+
def offset(line: int, col: int) -> int:
|
|
188
|
+
prefix_bytes = len("".join(lines[: line - 1]).encode("utf-8"))
|
|
189
|
+
col_bytes = len(lines[line - 1][:col].encode("utf-8")) if line - 1 < len(lines) else 0
|
|
190
|
+
return prefix_bytes + col_bytes
|
|
191
|
+
return offset(start_line, start_col), offset(end_line, end_col)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
@builder
|
|
195
|
+
@msgpk
|
|
196
|
+
class Span(BaseModel):
|
|
197
|
+
"""Where a node lives in source. `start`/`end` are [line, col] (1-based line,
|
|
198
|
+
0-based col, ast semantics); `bytes` are utf-8 offsets into module.source."""
|
|
199
|
+
start: Tuple[int, int]
|
|
200
|
+
end: Tuple[int, int]
|
|
201
|
+
bytes: Tuple[int, int]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@builder
|
|
205
|
+
@msgpk
|
|
206
|
+
class BodyNode(BaseModel):
|
|
207
|
+
"""A node in a callable's `body`: an AST region (statement/call/branch/…) or
|
|
208
|
+
a synthetic analysis vertex (entry/exit/formal_in/out/actual_in/out)."""
|
|
209
|
+
kind: str
|
|
210
|
+
span: Optional[Span] = None
|
|
211
|
+
callee: Optional[str] = None # only on `call` nodes; the sanctioned null→id slot
|
|
212
|
+
of: Optional[str] = None # param vertices: the variable/return they carry
|
|
213
|
+
parent: Optional[str] = None # actuals: owning callsite ordinal id
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@builder
|
|
217
|
+
@msgpk
|
|
218
|
+
class CfgEdge(BaseModel):
|
|
219
|
+
src: str; dst: str; kind: str = "fallthrough"
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
@builder
|
|
223
|
+
@msgpk
|
|
224
|
+
class CdgEdge(BaseModel):
|
|
225
|
+
src: str; dst: str
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@builder
|
|
229
|
+
@msgpk
|
|
230
|
+
class DdgEdge(BaseModel):
|
|
231
|
+
src: str; dst: str; var: Optional[str] = None; prov: List[str] = []
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@builder
|
|
235
|
+
@msgpk
|
|
236
|
+
class SummaryEdge(BaseModel):
|
|
237
|
+
src: str; dst: str
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
@builder
|
|
241
|
+
@msgpk
|
|
242
|
+
class ParamEdge(BaseModel):
|
|
243
|
+
src: str; dst: str
|
|
244
|
+
|
|
245
|
+
|
|
171
246
|
@builder
|
|
172
247
|
@msgpk
|
|
173
248
|
class PyImport(BaseModel):
|
|
@@ -176,6 +251,7 @@ class PyImport(BaseModel):
|
|
|
176
251
|
module: str
|
|
177
252
|
name: str
|
|
178
253
|
alias: Optional[str] = None
|
|
254
|
+
resolved_module: Optional[str] = None
|
|
179
255
|
start_line: int = -1
|
|
180
256
|
end_line: int = -1
|
|
181
257
|
start_column: int = -1
|
|
@@ -216,7 +292,10 @@ class PyVariableDeclaration(BaseModel):
|
|
|
216
292
|
"""Represents a Python variable declaration."""
|
|
217
293
|
|
|
218
294
|
name: str
|
|
219
|
-
|
|
295
|
+
# Optional WITH a default: emission drops None (exclude_none), so a
|
|
296
|
+
# required-but-nullable field would make the emitted JSON fail its own
|
|
297
|
+
# model's validation whenever the type is uninferred.
|
|
298
|
+
type: Optional[str] = None
|
|
220
299
|
initializer: Optional[str] = None
|
|
221
300
|
value: Optional[Any] = None
|
|
222
301
|
scope: Literal["module", "class", "function"] = "module"
|
|
@@ -240,6 +319,19 @@ class PyCallableParameter(BaseModel):
|
|
|
240
319
|
end_column: int = -1
|
|
241
320
|
|
|
242
321
|
|
|
322
|
+
@builder
|
|
323
|
+
@msgpk
|
|
324
|
+
class PyCallArgument(BaseModel):
|
|
325
|
+
"""One call-site argument: AST category + inferred type, kept separate.
|
|
326
|
+
|
|
327
|
+
The legacy ``PyCallsite.argument_types`` mixed these two vocabularies
|
|
328
|
+
in one list; this model is the disambiguated replacement (#86).
|
|
329
|
+
"""
|
|
330
|
+
|
|
331
|
+
ast_kind: str
|
|
332
|
+
inferred_type: Optional[str] = None
|
|
333
|
+
|
|
334
|
+
|
|
243
335
|
@builder
|
|
244
336
|
@msgpk
|
|
245
337
|
class PyCallsite(BaseModel):
|
|
@@ -249,6 +341,7 @@ class PyCallsite(BaseModel):
|
|
|
249
341
|
receiver_expr: Optional[str] = None
|
|
250
342
|
receiver_type: Optional[str] = None
|
|
251
343
|
argument_types: List[str] = []
|
|
344
|
+
arguments: List[PyCallArgument] = []
|
|
252
345
|
return_type: Optional[str] = None
|
|
253
346
|
callee_signature: Optional[str] = None
|
|
254
347
|
is_constructor_call: bool = False
|
|
@@ -266,20 +359,27 @@ class PyCallable(BaseModel):
|
|
|
266
359
|
name: str
|
|
267
360
|
path: str
|
|
268
361
|
signature: str # e.g., module.<class_name>.function_name
|
|
362
|
+
id: str = ""
|
|
363
|
+
kind: str = "function"
|
|
364
|
+
span: Optional[Span] = None
|
|
269
365
|
comments: List[PyComment] = []
|
|
270
366
|
decorators: List[str] = []
|
|
271
367
|
parameters: List[PyCallableParameter] = []
|
|
272
368
|
return_type: Optional[str] = None
|
|
273
|
-
code: str = None
|
|
274
369
|
start_line: int = -1
|
|
275
370
|
end_line: int = -1
|
|
276
371
|
code_start_line: int = -1
|
|
277
372
|
accessed_symbols: List[PySymbol] = []
|
|
278
373
|
call_sites: List[PyCallsite] = []
|
|
279
|
-
|
|
280
|
-
|
|
374
|
+
callables: Dict[str, "PyCallable"] = {} # nested callables (closures)
|
|
375
|
+
types: Dict[str, "PyClass"] = {} # nested (local) classes
|
|
281
376
|
local_variables: List[PyVariableDeclaration] = []
|
|
282
377
|
cyclomatic_complexity: int = 0
|
|
378
|
+
body: Dict[str, BodyNode] = {}
|
|
379
|
+
cfg: List[CfgEdge] = []
|
|
380
|
+
cdg: List[CdgEdge] = []
|
|
381
|
+
ddg: List[DdgEdge] = []
|
|
382
|
+
summary: List[SummaryEdge] = []
|
|
283
383
|
|
|
284
384
|
def __hash__(self) -> int:
|
|
285
385
|
"""Generate a hash based on the callable's signature."""
|
|
@@ -295,6 +395,7 @@ class PyClassAttribute(BaseModel):
|
|
|
295
395
|
|
|
296
396
|
name: str
|
|
297
397
|
type: Optional[str] = None
|
|
398
|
+
initializer: Optional[str] = None
|
|
298
399
|
comments: List[PyComment] = []
|
|
299
400
|
start_line: int = -1
|
|
300
401
|
end_line: int = -1
|
|
@@ -307,12 +408,14 @@ class PyClass(BaseModel):
|
|
|
307
408
|
|
|
308
409
|
name: str
|
|
309
410
|
signature: str # e.g., module.class_name
|
|
411
|
+
id: str = ""
|
|
412
|
+
kind: str = "class"
|
|
413
|
+
span: Optional[Span] = None
|
|
310
414
|
comments: List[PyComment] = []
|
|
311
|
-
code: str = None
|
|
312
415
|
base_classes: List[str] = []
|
|
313
|
-
|
|
416
|
+
callables: Dict[str, PyCallable] = {} # methods, keystone containment name
|
|
314
417
|
attributes: Dict[str, PyClassAttribute] = {}
|
|
315
|
-
|
|
418
|
+
types: Dict[str, "PyClass"] = {} # inner classes, keystone containment name
|
|
316
419
|
start_line: int = -1
|
|
317
420
|
end_line: int = -1
|
|
318
421
|
|
|
@@ -328,9 +431,12 @@ class PyModule(BaseModel):
|
|
|
328
431
|
|
|
329
432
|
file_path: str
|
|
330
433
|
module_name: str
|
|
434
|
+
id: str = ""
|
|
435
|
+
kind: str = "module"
|
|
436
|
+
source: str = ""
|
|
331
437
|
imports: List[PyImport] = []
|
|
332
438
|
comments: List[PyComment] = []
|
|
333
|
-
|
|
439
|
+
types: Dict[str, PyClass] = {} # classes, keystone containment name
|
|
334
440
|
functions: Dict[str, PyCallable] = {}
|
|
335
441
|
variables: List[PyVariableDeclaration] = []
|
|
336
442
|
# Metadata for caching
|
|
@@ -342,41 +448,84 @@ class PyModule(BaseModel):
|
|
|
342
448
|
@builder
|
|
343
449
|
@msgpk
|
|
344
450
|
class PyCallEdge(BaseModel):
|
|
345
|
-
"""Identity-only call-graph edge with weight
|
|
451
|
+
"""Identity-only call-graph edge with weight (keystone shape: the list name
|
|
452
|
+
IS the edge type, so there is no ``type`` field).
|
|
346
453
|
|
|
347
|
-
|
|
348
|
-
``
|
|
349
|
-
``PyCallable`` entries in the symbol table, not a separate vertex type.
|
|
454
|
+
``src`` and ``dst`` are node ids — the caller's ``can://`` id and the
|
|
455
|
+
callee's ``can://`` id (a symbol-table callable or an ``@external`` home).
|
|
350
456
|
Rich per-call metadata (receiver, arguments, location, ...) lives on
|
|
351
457
|
``PyCallsite`` inside the source ``PyCallable.call_sites``.
|
|
352
458
|
"""
|
|
353
459
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
type: Literal["CALL_DEP"] = "CALL_DEP"
|
|
460
|
+
src: str # caller callable id
|
|
461
|
+
dst: str # callee callable (or external) id
|
|
357
462
|
weight: int = 1
|
|
358
|
-
|
|
463
|
+
prov: List[Literal["jedi", "pycg", "joern"]] = []
|
|
359
464
|
|
|
360
465
|
|
|
361
466
|
@builder
|
|
362
467
|
@msgpk
|
|
363
468
|
class PyExternalSymbol(BaseModel):
|
|
364
469
|
"""A call-graph target outside the analyzed project -- an imported library or
|
|
365
|
-
builtin member.
|
|
366
|
-
|
|
470
|
+
builtin member. An edge-endpoint id home, not a tree node: keyed in
|
|
471
|
+
``PyApplication.external_symbols`` by its ``can://…/@external/…`` id."""
|
|
367
472
|
|
|
473
|
+
id: str = "" # can://python/<app>/@external/<module>/<name>
|
|
474
|
+
kind: str = "external"
|
|
368
475
|
name: str # the member/short name, e.g. "get" for "requests.get"
|
|
369
476
|
module: Optional[str] = None # best-effort owning module, e.g. "requests"
|
|
370
477
|
|
|
371
478
|
|
|
479
|
+
@builder
|
|
480
|
+
@msgpk
|
|
481
|
+
class PyRepositoryInfo(BaseModel):
|
|
482
|
+
"""Where the analyzed source came from: git provenance captured at analysis time."""
|
|
483
|
+
|
|
484
|
+
uri: Optional[str] = None
|
|
485
|
+
revision: str
|
|
486
|
+
dirty: bool = False
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
@builder
|
|
490
|
+
@msgpk
|
|
491
|
+
class PyAnalyzerInfo(BaseModel):
|
|
492
|
+
"""Which analyzer produced this snapshot, and how it was configured.
|
|
493
|
+
Lives on the ``Analysis`` envelope (keystone ``analyzer{name,version}``;
|
|
494
|
+
``config`` rides additively)."""
|
|
495
|
+
|
|
496
|
+
name: str = "codeanalyzer-python"
|
|
497
|
+
version: str = "unknown"
|
|
498
|
+
config: Dict[str, Any] = {}
|
|
499
|
+
|
|
500
|
+
|
|
372
501
|
@builder
|
|
373
502
|
@msgpk
|
|
374
503
|
class PyApplication(BaseModel):
|
|
375
504
|
"""Represents a Python application."""
|
|
376
505
|
|
|
377
506
|
symbol_table: Dict[str, PyModule]
|
|
507
|
+
id: str = ""
|
|
508
|
+
kind: str = "application"
|
|
378
509
|
call_graph: List[PyCallEdge] = []
|
|
379
510
|
# Call-graph endpoints not declared in the symbol table (imported library /
|
|
380
511
|
# builtin members), keyed by signature. Populated by the analyzer so every
|
|
381
512
|
# backend (JSON and Neo4j) shares one authoritative external-symbol set.
|
|
382
513
|
external_symbols: Dict[str, PyExternalSymbol] = {}
|
|
514
|
+
# Git provenance of the analyzed checkout, captured at analysis time.
|
|
515
|
+
repository: Optional[PyRepositoryInfo] = None
|
|
516
|
+
# Interprocedural parameter-passing edges (formal↔actual); populated at L4.
|
|
517
|
+
param_in: List[ParamEdge] = []
|
|
518
|
+
param_out: List[ParamEdge] = []
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
@builder
|
|
522
|
+
@msgpk
|
|
523
|
+
class Analysis(BaseModel):
|
|
524
|
+
"""v2 payload root: envelope + the application tree node. ``k_limit`` is an
|
|
525
|
+
L3+ envelope key (None below the dataflow levels; exclude_none drops it)."""
|
|
526
|
+
schema_version: str = "2.0.0"
|
|
527
|
+
language: str = "python"
|
|
528
|
+
max_level: int = 1
|
|
529
|
+
k_limit: Optional[int] = None
|
|
530
|
+
analyzer: PyAnalyzerInfo = PyAnalyzerInfo()
|
|
531
|
+
application: PyApplication
|
|
@@ -38,24 +38,24 @@ from codeanalyzer.schema.py_schema import (
|
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
def _walk_class_callables(cls: PyClass) -> Iterator[PyCallable]:
|
|
41
|
-
for method in cls.
|
|
41
|
+
for method in cls.callables.values():
|
|
42
42
|
yield from _walk_callable(method)
|
|
43
|
-
for inner in cls.
|
|
43
|
+
for inner in cls.types.values():
|
|
44
44
|
yield from _walk_class_callables(inner)
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
def _walk_callable(c: PyCallable) -> Iterator[PyCallable]:
|
|
48
48
|
yield c
|
|
49
|
-
for inner in c.
|
|
49
|
+
for inner in c.callables.values():
|
|
50
50
|
yield from _walk_callable(inner)
|
|
51
|
-
for inner_cls in c.
|
|
51
|
+
for inner_cls in c.types.values():
|
|
52
52
|
yield from _walk_class_callables(inner_cls)
|
|
53
53
|
|
|
54
54
|
|
|
55
55
|
def _walk_module_callables(module: PyModule) -> Iterator[PyCallable]:
|
|
56
56
|
for fn in module.functions.values():
|
|
57
57
|
yield from _walk_callable(fn)
|
|
58
|
-
for cls in module.
|
|
58
|
+
for cls in module.types.values():
|
|
59
59
|
yield from _walk_class_callables(cls)
|
|
60
60
|
|
|
61
61
|
|
|
@@ -69,18 +69,18 @@ def iter_callables_in_symbol_table(
|
|
|
69
69
|
|
|
70
70
|
def _walk_classes_in_class(cls: PyClass) -> Iterator[PyClass]:
|
|
71
71
|
yield cls
|
|
72
|
-
for inner in cls.
|
|
72
|
+
for inner in cls.types.values():
|
|
73
73
|
yield from _walk_classes_in_class(inner)
|
|
74
74
|
# Classes can live inside methods (e.g. a factory method that defines
|
|
75
75
|
# a helper class). Recurse through every method's callable subtree.
|
|
76
|
-
for method in cls.
|
|
76
|
+
for method in cls.callables.values():
|
|
77
77
|
yield from _walk_classes_in_callable(method)
|
|
78
78
|
|
|
79
79
|
|
|
80
80
|
def _walk_classes_in_callable(c: PyCallable) -> Iterator[PyClass]:
|
|
81
|
-
for inner_cls in c.
|
|
81
|
+
for inner_cls in c.types.values():
|
|
82
82
|
yield from _walk_classes_in_class(inner_cls)
|
|
83
|
-
for inner in c.
|
|
83
|
+
for inner in c.callables.values():
|
|
84
84
|
yield from _walk_classes_in_callable(inner)
|
|
85
85
|
|
|
86
86
|
|
|
@@ -91,7 +91,7 @@ def iter_classes_in_symbol_table(
|
|
|
91
91
|
inner classes, classes nested in functions, and classes nested in
|
|
92
92
|
class methods."""
|
|
93
93
|
for module in symbol_table.values():
|
|
94
|
-
for cls in module.
|
|
94
|
+
for cls in module.types.values():
|
|
95
95
|
yield from _walk_classes_in_class(cls)
|
|
96
96
|
for fn in module.functions.values():
|
|
97
97
|
yield from _walk_classes_in_callable(fn)
|
|
@@ -118,22 +118,21 @@ def to_digraph(app: PyApplication) -> nx.DiGraph:
|
|
|
118
118
|
added as **ghost** nodes (``callable=None``, ``ghost=True``) so the
|
|
119
119
|
edges are preserved.
|
|
120
120
|
|
|
121
|
-
Edges carry ``
|
|
121
|
+
Edges carry ``weight`` and ``prov`` attributes.
|
|
122
122
|
"""
|
|
123
123
|
g = nx.DiGraph()
|
|
124
124
|
by_sig = callables_by_signature(app)
|
|
125
125
|
for sig, c in by_sig.items():
|
|
126
126
|
g.add_node(sig, callable=c, ghost=False)
|
|
127
127
|
for e in app.call_graph:
|
|
128
|
-
for sig in (e.
|
|
128
|
+
for sig in (e.src, e.dst):
|
|
129
129
|
if sig not in g.nodes:
|
|
130
130
|
g.add_node(sig, callable=None, ghost=True)
|
|
131
131
|
g.add_edge(
|
|
132
|
-
e.
|
|
133
|
-
e.
|
|
134
|
-
type=e.type,
|
|
132
|
+
e.src,
|
|
133
|
+
e.dst,
|
|
135
134
|
weight=e.weight,
|
|
136
|
-
|
|
135
|
+
prov=list(e.prov),
|
|
137
136
|
)
|
|
138
137
|
return g
|
|
139
138
|
|
|
@@ -143,18 +142,16 @@ def from_digraph(g: nx.DiGraph) -> list:
|
|
|
143
142
|
|
|
144
143
|
Only edges are extracted; nodes are not serialized here — they are
|
|
145
144
|
expected to already exist as ``PyCallable`` entries in the symbol
|
|
146
|
-
table. Edge attributes default to
|
|
147
|
-
provenance when missing.
|
|
145
|
+
table. Edge attributes default to weight 1 / empty prov when missing.
|
|
148
146
|
"""
|
|
149
147
|
edges = []
|
|
150
148
|
for src, dst, data in g.edges(data=True):
|
|
151
149
|
edges.append(
|
|
152
150
|
PyCallEdge(
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
type=data.get("type", "CALL_DEP"),
|
|
151
|
+
src=src,
|
|
152
|
+
dst=dst,
|
|
156
153
|
weight=int(data.get("weight", 1)),
|
|
157
|
-
|
|
154
|
+
prov=list(data.get("prov", data.get("provenance", []))),
|
|
158
155
|
)
|
|
159
156
|
)
|
|
160
157
|
return edges
|
|
@@ -183,7 +180,7 @@ def jedi_call_graph_edges(
|
|
|
183
180
|
counts[(caller.signature, site.callee_signature)] += 1
|
|
184
181
|
|
|
185
182
|
return [
|
|
186
|
-
PyCallEdge(
|
|
183
|
+
PyCallEdge(src=src, dst=dst, weight=n, prov=["jedi"])
|
|
187
184
|
for (src, dst), n in counts.items()
|
|
188
185
|
]
|
|
189
186
|
|
|
@@ -255,7 +252,7 @@ def filter_external_edges(
|
|
|
255
252
|
Edges where an app callable calls a library function (or vice-versa) are
|
|
256
253
|
retained; only lib→lib edges are dropped. The app symbol set is built by
|
|
257
254
|
walking every callable in the symbol table recursively (including nested
|
|
258
|
-
functions and closures via ``
|
|
255
|
+
functions and closures via ``callables``) plus every class, so
|
|
259
256
|
PyCG-discovered closure nodes are correctly recognised as app symbols.
|
|
260
257
|
"""
|
|
261
258
|
app_symbols: set = {c.signature for c in iter_callables_in_symbol_table(symbol_table)}
|
|
@@ -263,7 +260,7 @@ def filter_external_edges(
|
|
|
263
260
|
|
|
264
261
|
return [
|
|
265
262
|
e for e in edges
|
|
266
|
-
if e.
|
|
263
|
+
if e.src in app_symbols or e.dst in app_symbols
|
|
267
264
|
]
|
|
268
265
|
|
|
269
266
|
|
|
@@ -277,11 +274,11 @@ def merge_edges(*edge_lists: list) -> list:
|
|
|
277
274
|
by_key: Dict[Tuple[str, str], PyCallEdge] = {}
|
|
278
275
|
for edges in edge_lists:
|
|
279
276
|
for e in edges:
|
|
280
|
-
k = (e.
|
|
277
|
+
k = (e.src, e.dst)
|
|
281
278
|
if k in by_key:
|
|
282
279
|
cur = by_key[k]
|
|
283
280
|
cur.weight += e.weight
|
|
284
|
-
cur.
|
|
281
|
+
cur.prov = sorted(set(cur.prov) | set(e.prov))
|
|
285
282
|
else:
|
|
286
283
|
by_key[k] = e.model_copy()
|
|
287
284
|
return list(by_key.values())
|
|
@@ -431,14 +431,14 @@ class PyCG:
|
|
|
431
431
|
"""Sum weights of duplicate ``(source, target)`` pairs across shards."""
|
|
432
432
|
merged: Dict[tuple, PyCallEdge] = {}
|
|
433
433
|
for edge in edges:
|
|
434
|
-
key = (edge.
|
|
434
|
+
key = (edge.src, edge.dst)
|
|
435
435
|
if key in merged:
|
|
436
436
|
existing = merged[key]
|
|
437
437
|
merged[key] = PyCallEdge(
|
|
438
438
|
source=existing.source,
|
|
439
439
|
target=existing.target,
|
|
440
440
|
weight=existing.weight + edge.weight,
|
|
441
|
-
|
|
441
|
+
prov=existing.prov,
|
|
442
442
|
)
|
|
443
443
|
else:
|
|
444
444
|
merged[key] = edge
|
|
@@ -564,7 +564,7 @@ class PyCG:
|
|
|
564
564
|
edge_counts[(resolver.resolve(src), resolver.resolve(dst))] += 1
|
|
565
565
|
|
|
566
566
|
return [
|
|
567
|
-
PyCallEdge(
|
|
567
|
+
PyCallEdge(src=src, dst=dst, weight=count, prov=["pycg"])
|
|
568
568
|
for (src, dst), count in edge_counts.items()
|
|
569
569
|
]
|
|
570
570
|
|
|
@@ -751,7 +751,7 @@ class PyCG:
|
|
|
751
751
|
try:
|
|
752
752
|
triples = ray.get(fut)
|
|
753
753
|
edges_all.extend(
|
|
754
|
-
PyCallEdge(
|
|
754
|
+
PyCallEdge(src=s, dst=t, weight=w, prov=["pycg"])
|
|
755
755
|
for s, t, w in triples
|
|
756
756
|
)
|
|
757
757
|
except Exception:
|
|
@@ -841,14 +841,14 @@ class PyCG:
|
|
|
841
841
|
# Merge duplicate (source, target) pairs that appear in multiple shards.
|
|
842
842
|
merged: Dict[tuple, PyCallEdge] = {}
|
|
843
843
|
for edge in all_edges:
|
|
844
|
-
key = (edge.
|
|
844
|
+
key = (edge.src, edge.dst)
|
|
845
845
|
if key in merged:
|
|
846
846
|
existing = merged[key]
|
|
847
847
|
merged[key] = PyCallEdge(
|
|
848
848
|
source=existing.source,
|
|
849
849
|
target=existing.target,
|
|
850
850
|
weight=existing.weight + edge.weight,
|
|
851
|
-
|
|
851
|
+
prov=existing.prov,
|
|
852
852
|
)
|
|
853
853
|
else:
|
|
854
854
|
merged[key] = edge
|
|
@@ -923,7 +923,7 @@ class PyCG:
|
|
|
923
923
|
try:
|
|
924
924
|
triples = ray.get(fut)
|
|
925
925
|
edges = [
|
|
926
|
-
PyCallEdge(
|
|
926
|
+
PyCallEdge(src=s, dst=t, weight=w, prov=["pycg"])
|
|
927
927
|
for s, t, w in triples
|
|
928
928
|
]
|
|
929
929
|
all_edges.extend(edges)
|
|
@@ -956,14 +956,14 @@ class PyCG:
|
|
|
956
956
|
|
|
957
957
|
merged: Dict[tuple, PyCallEdge] = {}
|
|
958
958
|
for edge in all_edges:
|
|
959
|
-
key = (edge.
|
|
959
|
+
key = (edge.src, edge.dst)
|
|
960
960
|
if key in merged:
|
|
961
961
|
existing = merged[key]
|
|
962
962
|
merged[key] = PyCallEdge(
|
|
963
963
|
source=existing.source,
|
|
964
964
|
target=existing.target,
|
|
965
965
|
weight=existing.weight + edge.weight,
|
|
966
|
-
|
|
966
|
+
prov=existing.prov,
|
|
967
967
|
)
|
|
968
968
|
else:
|
|
969
969
|
merged[key] = edge
|
|
@@ -984,7 +984,7 @@ class PyCG:
|
|
|
984
984
|
symbol_table: Dict[str, PyModule],
|
|
985
985
|
jedi_edges: Optional[List[PyCallEdge]] = None,
|
|
986
986
|
) -> List[PyCallEdge]:
|
|
987
|
-
"""Run PyCG and return ``PyCallEdge`` entries with ``
|
|
987
|
+
"""Run PyCG and return ``PyCallEdge`` entries with ``prov=["pycg"]``.
|
|
988
988
|
|
|
989
989
|
Edges are coalesced on ``(source, target)`` — ``weight`` equals the
|
|
990
990
|
number of times PyCG reports the same (caller, callee) pair (always 1
|
|
@@ -66,17 +66,17 @@ logger = logging.getLogger(__name__)
|
|
|
66
66
|
|
|
67
67
|
def _walk_callable_sigs(c: PyCallable) -> Iterator[str]:
|
|
68
68
|
yield c.signature
|
|
69
|
-
for inner in c.
|
|
69
|
+
for inner in c.callables.values():
|
|
70
70
|
yield from _walk_callable_sigs(inner)
|
|
71
|
-
for inner_cls in c.
|
|
71
|
+
for inner_cls in c.types.values():
|
|
72
72
|
yield from _walk_class_sigs(inner_cls)
|
|
73
73
|
|
|
74
74
|
|
|
75
75
|
def _walk_class_sigs(cls: PyClass) -> Iterator[str]:
|
|
76
76
|
yield cls.signature
|
|
77
|
-
for method in cls.
|
|
77
|
+
for method in cls.callables.values():
|
|
78
78
|
yield from _walk_callable_sigs(method)
|
|
79
|
-
for inner in cls.
|
|
79
|
+
for inner in cls.types.values():
|
|
80
80
|
yield from _walk_class_sigs(inner)
|
|
81
81
|
|
|
82
82
|
|
|
@@ -95,7 +95,7 @@ def _signature_to_file(symbol_table: Dict[str, PyModule]) -> Dict[str, str]:
|
|
|
95
95
|
for fn in module.functions.values():
|
|
96
96
|
for sig in _walk_callable_sigs(fn):
|
|
97
97
|
sig_to_file[sig] = module.file_path
|
|
98
|
-
for cls in module.
|
|
98
|
+
for cls in module.types.values():
|
|
99
99
|
for sig in _walk_class_sigs(cls):
|
|
100
100
|
sig_to_file[sig] = module.file_path
|
|
101
101
|
return sig_to_file
|
|
@@ -152,8 +152,8 @@ def build_module_graph(
|
|
|
152
152
|
g.add_node(module.file_path, module_name=module.module_name)
|
|
153
153
|
|
|
154
154
|
for edge in jedi_edges:
|
|
155
|
-
src = sig_to_file.get(edge.
|
|
156
|
-
dst = sig_to_file.get(edge.
|
|
155
|
+
src = sig_to_file.get(edge.src)
|
|
156
|
+
dst = sig_to_file.get(edge.dst)
|
|
157
157
|
if src is None or dst is None or src == dst:
|
|
158
158
|
continue
|
|
159
159
|
if g.has_edge(src, dst):
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Static resolution of import spellings against the analyzed module set.
|
|
2
|
+
|
|
3
|
+
Pure post-pass over a built ``PyApplication``: no filesystem access, no
|
|
4
|
+
sys.path semantics — a spelling resolves iff it names a module that was
|
|
5
|
+
itself analyzed (issue #82). External/library imports stay unresolved by
|
|
6
|
+
design and keep their :PyPackage projection.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Dict, Optional, Union
|
|
13
|
+
|
|
14
|
+
from codeanalyzer.schema.py_schema import PyApplication
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _dotted_candidates(app: PyApplication, project_dir: Union[Path, str]) -> Dict[str, str]:
|
|
18
|
+
"""dotted module path -> file_key, for every analyzed module.
|
|
19
|
+
|
|
20
|
+
``pkg/util.py`` -> ``pkg.util``; ``pkg/__init__.py`` -> ``pkg``.
|
|
21
|
+
file_keys share project_dir's form (both come from the same CLI arg),
|
|
22
|
+
so os.path.relpath keeps mixed absolute/relative setups consistent.
|
|
23
|
+
"""
|
|
24
|
+
mapping: Dict[str, str] = {}
|
|
25
|
+
for file_key in app.symbol_table:
|
|
26
|
+
rel = os.path.relpath(file_key, str(project_dir))
|
|
27
|
+
if rel.startswith(".."):
|
|
28
|
+
continue
|
|
29
|
+
parts = Path(rel).with_suffix("").parts
|
|
30
|
+
if parts and parts[-1] == "__init__":
|
|
31
|
+
parts = parts[:-1]
|
|
32
|
+
if parts:
|
|
33
|
+
mapping[".".join(parts)] = file_key
|
|
34
|
+
return mapping
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _resolve_one(
|
|
38
|
+
spelling: str, original_name: str, importer_rel_parts: tuple, candidates: Dict[str, str]
|
|
39
|
+
) -> Optional[str]:
|
|
40
|
+
if spelling.startswith("."):
|
|
41
|
+
level = len(spelling) - len(spelling.lstrip("."))
|
|
42
|
+
suffix = spelling.lstrip(".")
|
|
43
|
+
# level 1 = the importer's own package; each extra dot walks one up.
|
|
44
|
+
package_parts = importer_rel_parts[:-1] # drop the filename
|
|
45
|
+
if level - 1 > len(package_parts):
|
|
46
|
+
return None
|
|
47
|
+
base = package_parts[: len(package_parts) - (level - 1)]
|
|
48
|
+
stems = list(base) + (suffix.split(".") if suffix else [])
|
|
49
|
+
else:
|
|
50
|
+
stems = spelling.split(".")
|
|
51
|
+
dotted = ".".join(stems)
|
|
52
|
+
with_name = f"{dotted}.{original_name}" if dotted else original_name
|
|
53
|
+
return candidates.get(with_name) or candidates.get(dotted)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def resolve_imports(app: PyApplication, project_dir: Union[Path, str]) -> None:
|
|
57
|
+
"""Stamp ``resolved_module`` on every import of every module, in place."""
|
|
58
|
+
candidates = _dotted_candidates(app, project_dir)
|
|
59
|
+
for file_key, module in app.symbol_table.items():
|
|
60
|
+
rel = os.path.relpath(file_key, str(project_dir))
|
|
61
|
+
importer_parts = Path(rel).parts
|
|
62
|
+
for im in module.imports or []:
|
|
63
|
+
if not im.module:
|
|
64
|
+
im.resolved_module = None
|
|
65
|
+
continue
|
|
66
|
+
original = im.alias or im.name
|
|
67
|
+
im.resolved_module = _resolve_one(im.module, original, importer_parts, candidates)
|