codeanalyzer-python 0.3.1__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +51 -4
- codeanalyzer/core.py +153 -82
- codeanalyzer/dataflow/__init__.py +35 -0
- codeanalyzer/dataflow/access_paths.py +563 -0
- codeanalyzer/dataflow/alias.py +93 -0
- codeanalyzer/dataflow/builder.py +688 -0
- codeanalyzer/dataflow/cfg.py +605 -0
- codeanalyzer/dataflow/defuse.py +113 -0
- codeanalyzer/dataflow/dominance.py +140 -0
- codeanalyzer/dataflow/identity.py +91 -0
- codeanalyzer/dataflow/pdg.py +100 -0
- codeanalyzer/dataflow/scalpel_oracle.py +269 -0
- codeanalyzer/dataflow/scc.py +91 -0
- codeanalyzer/dataflow/sdg.py +424 -0
- codeanalyzer/dataflow/slicing.py +93 -0
- codeanalyzer/dataflow/summaries.py +217 -0
- codeanalyzer/dataflow/syntactic.py +26 -0
- codeanalyzer/neo4j/bolt.py +19 -4
- codeanalyzer/neo4j/cypher.py +9 -3
- codeanalyzer/neo4j/emit.py +8 -3
- codeanalyzer/neo4j/project.py +241 -60
- codeanalyzer/neo4j/rows.py +18 -15
- codeanalyzer/neo4j/schema.py +43 -7
- codeanalyzer/options/options.py +4 -0
- codeanalyzer/schema/__init__.py +19 -0
- codeanalyzer/schema/assign_ids.py +37 -0
- codeanalyzer/schema/call_graph_ids.py +12 -0
- codeanalyzer/schema/ids.py +23 -0
- codeanalyzer/schema/l1_body.py +29 -0
- codeanalyzer/schema/l2_callees.py +36 -0
- codeanalyzer/schema/py_schema.py +141 -30
- codeanalyzer/semantic_analysis/call_graph.py +24 -27
- codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +10 -10
- codeanalyzer/semantic_analysis/pycg/shard_planner.py +7 -7
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +29 -10
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/METADATA +230 -43
- codeanalyzer_python-1.0.0.dist-info/RECORD +59 -0
- codeanalyzer_python-0.3.1.dist-info/RECORD +0 -39
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-0.3.1.dist-info → codeanalyzer_python-1.0.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Walk the symbol-table tree and stamp every node with its can:// id."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
from typing import Dict
|
|
4
|
+
from codeanalyzer.schema import ids
|
|
5
|
+
from codeanalyzer.schema.py_schema import PyApplication, PyModule, PyClass, PyCallable
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def assign_ids(app: PyApplication, app_name: str) -> Dict[str, str]:
|
|
9
|
+
"""Sets `.id` on the app + every module/class/callable. Returns a
|
|
10
|
+
`signature -> can://id` map for later stages (identity layer input)."""
|
|
11
|
+
app.id = ids.application_id(app_name); app.kind = "application"
|
|
12
|
+
sig_to_id: Dict[str, str] = {}
|
|
13
|
+
|
|
14
|
+
def do_callable(parent_id: str, c: PyCallable) -> None:
|
|
15
|
+
seg = ids.callable_sig_segment(c.name, [p.name for p in c.parameters])
|
|
16
|
+
c.id = ids.child_id(parent_id, seg)
|
|
17
|
+
sig_to_id[c.signature] = c.id
|
|
18
|
+
for ic in (c.callables or {}).values():
|
|
19
|
+
do_callable(c.id, ic)
|
|
20
|
+
for icl in (c.types or {}).values():
|
|
21
|
+
do_class(c.id, icl)
|
|
22
|
+
|
|
23
|
+
def do_class(parent_id: str, cl: PyClass) -> None:
|
|
24
|
+
cl.id = ids.child_id(parent_id, cl.name); cl.kind = "class"
|
|
25
|
+
sig_to_id[cl.signature] = cl.id
|
|
26
|
+
for m in (cl.callables or {}).values():
|
|
27
|
+
do_callable(cl.id, m)
|
|
28
|
+
for ic in (cl.types or {}).values():
|
|
29
|
+
do_class(cl.id, ic)
|
|
30
|
+
|
|
31
|
+
for file_key, mod in app.symbol_table.items():
|
|
32
|
+
mod.id = ids.module_id(app_name, file_key); mod.kind = "module"
|
|
33
|
+
for fn in (mod.functions or {}).values():
|
|
34
|
+
do_callable(mod.id, fn)
|
|
35
|
+
for cl in (mod.types or {}).values():
|
|
36
|
+
do_class(mod.id, cl)
|
|
37
|
+
return sig_to_id
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Re-identify call-graph edge endpoints onto canonical can:// ids so the JSON
|
|
2
|
+
call_graph agrees with the Neo4j PY_CALLS projection. Declared endpoints map
|
|
3
|
+
through sig_to_id; external/library endpoints keep their dotted signature
|
|
4
|
+
(they have no can:// id)."""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
from codeanalyzer.schema.py_schema import PyApplication
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def reidentify_call_graph(app: PyApplication, sig_to_id: dict) -> None:
|
|
10
|
+
for edge in app.call_graph or []:
|
|
11
|
+
edge.src = sig_to_id.get(edge.src, edge.src)
|
|
12
|
+
edge.dst = sig_to_id.get(edge.dst, edge.dst)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Canonical `can://` id construction for schema v2 (durable ids, ≥ callable).
|
|
2
|
+
Ordinal ids (< callable) are `ordinal_id(callable_id, tag)`. Pure functions;
|
|
3
|
+
ids are opaque handles (the <file> segment itself contains '/')."""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
from typing import List
|
|
6
|
+
|
|
7
|
+
_SCHEME = "can://python"
|
|
8
|
+
|
|
9
|
+
def application_id(app_name: str) -> str:
|
|
10
|
+
return f"{_SCHEME}/{app_name}"
|
|
11
|
+
|
|
12
|
+
def module_id(app_name: str, file_key: str) -> str:
|
|
13
|
+
rel = file_key.replace("\\", "/").lstrip("./")
|
|
14
|
+
return f"{application_id(app_name)}/{rel}"
|
|
15
|
+
|
|
16
|
+
def child_id(parent_id: str, segment: str) -> str:
|
|
17
|
+
return f"{parent_id}/{segment}"
|
|
18
|
+
|
|
19
|
+
def callable_sig_segment(name: str, param_names: List[str]) -> str:
|
|
20
|
+
return f"{name}({','.join(param_names)})"
|
|
21
|
+
|
|
22
|
+
def ordinal_id(callable_id: str, tag: str) -> str:
|
|
23
|
+
return f"{callable_id}@{tag}"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""L1 body population: materialize `call` nodes from existing call sites.
|
|
2
|
+
`callee` is left None here — the sanctioned null→id refinement happens at L2."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
from codeanalyzer.schema.py_schema import PyApplication, PyClass, PyCallable, BodyNode, Span, byte_offsets
|
|
5
|
+
|
|
6
|
+
def _do_callable(source: str, c: PyCallable) -> None:
|
|
7
|
+
for cs in c.call_sites or []:
|
|
8
|
+
key = f"{cs.start_line}:{cs.start_column}"
|
|
9
|
+
span = Span(start=(cs.start_line, cs.start_column),
|
|
10
|
+
end=(cs.end_line, cs.end_column),
|
|
11
|
+
bytes=byte_offsets(source, cs.start_line, cs.start_column, cs.end_line, cs.end_column)) if source else None
|
|
12
|
+
c.body[key] = BodyNode(kind="call", span=span, callee=None)
|
|
13
|
+
for ic in (c.callables or {}).values():
|
|
14
|
+
_do_callable(source, ic)
|
|
15
|
+
for icl in (c.types or {}).values():
|
|
16
|
+
_do_class(source, icl)
|
|
17
|
+
|
|
18
|
+
def _do_class(source: str, cl: PyClass) -> None:
|
|
19
|
+
for m in (cl.callables or {}).values():
|
|
20
|
+
_do_callable(source, m)
|
|
21
|
+
for ic in (cl.types or {}).values():
|
|
22
|
+
_do_class(source, ic)
|
|
23
|
+
|
|
24
|
+
def populate_l1_body(app: PyApplication) -> None:
|
|
25
|
+
for mod in app.symbol_table.values():
|
|
26
|
+
for fn in (mod.functions or {}).values():
|
|
27
|
+
_do_callable(mod.source, fn)
|
|
28
|
+
for cl in (mod.types or {}).values():
|
|
29
|
+
_do_class(mod.source, cl)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""L2 refinement: fill each L1 `call` body node's `callee` (null→id) from the
|
|
2
|
+
call site's resolved signature — the one sanctioned value change. A declared
|
|
3
|
+
target becomes its can:// id; an external/library target keeps its dotted
|
|
4
|
+
signature; an unresolved call site leaves `callee` absent."""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
from codeanalyzer.schema.py_schema import PyApplication, PyClass, PyCallable
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _do_callable(c: PyCallable, sig_to_id: dict) -> None:
|
|
10
|
+
for cs in c.call_sites or []:
|
|
11
|
+
if cs.callee_signature is None:
|
|
12
|
+
continue
|
|
13
|
+
key = f"{cs.start_line}:{cs.start_column}"
|
|
14
|
+
node = c.body.get(key)
|
|
15
|
+
if node is None or node.kind != "call":
|
|
16
|
+
continue
|
|
17
|
+
node.callee = sig_to_id.get(cs.callee_signature, cs.callee_signature)
|
|
18
|
+
for ic in (c.callables or {}).values():
|
|
19
|
+
_do_callable(ic, sig_to_id)
|
|
20
|
+
for icl in (c.types or {}).values():
|
|
21
|
+
_do_class(icl, sig_to_id)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _do_class(cl: PyClass, sig_to_id: dict) -> None:
|
|
25
|
+
for m in (cl.callables or {}).values():
|
|
26
|
+
_do_callable(m, sig_to_id)
|
|
27
|
+
for ic in (cl.types or {}).values():
|
|
28
|
+
_do_class(ic, sig_to_id)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def backfill_callees(app: PyApplication, sig_to_id: dict) -> None:
|
|
32
|
+
for mod in app.symbol_table.values():
|
|
33
|
+
for fn in (mod.functions or {}).values():
|
|
34
|
+
_do_callable(fn, sig_to_id)
|
|
35
|
+
for cl in (mod.types or {}).values():
|
|
36
|
+
_do_class(cl, sig_to_id)
|
codeanalyzer/schema/py_schema.py
CHANGED
|
@@ -20,9 +20,8 @@ This module defines the data models used to represent Python code structures
|
|
|
20
20
|
for static analysis purposes.
|
|
21
21
|
"""
|
|
22
22
|
from __future__ import annotations
|
|
23
|
-
import inspect
|
|
24
23
|
from pathlib import Path
|
|
25
|
-
from typing import Any, Dict, List, Optional
|
|
24
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
26
25
|
import gzip
|
|
27
26
|
|
|
28
27
|
from pydantic import BaseModel
|
|
@@ -120,12 +119,23 @@ def builder(cls):
|
|
|
120
119
|
# Get type hints and default values for the fields in the model.
|
|
121
120
|
# For example, {file_path: Path, module_name: str, imports: List[PyImport], ...}
|
|
122
121
|
annotations = cls.__annotations__
|
|
123
|
-
# Get default values for the fields in the model.
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
122
|
+
# Get default values for the fields in the model. `inspect.signature` is
|
|
123
|
+
# unreliable for models carrying forward references (e.g. PyCallable's
|
|
124
|
+
# self-referential ``callables``): Pydantic falls back to a generic
|
|
125
|
+
# ``(**data)`` signature that drops the per-field defaults, so the builder
|
|
126
|
+
# would seed those fields with ``None`` and fail validation. Read the declared
|
|
127
|
+
# defaults straight off the model instead. Required fields are intentionally
|
|
128
|
+
# omitted (seeded ``None``) — the builder chain must set them.
|
|
129
|
+
defaults = {}
|
|
130
|
+
model_fields = getattr(cls, "model_fields", None) # Pydantic v2
|
|
131
|
+
if model_fields:
|
|
132
|
+
for name, field in model_fields.items():
|
|
133
|
+
if not field.is_required():
|
|
134
|
+
defaults[name] = field.get_default(call_default_factory=True)
|
|
135
|
+
else: # Pydantic v1
|
|
136
|
+
for name, field in getattr(cls, "__fields__", {}).items():
|
|
137
|
+
if not field.required:
|
|
138
|
+
defaults[name] = field.get_default()
|
|
129
139
|
# Create a namespace for the builder class.
|
|
130
140
|
namespace = {}
|
|
131
141
|
|
|
@@ -168,6 +178,71 @@ def builder(cls):
|
|
|
168
178
|
return cls
|
|
169
179
|
|
|
170
180
|
|
|
181
|
+
def byte_offsets(source: str, start_line: int, start_col: int,
|
|
182
|
+
end_line: int, end_col: int) -> Tuple[int, int]:
|
|
183
|
+
"""Convert (1-based line, 0-based col) ast positions to utf-8 byte offsets
|
|
184
|
+
into `source`. `col` is a character offset within the line (ast semantics);
|
|
185
|
+
we re-encode the line prefix to bytes so multibyte chars are handled."""
|
|
186
|
+
lines = source.splitlines(keepends=True)
|
|
187
|
+
def offset(line: int, col: int) -> int:
|
|
188
|
+
prefix_bytes = len("".join(lines[: line - 1]).encode("utf-8"))
|
|
189
|
+
col_bytes = len(lines[line - 1][:col].encode("utf-8")) if line - 1 < len(lines) else 0
|
|
190
|
+
return prefix_bytes + col_bytes
|
|
191
|
+
return offset(start_line, start_col), offset(end_line, end_col)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
@builder
|
|
195
|
+
@msgpk
|
|
196
|
+
class Span(BaseModel):
|
|
197
|
+
"""Where a node lives in source. `start`/`end` are [line, col] (1-based line,
|
|
198
|
+
0-based col, ast semantics); `bytes` are utf-8 offsets into module.source."""
|
|
199
|
+
start: Tuple[int, int]
|
|
200
|
+
end: Tuple[int, int]
|
|
201
|
+
bytes: Tuple[int, int]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@builder
|
|
205
|
+
@msgpk
|
|
206
|
+
class BodyNode(BaseModel):
|
|
207
|
+
"""A node in a callable's `body`: an AST region (statement/call/branch/…) or
|
|
208
|
+
a synthetic analysis vertex (entry/exit/formal_in/out/actual_in/out)."""
|
|
209
|
+
kind: str
|
|
210
|
+
span: Optional[Span] = None
|
|
211
|
+
callee: Optional[str] = None # only on `call` nodes; the sanctioned null→id slot
|
|
212
|
+
of: Optional[str] = None # param vertices: the variable/return they carry
|
|
213
|
+
parent: Optional[str] = None # actuals: owning callsite ordinal id
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@builder
|
|
217
|
+
@msgpk
|
|
218
|
+
class CfgEdge(BaseModel):
|
|
219
|
+
src: str; dst: str; kind: str = "fallthrough"
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
@builder
|
|
223
|
+
@msgpk
|
|
224
|
+
class CdgEdge(BaseModel):
|
|
225
|
+
src: str; dst: str
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@builder
|
|
229
|
+
@msgpk
|
|
230
|
+
class DdgEdge(BaseModel):
|
|
231
|
+
src: str; dst: str; var: Optional[str] = None; prov: List[str] = []
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@builder
|
|
235
|
+
@msgpk
|
|
236
|
+
class SummaryEdge(BaseModel):
|
|
237
|
+
src: str; dst: str
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
@builder
|
|
241
|
+
@msgpk
|
|
242
|
+
class ParamEdge(BaseModel):
|
|
243
|
+
src: str; dst: str
|
|
244
|
+
|
|
245
|
+
|
|
171
246
|
@builder
|
|
172
247
|
@msgpk
|
|
173
248
|
class PyImport(BaseModel):
|
|
@@ -217,7 +292,10 @@ class PyVariableDeclaration(BaseModel):
|
|
|
217
292
|
"""Represents a Python variable declaration."""
|
|
218
293
|
|
|
219
294
|
name: str
|
|
220
|
-
|
|
295
|
+
# Optional WITH a default: emission drops None (exclude_none), so a
|
|
296
|
+
# required-but-nullable field would make the emitted JSON fail its own
|
|
297
|
+
# model's validation whenever the type is uninferred.
|
|
298
|
+
type: Optional[str] = None
|
|
221
299
|
initializer: Optional[str] = None
|
|
222
300
|
value: Optional[Any] = None
|
|
223
301
|
scope: Literal["module", "class", "function"] = "module"
|
|
@@ -281,20 +359,27 @@ class PyCallable(BaseModel):
|
|
|
281
359
|
name: str
|
|
282
360
|
path: str
|
|
283
361
|
signature: str # e.g., module.<class_name>.function_name
|
|
362
|
+
id: str = ""
|
|
363
|
+
kind: str = "function"
|
|
364
|
+
span: Optional[Span] = None
|
|
284
365
|
comments: List[PyComment] = []
|
|
285
366
|
decorators: List[str] = []
|
|
286
367
|
parameters: List[PyCallableParameter] = []
|
|
287
368
|
return_type: Optional[str] = None
|
|
288
|
-
code: str = None
|
|
289
369
|
start_line: int = -1
|
|
290
370
|
end_line: int = -1
|
|
291
371
|
code_start_line: int = -1
|
|
292
372
|
accessed_symbols: List[PySymbol] = []
|
|
293
373
|
call_sites: List[PyCallsite] = []
|
|
294
|
-
|
|
295
|
-
|
|
374
|
+
callables: Dict[str, "PyCallable"] = {} # nested callables (closures)
|
|
375
|
+
types: Dict[str, "PyClass"] = {} # nested (local) classes
|
|
296
376
|
local_variables: List[PyVariableDeclaration] = []
|
|
297
377
|
cyclomatic_complexity: int = 0
|
|
378
|
+
body: Dict[str, BodyNode] = {}
|
|
379
|
+
cfg: List[CfgEdge] = []
|
|
380
|
+
cdg: List[CdgEdge] = []
|
|
381
|
+
ddg: List[DdgEdge] = []
|
|
382
|
+
summary: List[SummaryEdge] = []
|
|
298
383
|
|
|
299
384
|
def __hash__(self) -> int:
|
|
300
385
|
"""Generate a hash based on the callable's signature."""
|
|
@@ -323,12 +408,14 @@ class PyClass(BaseModel):
|
|
|
323
408
|
|
|
324
409
|
name: str
|
|
325
410
|
signature: str # e.g., module.class_name
|
|
411
|
+
id: str = ""
|
|
412
|
+
kind: str = "class"
|
|
413
|
+
span: Optional[Span] = None
|
|
326
414
|
comments: List[PyComment] = []
|
|
327
|
-
code: str = None
|
|
328
415
|
base_classes: List[str] = []
|
|
329
|
-
|
|
416
|
+
callables: Dict[str, PyCallable] = {} # methods, keystone containment name
|
|
330
417
|
attributes: Dict[str, PyClassAttribute] = {}
|
|
331
|
-
|
|
418
|
+
types: Dict[str, "PyClass"] = {} # inner classes, keystone containment name
|
|
332
419
|
start_line: int = -1
|
|
333
420
|
end_line: int = -1
|
|
334
421
|
|
|
@@ -344,9 +431,12 @@ class PyModule(BaseModel):
|
|
|
344
431
|
|
|
345
432
|
file_path: str
|
|
346
433
|
module_name: str
|
|
434
|
+
id: str = ""
|
|
435
|
+
kind: str = "module"
|
|
436
|
+
source: str = ""
|
|
347
437
|
imports: List[PyImport] = []
|
|
348
438
|
comments: List[PyComment] = []
|
|
349
|
-
|
|
439
|
+
types: Dict[str, PyClass] = {} # classes, keystone containment name
|
|
350
440
|
functions: Dict[str, PyCallable] = {}
|
|
351
441
|
variables: List[PyVariableDeclaration] = []
|
|
352
442
|
# Metadata for caching
|
|
@@ -358,29 +448,30 @@ class PyModule(BaseModel):
|
|
|
358
448
|
@builder
|
|
359
449
|
@msgpk
|
|
360
450
|
class PyCallEdge(BaseModel):
|
|
361
|
-
"""Identity-only call-graph edge with weight
|
|
451
|
+
"""Identity-only call-graph edge with weight (keystone shape: the list name
|
|
452
|
+
IS the edge type, so there is no ``type`` field).
|
|
362
453
|
|
|
363
|
-
|
|
364
|
-
``
|
|
365
|
-
``PyCallable`` entries in the symbol table, not a separate vertex type.
|
|
454
|
+
``src`` and ``dst`` are node ids — the caller's ``can://`` id and the
|
|
455
|
+
callee's ``can://`` id (a symbol-table callable or an ``@external`` home).
|
|
366
456
|
Rich per-call metadata (receiver, arguments, location, ...) lives on
|
|
367
457
|
``PyCallsite`` inside the source ``PyCallable.call_sites``.
|
|
368
458
|
"""
|
|
369
459
|
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
type: Literal["CALL_DEP"] = "CALL_DEP"
|
|
460
|
+
src: str # caller callable id
|
|
461
|
+
dst: str # callee callable (or external) id
|
|
373
462
|
weight: int = 1
|
|
374
|
-
|
|
463
|
+
prov: List[Literal["jedi", "pycg", "joern"]] = []
|
|
375
464
|
|
|
376
465
|
|
|
377
466
|
@builder
|
|
378
467
|
@msgpk
|
|
379
468
|
class PyExternalSymbol(BaseModel):
|
|
380
469
|
"""A call-graph target outside the analyzed project -- an imported library or
|
|
381
|
-
builtin member.
|
|
382
|
-
|
|
470
|
+
builtin member. An edge-endpoint id home, not a tree node: keyed in
|
|
471
|
+
``PyApplication.external_symbols`` by its ``can://…/@external/…`` id."""
|
|
383
472
|
|
|
473
|
+
id: str = "" # can://python/<app>/@external/<module>/<name>
|
|
474
|
+
kind: str = "external"
|
|
384
475
|
name: str # the member/short name, e.g. "get" for "requests.get"
|
|
385
476
|
module: Optional[str] = None # best-effort owning module, e.g. "requests"
|
|
386
477
|
|
|
@@ -398,10 +489,12 @@ class PyRepositoryInfo(BaseModel):
|
|
|
398
489
|
@builder
|
|
399
490
|
@msgpk
|
|
400
491
|
class PyAnalyzerInfo(BaseModel):
|
|
401
|
-
"""Which analyzer produced this snapshot, and how it was configured.
|
|
492
|
+
"""Which analyzer produced this snapshot, and how it was configured.
|
|
493
|
+
Lives on the ``Analysis`` envelope (keystone ``analyzer{name,version}``;
|
|
494
|
+
``config`` rides additively)."""
|
|
402
495
|
|
|
403
|
-
name: str
|
|
404
|
-
version: str
|
|
496
|
+
name: str = "codeanalyzer-python"
|
|
497
|
+
version: str = "unknown"
|
|
405
498
|
config: Dict[str, Any] = {}
|
|
406
499
|
|
|
407
500
|
|
|
@@ -411,10 +504,28 @@ class PyApplication(BaseModel):
|
|
|
411
504
|
"""Represents a Python application."""
|
|
412
505
|
|
|
413
506
|
symbol_table: Dict[str, PyModule]
|
|
507
|
+
id: str = ""
|
|
508
|
+
kind: str = "application"
|
|
414
509
|
call_graph: List[PyCallEdge] = []
|
|
415
510
|
# Call-graph endpoints not declared in the symbol table (imported library /
|
|
416
511
|
# builtin members), keyed by signature. Populated by the analyzer so every
|
|
417
512
|
# backend (JSON and Neo4j) shares one authoritative external-symbol set.
|
|
418
513
|
external_symbols: Dict[str, PyExternalSymbol] = {}
|
|
419
|
-
|
|
514
|
+
# Git provenance of the analyzed checkout, captured at analysis time.
|
|
420
515
|
repository: Optional[PyRepositoryInfo] = None
|
|
516
|
+
# Interprocedural parameter-passing edges (formal↔actual); populated at L4.
|
|
517
|
+
param_in: List[ParamEdge] = []
|
|
518
|
+
param_out: List[ParamEdge] = []
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
@builder
|
|
522
|
+
@msgpk
|
|
523
|
+
class Analysis(BaseModel):
|
|
524
|
+
"""v2 payload root: envelope + the application tree node. ``k_limit`` is an
|
|
525
|
+
L3+ envelope key (None below the dataflow levels; exclude_none drops it)."""
|
|
526
|
+
schema_version: str = "2.0.0"
|
|
527
|
+
language: str = "python"
|
|
528
|
+
max_level: int = 1
|
|
529
|
+
k_limit: Optional[int] = None
|
|
530
|
+
analyzer: PyAnalyzerInfo = PyAnalyzerInfo()
|
|
531
|
+
application: PyApplication
|
|
@@ -38,24 +38,24 @@ from codeanalyzer.schema.py_schema import (
|
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
def _walk_class_callables(cls: PyClass) -> Iterator[PyCallable]:
|
|
41
|
-
for method in cls.
|
|
41
|
+
for method in cls.callables.values():
|
|
42
42
|
yield from _walk_callable(method)
|
|
43
|
-
for inner in cls.
|
|
43
|
+
for inner in cls.types.values():
|
|
44
44
|
yield from _walk_class_callables(inner)
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
def _walk_callable(c: PyCallable) -> Iterator[PyCallable]:
|
|
48
48
|
yield c
|
|
49
|
-
for inner in c.
|
|
49
|
+
for inner in c.callables.values():
|
|
50
50
|
yield from _walk_callable(inner)
|
|
51
|
-
for inner_cls in c.
|
|
51
|
+
for inner_cls in c.types.values():
|
|
52
52
|
yield from _walk_class_callables(inner_cls)
|
|
53
53
|
|
|
54
54
|
|
|
55
55
|
def _walk_module_callables(module: PyModule) -> Iterator[PyCallable]:
|
|
56
56
|
for fn in module.functions.values():
|
|
57
57
|
yield from _walk_callable(fn)
|
|
58
|
-
for cls in module.
|
|
58
|
+
for cls in module.types.values():
|
|
59
59
|
yield from _walk_class_callables(cls)
|
|
60
60
|
|
|
61
61
|
|
|
@@ -69,18 +69,18 @@ def iter_callables_in_symbol_table(
|
|
|
69
69
|
|
|
70
70
|
def _walk_classes_in_class(cls: PyClass) -> Iterator[PyClass]:
|
|
71
71
|
yield cls
|
|
72
|
-
for inner in cls.
|
|
72
|
+
for inner in cls.types.values():
|
|
73
73
|
yield from _walk_classes_in_class(inner)
|
|
74
74
|
# Classes can live inside methods (e.g. a factory method that defines
|
|
75
75
|
# a helper class). Recurse through every method's callable subtree.
|
|
76
|
-
for method in cls.
|
|
76
|
+
for method in cls.callables.values():
|
|
77
77
|
yield from _walk_classes_in_callable(method)
|
|
78
78
|
|
|
79
79
|
|
|
80
80
|
def _walk_classes_in_callable(c: PyCallable) -> Iterator[PyClass]:
|
|
81
|
-
for inner_cls in c.
|
|
81
|
+
for inner_cls in c.types.values():
|
|
82
82
|
yield from _walk_classes_in_class(inner_cls)
|
|
83
|
-
for inner in c.
|
|
83
|
+
for inner in c.callables.values():
|
|
84
84
|
yield from _walk_classes_in_callable(inner)
|
|
85
85
|
|
|
86
86
|
|
|
@@ -91,7 +91,7 @@ def iter_classes_in_symbol_table(
|
|
|
91
91
|
inner classes, classes nested in functions, and classes nested in
|
|
92
92
|
class methods."""
|
|
93
93
|
for module in symbol_table.values():
|
|
94
|
-
for cls in module.
|
|
94
|
+
for cls in module.types.values():
|
|
95
95
|
yield from _walk_classes_in_class(cls)
|
|
96
96
|
for fn in module.functions.values():
|
|
97
97
|
yield from _walk_classes_in_callable(fn)
|
|
@@ -118,22 +118,21 @@ def to_digraph(app: PyApplication) -> nx.DiGraph:
|
|
|
118
118
|
added as **ghost** nodes (``callable=None``, ``ghost=True``) so the
|
|
119
119
|
edges are preserved.
|
|
120
120
|
|
|
121
|
-
Edges carry ``
|
|
121
|
+
Edges carry ``weight`` and ``prov`` attributes.
|
|
122
122
|
"""
|
|
123
123
|
g = nx.DiGraph()
|
|
124
124
|
by_sig = callables_by_signature(app)
|
|
125
125
|
for sig, c in by_sig.items():
|
|
126
126
|
g.add_node(sig, callable=c, ghost=False)
|
|
127
127
|
for e in app.call_graph:
|
|
128
|
-
for sig in (e.
|
|
128
|
+
for sig in (e.src, e.dst):
|
|
129
129
|
if sig not in g.nodes:
|
|
130
130
|
g.add_node(sig, callable=None, ghost=True)
|
|
131
131
|
g.add_edge(
|
|
132
|
-
e.
|
|
133
|
-
e.
|
|
134
|
-
type=e.type,
|
|
132
|
+
e.src,
|
|
133
|
+
e.dst,
|
|
135
134
|
weight=e.weight,
|
|
136
|
-
|
|
135
|
+
prov=list(e.prov),
|
|
137
136
|
)
|
|
138
137
|
return g
|
|
139
138
|
|
|
@@ -143,18 +142,16 @@ def from_digraph(g: nx.DiGraph) -> list:
|
|
|
143
142
|
|
|
144
143
|
Only edges are extracted; nodes are not serialized here — they are
|
|
145
144
|
expected to already exist as ``PyCallable`` entries in the symbol
|
|
146
|
-
table. Edge attributes default to
|
|
147
|
-
provenance when missing.
|
|
145
|
+
table. Edge attributes default to weight 1 / empty prov when missing.
|
|
148
146
|
"""
|
|
149
147
|
edges = []
|
|
150
148
|
for src, dst, data in g.edges(data=True):
|
|
151
149
|
edges.append(
|
|
152
150
|
PyCallEdge(
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
type=data.get("type", "CALL_DEP"),
|
|
151
|
+
src=src,
|
|
152
|
+
dst=dst,
|
|
156
153
|
weight=int(data.get("weight", 1)),
|
|
157
|
-
|
|
154
|
+
prov=list(data.get("prov", data.get("provenance", []))),
|
|
158
155
|
)
|
|
159
156
|
)
|
|
160
157
|
return edges
|
|
@@ -183,7 +180,7 @@ def jedi_call_graph_edges(
|
|
|
183
180
|
counts[(caller.signature, site.callee_signature)] += 1
|
|
184
181
|
|
|
185
182
|
return [
|
|
186
|
-
PyCallEdge(
|
|
183
|
+
PyCallEdge(src=src, dst=dst, weight=n, prov=["jedi"])
|
|
187
184
|
for (src, dst), n in counts.items()
|
|
188
185
|
]
|
|
189
186
|
|
|
@@ -255,7 +252,7 @@ def filter_external_edges(
|
|
|
255
252
|
Edges where an app callable calls a library function (or vice-versa) are
|
|
256
253
|
retained; only lib→lib edges are dropped. The app symbol set is built by
|
|
257
254
|
walking every callable in the symbol table recursively (including nested
|
|
258
|
-
functions and closures via ``
|
|
255
|
+
functions and closures via ``callables``) plus every class, so
|
|
259
256
|
PyCG-discovered closure nodes are correctly recognised as app symbols.
|
|
260
257
|
"""
|
|
261
258
|
app_symbols: set = {c.signature for c in iter_callables_in_symbol_table(symbol_table)}
|
|
@@ -263,7 +260,7 @@ def filter_external_edges(
|
|
|
263
260
|
|
|
264
261
|
return [
|
|
265
262
|
e for e in edges
|
|
266
|
-
if e.
|
|
263
|
+
if e.src in app_symbols or e.dst in app_symbols
|
|
267
264
|
]
|
|
268
265
|
|
|
269
266
|
|
|
@@ -277,11 +274,11 @@ def merge_edges(*edge_lists: list) -> list:
|
|
|
277
274
|
by_key: Dict[Tuple[str, str], PyCallEdge] = {}
|
|
278
275
|
for edges in edge_lists:
|
|
279
276
|
for e in edges:
|
|
280
|
-
k = (e.
|
|
277
|
+
k = (e.src, e.dst)
|
|
281
278
|
if k in by_key:
|
|
282
279
|
cur = by_key[k]
|
|
283
280
|
cur.weight += e.weight
|
|
284
|
-
cur.
|
|
281
|
+
cur.prov = sorted(set(cur.prov) | set(e.prov))
|
|
285
282
|
else:
|
|
286
283
|
by_key[k] = e.model_copy()
|
|
287
284
|
return list(by_key.values())
|
|
@@ -431,14 +431,14 @@ class PyCG:
|
|
|
431
431
|
"""Sum weights of duplicate ``(source, target)`` pairs across shards."""
|
|
432
432
|
merged: Dict[tuple, PyCallEdge] = {}
|
|
433
433
|
for edge in edges:
|
|
434
|
-
key = (edge.
|
|
434
|
+
key = (edge.src, edge.dst)
|
|
435
435
|
if key in merged:
|
|
436
436
|
existing = merged[key]
|
|
437
437
|
merged[key] = PyCallEdge(
|
|
438
438
|
source=existing.source,
|
|
439
439
|
target=existing.target,
|
|
440
440
|
weight=existing.weight + edge.weight,
|
|
441
|
-
|
|
441
|
+
prov=existing.prov,
|
|
442
442
|
)
|
|
443
443
|
else:
|
|
444
444
|
merged[key] = edge
|
|
@@ -564,7 +564,7 @@ class PyCG:
|
|
|
564
564
|
edge_counts[(resolver.resolve(src), resolver.resolve(dst))] += 1
|
|
565
565
|
|
|
566
566
|
return [
|
|
567
|
-
PyCallEdge(
|
|
567
|
+
PyCallEdge(src=src, dst=dst, weight=count, prov=["pycg"])
|
|
568
568
|
for (src, dst), count in edge_counts.items()
|
|
569
569
|
]
|
|
570
570
|
|
|
@@ -751,7 +751,7 @@ class PyCG:
|
|
|
751
751
|
try:
|
|
752
752
|
triples = ray.get(fut)
|
|
753
753
|
edges_all.extend(
|
|
754
|
-
PyCallEdge(
|
|
754
|
+
PyCallEdge(src=s, dst=t, weight=w, prov=["pycg"])
|
|
755
755
|
for s, t, w in triples
|
|
756
756
|
)
|
|
757
757
|
except Exception:
|
|
@@ -841,14 +841,14 @@ class PyCG:
|
|
|
841
841
|
# Merge duplicate (source, target) pairs that appear in multiple shards.
|
|
842
842
|
merged: Dict[tuple, PyCallEdge] = {}
|
|
843
843
|
for edge in all_edges:
|
|
844
|
-
key = (edge.
|
|
844
|
+
key = (edge.src, edge.dst)
|
|
845
845
|
if key in merged:
|
|
846
846
|
existing = merged[key]
|
|
847
847
|
merged[key] = PyCallEdge(
|
|
848
848
|
source=existing.source,
|
|
849
849
|
target=existing.target,
|
|
850
850
|
weight=existing.weight + edge.weight,
|
|
851
|
-
|
|
851
|
+
prov=existing.prov,
|
|
852
852
|
)
|
|
853
853
|
else:
|
|
854
854
|
merged[key] = edge
|
|
@@ -923,7 +923,7 @@ class PyCG:
|
|
|
923
923
|
try:
|
|
924
924
|
triples = ray.get(fut)
|
|
925
925
|
edges = [
|
|
926
|
-
PyCallEdge(
|
|
926
|
+
PyCallEdge(src=s, dst=t, weight=w, prov=["pycg"])
|
|
927
927
|
for s, t, w in triples
|
|
928
928
|
]
|
|
929
929
|
all_edges.extend(edges)
|
|
@@ -956,14 +956,14 @@ class PyCG:
|
|
|
956
956
|
|
|
957
957
|
merged: Dict[tuple, PyCallEdge] = {}
|
|
958
958
|
for edge in all_edges:
|
|
959
|
-
key = (edge.
|
|
959
|
+
key = (edge.src, edge.dst)
|
|
960
960
|
if key in merged:
|
|
961
961
|
existing = merged[key]
|
|
962
962
|
merged[key] = PyCallEdge(
|
|
963
963
|
source=existing.source,
|
|
964
964
|
target=existing.target,
|
|
965
965
|
weight=existing.weight + edge.weight,
|
|
966
|
-
|
|
966
|
+
prov=existing.prov,
|
|
967
967
|
)
|
|
968
968
|
else:
|
|
969
969
|
merged[key] = edge
|
|
@@ -984,7 +984,7 @@ class PyCG:
|
|
|
984
984
|
symbol_table: Dict[str, PyModule],
|
|
985
985
|
jedi_edges: Optional[List[PyCallEdge]] = None,
|
|
986
986
|
) -> List[PyCallEdge]:
|
|
987
|
-
"""Run PyCG and return ``PyCallEdge`` entries with ``
|
|
987
|
+
"""Run PyCG and return ``PyCallEdge`` entries with ``prov=["pycg"]``.
|
|
988
988
|
|
|
989
989
|
Edges are coalesced on ``(source, target)`` — ``weight`` equals the
|
|
990
990
|
number of times PyCG reports the same (caller, callee) pair (always 1
|