codeanalyzer-python 1.0.2__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/core.py +161 -8
- codeanalyzer/dataflow/scalpel/README.md +34 -0
- codeanalyzer/dataflow/scalpel/SSA/__init__.py +8 -0
- codeanalyzer/dataflow/scalpel/SSA/const.py +398 -0
- codeanalyzer/dataflow/scalpel/__init__.py +11 -0
- codeanalyzer/dataflow/scalpel/cfg/__init__.py +10 -0
- codeanalyzer/dataflow/scalpel/cfg/builder.py +700 -0
- codeanalyzer/dataflow/scalpel/cfg/model.py +331 -0
- codeanalyzer/dataflow/scalpel/core/__init__.py +0 -0
- codeanalyzer/dataflow/scalpel/core/func_call_visitor.py +234 -0
- codeanalyzer/dataflow/scalpel/core/vars_visitor.py +205 -0
- codeanalyzer/dataflow/scalpel_oracle.py +9 -12
- {codeanalyzer_python-1.0.2.dist-info → codeanalyzer_python-1.1.0.dist-info}/METADATA +12 -14
- {codeanalyzer_python-1.0.2.dist-info → codeanalyzer_python-1.1.0.dist-info}/RECORD +19 -8
- codeanalyzer_python-1.1.0.dist-info/licenses/LICENSE +201 -0
- {codeanalyzer_python-1.0.2.dist-info → codeanalyzer_python-1.1.0.dist-info}/licenses/NOTICE +7 -0
- {codeanalyzer_python-1.0.2.dist-info/licenses → codeanalyzer/dataflow/scalpel}/LICENSE +0 -0
- {codeanalyzer_python-1.0.2.dist-info → codeanalyzer_python-1.1.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.0.2.dist-info → codeanalyzer_python-1.1.0.dist-info}/entry_points.txt +0 -0
codeanalyzer/core.py
CHANGED
|
@@ -163,8 +163,159 @@ class Codeanalyzer:
|
|
|
163
163
|
stderr=None,
|
|
164
164
|
)
|
|
165
165
|
|
|
166
|
+
@classmethod
|
|
167
|
+
def _get_base_interpreter(cls) -> Path:
|
|
168
|
+
"""The interpreter used to provision the analysis virtualenv.
|
|
169
|
+
|
|
170
|
+
jedi parses the *analysis environment's* Python version with parso,
|
|
171
|
+
which ships one hardcoded grammar file per minor version — an
|
|
172
|
+
environment newer than the newest shipped grammar makes every file
|
|
173
|
+
fail with "Python version X.Y is currently not supported" while the
|
|
174
|
+
run still exits 0 (#107). So the default choice is gated on the
|
|
175
|
+
installed parso's ceiling: a too-new default is swapped for the
|
|
176
|
+
newest supported interpreter found on the host, falling back to the
|
|
177
|
+
default (loudly) only when none exists. An explicit ``SYSTEM_PYTHON``
|
|
178
|
+
always wins, with a warning when parso cannot parse its version.
|
|
179
|
+
"""
|
|
180
|
+
# An explicit SYSTEM_PYTHON override wins (consulted only when running
|
|
181
|
+
# inside a virtualenv, matching the historical behavior).
|
|
182
|
+
if sys.prefix != sys.base_prefix:
|
|
183
|
+
system_python = os.getenv("SYSTEM_PYTHON")
|
|
184
|
+
if system_python:
|
|
185
|
+
system_python_path = Path(system_python)
|
|
186
|
+
if system_python_path.exists() and system_python_path.is_file():
|
|
187
|
+
ceiling = cls._parso_supported_ceiling()
|
|
188
|
+
version = cls._interpreter_version(system_python_path)
|
|
189
|
+
if ceiling is not None and version is not None and version > ceiling:
|
|
190
|
+
logger.warning(
|
|
191
|
+
f"SYSTEM_PYTHON={system_python} is Python "
|
|
192
|
+
f"{version[0]}.{version[1]}, newer than the newest grammar "
|
|
193
|
+
f"the installed parso ships ({ceiling[0]}.{ceiling[1]}). "
|
|
194
|
+
"jedi will likely reject every file in the analysis "
|
|
195
|
+
"environment (#107); honoring the explicit override anyway."
|
|
196
|
+
)
|
|
197
|
+
return system_python_path
|
|
198
|
+
|
|
199
|
+
candidate = cls._default_base_interpreter()
|
|
200
|
+
ceiling = cls._parso_supported_ceiling()
|
|
201
|
+
if ceiling is None:
|
|
202
|
+
return candidate
|
|
203
|
+
version = cls._interpreter_version(candidate)
|
|
204
|
+
if version is None or version <= ceiling:
|
|
205
|
+
return candidate
|
|
206
|
+
logger.warning(
|
|
207
|
+
f"Default interpreter {candidate} is Python {version[0]}.{version[1]}, "
|
|
208
|
+
f"newer than the newest grammar the installed parso ships "
|
|
209
|
+
f"({ceiling[0]}.{ceiling[1]}) — looking for a supported interpreter "
|
|
210
|
+
"for the analysis environment (#107)."
|
|
211
|
+
)
|
|
212
|
+
supported = cls._find_supported_interpreter(ceiling)
|
|
213
|
+
if supported is not None:
|
|
214
|
+
logger.info(f"Provisioning the analysis environment with {supported}.")
|
|
215
|
+
return supported
|
|
216
|
+
logger.warning(
|
|
217
|
+
f"No interpreter <= {ceiling[0]}.{ceiling[1]} found on this host; "
|
|
218
|
+
f"falling back to {candidate}. jedi/parso will likely reject every "
|
|
219
|
+
"file — install a supported Python or upgrade parso."
|
|
220
|
+
)
|
|
221
|
+
return candidate
|
|
222
|
+
|
|
223
|
+
@staticmethod
|
|
224
|
+
def _versions_from_grammar_stems(stems: List[str]) -> List[tuple]:
|
|
225
|
+
"""``grammar313`` → ``(3, 13)``, sorted ascending; malformed stems dropped."""
|
|
226
|
+
versions = []
|
|
227
|
+
for stem in stems:
|
|
228
|
+
digits = stem[len("grammar"):]
|
|
229
|
+
if len(digits) >= 2 and digits.isdigit():
|
|
230
|
+
versions.append((int(digits[0]), int(digits[1:])))
|
|
231
|
+
return sorted(versions)
|
|
232
|
+
|
|
233
|
+
@classmethod
|
|
234
|
+
def _parso_supported_ceiling(cls) -> Optional[tuple]:
|
|
235
|
+
"""Newest ``(major, minor)`` the installed parso ships a grammar for,
|
|
236
|
+
derived from its ``python/grammar*.txt`` files so the ceiling moves
|
|
237
|
+
automatically when parso adds a version. ``None`` if undeterminable."""
|
|
238
|
+
try:
|
|
239
|
+
import parso
|
|
240
|
+
|
|
241
|
+
stems = [
|
|
242
|
+
p.stem
|
|
243
|
+
for p in (Path(parso.__file__).parent / "python").glob("grammar*.txt")
|
|
244
|
+
]
|
|
245
|
+
versions = cls._versions_from_grammar_stems(stems)
|
|
246
|
+
return versions[-1] if versions else None
|
|
247
|
+
except Exception:
|
|
248
|
+
return None
|
|
249
|
+
|
|
250
|
+
@staticmethod
|
|
251
|
+
def _interpreter_version(interpreter: Path) -> Optional[tuple]:
|
|
252
|
+
"""``(major, minor)`` of an interpreter, or ``None`` if it can't run."""
|
|
253
|
+
try:
|
|
254
|
+
result = subprocess.run(
|
|
255
|
+
[
|
|
256
|
+
str(interpreter),
|
|
257
|
+
"-c",
|
|
258
|
+
"import sys; print('%d.%d' % sys.version_info[:2])",
|
|
259
|
+
],
|
|
260
|
+
capture_output=True,
|
|
261
|
+
text=True,
|
|
262
|
+
timeout=5,
|
|
263
|
+
)
|
|
264
|
+
if result.returncode == 0:
|
|
265
|
+
major, minor = result.stdout.strip().split(".")
|
|
266
|
+
return (int(major), int(minor))
|
|
267
|
+
except (subprocess.TimeoutExpired, FileNotFoundError, PermissionError, ValueError):
|
|
268
|
+
pass
|
|
269
|
+
return None
|
|
270
|
+
|
|
271
|
+
@staticmethod
|
|
272
|
+
def _pick_supported_interpreter(
|
|
273
|
+
candidates: List[tuple], ceiling: tuple
|
|
274
|
+
) -> Optional[Path]:
|
|
275
|
+
"""Newest candidate whose version is within the ceiling.
|
|
276
|
+
|
|
277
|
+
``candidates`` is ``[(path, (major, minor) | None), ...]``."""
|
|
278
|
+
supported = [
|
|
279
|
+
(version, path)
|
|
280
|
+
for path, version in candidates
|
|
281
|
+
if version is not None and version <= ceiling
|
|
282
|
+
]
|
|
283
|
+
return max(supported)[1] if supported else None
|
|
284
|
+
|
|
285
|
+
@classmethod
|
|
286
|
+
def _find_supported_interpreter(cls, ceiling: tuple) -> Optional[Path]:
|
|
287
|
+
"""Search the host for the newest interpreter within the parso ceiling:
|
|
288
|
+
versioned names on PATH (``python3.13``, ``python3.12``, ...) first,
|
|
289
|
+
then pyenv installs."""
|
|
290
|
+
paths: List[Path] = []
|
|
291
|
+
for minor in range(ceiling[1], 7, -1):
|
|
292
|
+
which = shutil.which(f"python{ceiling[0]}.{minor}")
|
|
293
|
+
# Skip the current virtualenv's own interpreter (same rule as
|
|
294
|
+
# _default_base_interpreter): the analysis env must come from a
|
|
295
|
+
# base installation.
|
|
296
|
+
if which and not which.startswith(sys.prefix):
|
|
297
|
+
paths.append(Path(which))
|
|
298
|
+
for pyenv_root in (os.getenv("PYENV_ROOT"), str(Path.home() / ".pyenv")):
|
|
299
|
+
if not pyenv_root:
|
|
300
|
+
continue
|
|
301
|
+
versions_dir = Path(pyenv_root) / "versions"
|
|
302
|
+
if versions_dir.is_dir():
|
|
303
|
+
for install in sorted(versions_dir.iterdir(), reverse=True):
|
|
304
|
+
exe = install / "bin" / "python3"
|
|
305
|
+
if exe.exists():
|
|
306
|
+
paths.append(exe)
|
|
307
|
+
seen = set()
|
|
308
|
+
candidates = []
|
|
309
|
+
for path in paths:
|
|
310
|
+
key = str(path)
|
|
311
|
+
if key in seen:
|
|
312
|
+
continue
|
|
313
|
+
seen.add(key)
|
|
314
|
+
candidates.append((path, cls._interpreter_version(path)))
|
|
315
|
+
return cls._pick_supported_interpreter(candidates, ceiling)
|
|
316
|
+
|
|
166
317
|
@staticmethod
|
|
167
|
-
def
|
|
318
|
+
def _default_base_interpreter() -> Path:
|
|
168
319
|
"""Get the base Python interpreter path.
|
|
169
320
|
|
|
170
321
|
This method finds a suitable base Python interpreter that can be used
|
|
@@ -183,13 +334,6 @@ class Codeanalyzer:
|
|
|
183
334
|
|
|
184
335
|
# We're inside a virtual environment; need to find the base interpreter
|
|
185
336
|
|
|
186
|
-
# First, check if user explicitly set SYSTEM_PYTHON
|
|
187
|
-
system_python = os.getenv("SYSTEM_PYTHON")
|
|
188
|
-
if system_python:
|
|
189
|
-
system_python_path = Path(system_python)
|
|
190
|
-
if system_python_path.exists() and system_python_path.is_file():
|
|
191
|
-
return system_python_path
|
|
192
|
-
|
|
193
337
|
# Try to get the base interpreter from sys.base_executable (Python 3.3+)
|
|
194
338
|
if hasattr(sys, "base_executable") and sys.base_executable:
|
|
195
339
|
base_exec = Path(sys.base_executable)
|
|
@@ -778,6 +922,15 @@ class Codeanalyzer:
|
|
|
778
922
|
if files_from_cache > 0:
|
|
779
923
|
logger.info(f"Reused {files_from_cache} files from cache, processed {files_processed} new/changed files")
|
|
780
924
|
|
|
925
|
+
if py_files and not symbol_table:
|
|
926
|
+
logger.error(
|
|
927
|
+
"Every one of the %d discovered Python files failed to process — "
|
|
928
|
+
"the symbol table is empty. This usually means the analysis "
|
|
929
|
+
"environment's interpreter is newer than the installed jedi/parso "
|
|
930
|
+
"stack supports (#107); check the per-file errors above.",
|
|
931
|
+
len(py_files),
|
|
932
|
+
)
|
|
933
|
+
|
|
781
934
|
logger.info(
|
|
782
935
|
"✅ Symbol table: %d modules in %.1fs",
|
|
783
936
|
len(symbol_table), time.perf_counter() - t0_st,
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Vendored Scalpel (typed_ast-free slice)
|
|
2
|
+
|
|
3
|
+
Vendored from **[SMAT-Lab/Scalpel](https://github.com/SMAT-Lab/Scalpel)**,
|
|
4
|
+
package `python-scalpel==1.0b0`, licensed **Apache-2.0** (see `LICENSE`).
|
|
5
|
+
|
|
6
|
+
## Why vendored
|
|
7
|
+
|
|
8
|
+
`python-scalpel` hard-depends on `typed_ast`, whose last release (1.5.5) has no
|
|
9
|
+
wheel for Python 3.12+ and fails to build from source on modern compilers — so
|
|
10
|
+
`pip install python-scalpel` fails on 3.12/3.13/3.14. `typed_ast` is imported by
|
|
11
|
+
exactly one scalpel module, `typeinfer/analysers.py`, which codeanalyzer does not
|
|
12
|
+
use. Vendoring the small slice the L4 may-alias oracle needs removes the
|
|
13
|
+
`typed_ast` dependency and makes scalpel the default oracle on every supported
|
|
14
|
+
Python.
|
|
15
|
+
|
|
16
|
+
## What is vendored
|
|
17
|
+
|
|
18
|
+
Exactly the 9-module closure that `scalpel.SSA.const` + `scalpel.cfg` load
|
|
19
|
+
(verified via `sys.modules`; provably free of `typeinfer`/`typed_ast`):
|
|
20
|
+
|
|
21
|
+
__init__.py
|
|
22
|
+
SSA/__init__.py, SSA/const.py
|
|
23
|
+
cfg/__init__.py, cfg/builder.py, cfg/model.py
|
|
24
|
+
core/__init__.py, core/func_call_visitor.py, core/vars_visitor.py
|
|
25
|
+
|
|
26
|
+
Copied verbatim except **one patch**: `cfg/model.py`'s top-level
|
|
27
|
+
`import graphviz as gv` is guarded (`try/except ImportError: gv = None`) so the
|
|
28
|
+
module imports without the `graphviz` package — the graphviz-using
|
|
29
|
+
`build_visual()` methods are unused here.
|
|
30
|
+
|
|
31
|
+
Runtime deps of this slice: `astor`, `networkx` (both core dependencies of
|
|
32
|
+
codeanalyzer). `typed_ast` and `graphviz` are NOT required.
|
|
33
|
+
|
|
34
|
+
To refresh: re-run the vendoring in `docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md` Task 1.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Static Single Assignment (SSA) is a technique of IR in the compiling thoery, it also shows great benefits to static anaysis tasks such as constant propagation, dead code elimination and etc.
|
|
3
|
+
Constant propagation is also a matured technique in static anaysis.
|
|
4
|
+
It is the process of evaluating or recognizing the actual constant values or expressions at a particular program point. This is realized by utilizing control flow and data flow information. Determining the possible values for variables before runtime gives great benefits to software anaysis.
|
|
5
|
+
For instance, with constant value propagation, we can detect and remove dead code or perfrom type checking.
|
|
6
|
+
In scalpel, we implement constant propagation along with the SSA for execution efficiency.
|
|
7
|
+
"""
|
|
8
|
+
__slots__ = ["const"]
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
"""
|
|
2
|
+
In this module, the single static assignment forms are implemented to allow
|
|
3
|
+
further analysis. The module contain a single class named SSA.
|
|
4
|
+
"""
|
|
5
|
+
import ast
|
|
6
|
+
import astor
|
|
7
|
+
from functools import reduce
|
|
8
|
+
from collections import OrderedDict
|
|
9
|
+
import networkx as nx
|
|
10
|
+
from ..core.vars_visitor import get_vars
|
|
11
|
+
|
|
12
|
+
def parse_val(node):
|
|
13
|
+
# does not return anything
|
|
14
|
+
if isinstance(node, ast.Constant):
|
|
15
|
+
return node.value
|
|
16
|
+
if isinstance(node, ast.Str):
|
|
17
|
+
if hasattr(node, "value"):
|
|
18
|
+
return node.value
|
|
19
|
+
else:
|
|
20
|
+
return node.s
|
|
21
|
+
return "other"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SSA:
|
|
25
|
+
"""
|
|
26
|
+
Build SSA graph from a given AST node based on the CFG.
|
|
27
|
+
"""
|
|
28
|
+
def __init__ (self):
|
|
29
|
+
"""
|
|
30
|
+
Args:
|
|
31
|
+
src: the source code as input.
|
|
32
|
+
"""
|
|
33
|
+
# the class SSA takes a module as the input
|
|
34
|
+
self.numbering = {} # numbering variables
|
|
35
|
+
self.var_values = {} # numbering variables
|
|
36
|
+
self.global_live_idents = []
|
|
37
|
+
self.ssa_blocks = []
|
|
38
|
+
self.error_paths = {}
|
|
39
|
+
self.dom = {}
|
|
40
|
+
|
|
41
|
+
self.block_ident_gen = {}
|
|
42
|
+
self.block_ident_use = {}
|
|
43
|
+
self.reachable_table = {}
|
|
44
|
+
id2block = {}
|
|
45
|
+
self.unreachable_names = {}
|
|
46
|
+
self.undefined_names_from = {}
|
|
47
|
+
self.global_names = []
|
|
48
|
+
|
|
49
|
+
def get_attribute_stmts(self, stmts):
|
|
50
|
+
call_stmts = []
|
|
51
|
+
for stmt in stmts:
|
|
52
|
+
if isinstance(stmt,ast.Call) and isinstance(stmt.func, ast.Attribute):
|
|
53
|
+
call_stmts += [stmt]
|
|
54
|
+
|
|
55
|
+
def get_identifiers(self, ast_node):
|
|
56
|
+
"""
|
|
57
|
+
Extract all identifiers from the given AST node.
|
|
58
|
+
Args:
|
|
59
|
+
ast_node: AST node.
|
|
60
|
+
"""
|
|
61
|
+
if ast_node is None:
|
|
62
|
+
return []
|
|
63
|
+
res = get_vars(ast_node)
|
|
64
|
+
idents = [r['name'] for r in res if r['name'] is not None and "." not in r['name']]
|
|
65
|
+
return idents
|
|
66
|
+
|
|
67
|
+
def compute_SSA(self, cfg):
|
|
68
|
+
"""
|
|
69
|
+
Compute single static assignment form representations for a given CFG.
|
|
70
|
+
During the computing, constant value and alias pairs are generated. The following steps are used to compute SSA representations:
|
|
71
|
+
step 1a: compute the dominance frontier
|
|
72
|
+
step 1b: use dominance frontier to place phi node
|
|
73
|
+
if node X contains assignment to a, put phi node for an in dominance frontier of X
|
|
74
|
+
adding phi function may require introducing additional phi function
|
|
75
|
+
start from the entry node
|
|
76
|
+
step2: rename variables so only one definition per name
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
cfg: a control flow graph.
|
|
80
|
+
"""
|
|
81
|
+
# to count how many times a var is defined
|
|
82
|
+
ident_name_counter = {}
|
|
83
|
+
# constant assignment dict
|
|
84
|
+
ident_const_dict = {}
|
|
85
|
+
# step 1a: compute the dominance frontier
|
|
86
|
+
all_blocks = cfg.get_all_blocks()
|
|
87
|
+
id2blocks = {block.id:block for block in all_blocks}
|
|
88
|
+
|
|
89
|
+
block_loaded_idents = {block.id:[] for block in all_blocks}
|
|
90
|
+
block_stored_idents = {block.id:[] for block in all_blocks}
|
|
91
|
+
|
|
92
|
+
block_const_dict = {block.id:[] for block in all_blocks}
|
|
93
|
+
|
|
94
|
+
block_renamed_stored = {block.id:[] for block in all_blocks}
|
|
95
|
+
block_renamed_loaded = {block.id:[] for block in all_blocks}
|
|
96
|
+
|
|
97
|
+
DF = self.compute_DF(all_blocks)
|
|
98
|
+
|
|
99
|
+
for block in all_blocks:
|
|
100
|
+
df_nodes = DF[block.id]
|
|
101
|
+
tmp_const_dict = {}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
for idx, stmt in enumerate(block.statements):
|
|
105
|
+
stmt_const_dict = {}
|
|
106
|
+
stored_idents, loaded_idents, func_names = self.get_stmt_idents_ctx(stmt, const_dict=stmt_const_dict)
|
|
107
|
+
tmp_const_dict[idx] = stmt_const_dict
|
|
108
|
+
block_loaded_idents[block.id] += [loaded_idents]
|
|
109
|
+
block_stored_idents[block.id] += [stored_idents]
|
|
110
|
+
block_renamed_loaded[block.id] += [{ident:set() for ident in loaded_idents}]
|
|
111
|
+
|
|
112
|
+
block_const_dict[block.id] = tmp_const_dict
|
|
113
|
+
|
|
114
|
+
for block in all_blocks:
|
|
115
|
+
stored_idents = block_stored_idents[block.id]
|
|
116
|
+
loaded_idents = block_loaded_idents[block.id]
|
|
117
|
+
n_stmts = len(stored_idents)
|
|
118
|
+
assert (n_stmts == len(loaded_idents))
|
|
119
|
+
affected_idents = []
|
|
120
|
+
tmp_const_dict = block_const_dict[block.id]
|
|
121
|
+
for i in range(n_stmts):
|
|
122
|
+
stmt_stored_idents = stored_idents[i]
|
|
123
|
+
stmt_loaded_idents = loaded_idents[i]
|
|
124
|
+
stmt_renamed_stored = {}
|
|
125
|
+
|
|
126
|
+
for ident in stmt_stored_idents:
|
|
127
|
+
affected_idents.append(ident)
|
|
128
|
+
if ident in ident_name_counter:
|
|
129
|
+
ident_name_counter[ident] += 1
|
|
130
|
+
else:
|
|
131
|
+
ident_name_counter[ident] = 0
|
|
132
|
+
# rename the var name as the number of assignments
|
|
133
|
+
stmt_const_dict = tmp_const_dict[i]
|
|
134
|
+
if ident in stmt_const_dict:
|
|
135
|
+
ident_const_dict[(ident, ident_name_counter[ident])] = stmt_const_dict[ident]
|
|
136
|
+
|
|
137
|
+
stmt_renamed_stored[ident] = ident_name_counter[ident]
|
|
138
|
+
block_renamed_stored[block.id] += [stmt_renamed_stored]
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
#same block, number used identifiers
|
|
142
|
+
for ident in stmt_loaded_idents:
|
|
143
|
+
# a list of dictions for each of idents used in this statement
|
|
144
|
+
phi_loaded_idents = block_renamed_loaded[block.id][i]
|
|
145
|
+
if ident in ident_name_counter:
|
|
146
|
+
phi_loaded_idents[ident].add(ident_name_counter[ident])
|
|
147
|
+
|
|
148
|
+
df_block_ids = DF[block.id]
|
|
149
|
+
for df_block_id in df_block_ids:
|
|
150
|
+
df_block = id2blocks[df_block_id]
|
|
151
|
+
block_ident_gen_produced = []
|
|
152
|
+
df_block_stored_idents = block_stored_idents[df_block_id]
|
|
153
|
+
for af_ident in affected_idents:
|
|
154
|
+
# this for-loop process every statement in the block
|
|
155
|
+
for idx, phi_loaded_idents in enumerate(block_renamed_loaded[df_block_id]):
|
|
156
|
+
block_ident_gen_produced.extend(df_block_stored_idents[idx])
|
|
157
|
+
if af_ident in block_ident_gen_produced:
|
|
158
|
+
continue
|
|
159
|
+
# place phi function here this var used
|
|
160
|
+
# if af_ident has been assigned in this block beforclee this statement, then discard it
|
|
161
|
+
# so theck af_ident has been generated in this block
|
|
162
|
+
if af_ident in phi_loaded_idents:
|
|
163
|
+
phi_loaded_idents[af_ident].add(ident_name_counter[af_ident])
|
|
164
|
+
|
|
165
|
+
return block_renamed_loaded, ident_const_dict
|
|
166
|
+
|
|
167
|
+
def get_stmt_idents_ctx(self, stmt, del_set=[], const_dict = {}):
|
|
168
|
+
"""
|
|
169
|
+
Extract the contextual information of each of identifiers.
|
|
170
|
+
For assignment statements, the assigned values for each of variables will be stored.
|
|
171
|
+
In addition, the del_set will store all deleted variables.
|
|
172
|
+
Args:
|
|
173
|
+
stmt: statement from AST trees.
|
|
174
|
+
del_set: deleted identifiers
|
|
175
|
+
const_dict: a mapping relationship between variables and their assigned values in this statement
|
|
176
|
+
"""
|
|
177
|
+
# if this is a definition of class/function, ignore
|
|
178
|
+
stored_idents = []
|
|
179
|
+
loaded_idents = []
|
|
180
|
+
func_names = []
|
|
181
|
+
# assignment with only one target
|
|
182
|
+
|
|
183
|
+
if isinstance(stmt, ast.Assign):
|
|
184
|
+
targets = stmt.targets
|
|
185
|
+
value = stmt.value
|
|
186
|
+
if len(targets) == 1:
|
|
187
|
+
if hasattr(targets[0], "id"):
|
|
188
|
+
left_name = stmt.targets[0].id
|
|
189
|
+
const_dict[left_name] = stmt.value
|
|
190
|
+
elif isinstance(targets[0], ast.Attribute):
|
|
191
|
+
left_name = astor.to_source(stmt.targets[0]).strip()
|
|
192
|
+
const_dict[left_name] = value
|
|
193
|
+
# multiple targets are represented as tuple
|
|
194
|
+
elif isinstance(targets[0], ast.Tuple):
|
|
195
|
+
# value is also represented as tuple
|
|
196
|
+
if isinstance(value, ast.Tuple):
|
|
197
|
+
for elt, val in zip(targets[0].elts, value.elts):
|
|
198
|
+
if hasattr(elt, "id"):
|
|
199
|
+
left_name = elt.id
|
|
200
|
+
const_dict[left_name] = val
|
|
201
|
+
elif isinstance(targets[0], ast.Attribute):
|
|
202
|
+
#TODO: resolve attributes
|
|
203
|
+
pass
|
|
204
|
+
# value is represented as call
|
|
205
|
+
if isinstance(value, ast.Call):
|
|
206
|
+
for elt in targets[0].elts:
|
|
207
|
+
if hasattr(elt, "id"):
|
|
208
|
+
left_name = elt.id
|
|
209
|
+
const_dict[left_name] = value
|
|
210
|
+
elif isinstance(targets[0], ast.Attribute):
|
|
211
|
+
#TODO: resolve attributes
|
|
212
|
+
pass
|
|
213
|
+
else:
|
|
214
|
+
# Note in some python versions, there are more than one target for an assignment
|
|
215
|
+
# while in some other python versions, multiple targets are deemed as ast.Tuple type in assignment statement
|
|
216
|
+
for target in stmt.targets:
|
|
217
|
+
# this is an assignment to tuple such as a,b = fun()
|
|
218
|
+
# then no valid constant value can be recorded for this statement
|
|
219
|
+
if hasattr(target, "id"):
|
|
220
|
+
left_name = target.id
|
|
221
|
+
const_dict[left_name] = None # TODO: design a type for these kind of values
|
|
222
|
+
elif isinstance(stmt.targets[0], ast.Attribute):
|
|
223
|
+
#TODO: resolve attributes
|
|
224
|
+
pass
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# one target assignment with type annotations
|
|
229
|
+
if isinstance(stmt, ast.AnnAssign):
|
|
230
|
+
if hasattr(stmt.target, "id"):
|
|
231
|
+
left_name = stmt.target.id
|
|
232
|
+
const_dict[left_name] = stmt.value
|
|
233
|
+
elif isinstance(stmt.target, ast.Attribute):
|
|
234
|
+
#TODO: resolve attributes
|
|
235
|
+
pass
|
|
236
|
+
if isinstance(stmt, ast.AugAssign):
|
|
237
|
+
# note here , we need to rewrite this value to its extended form
|
|
238
|
+
# if the statement is "a += 1", then the assigned value should be a+1
|
|
239
|
+
if hasattr(stmt.target, "id"):
|
|
240
|
+
left_name = stmt.target.id
|
|
241
|
+
extended_right = ast.BinOp(ast.Name(left_name, ast.Load()), stmt.op, stmt.value)
|
|
242
|
+
const_dict[left_name] = extended_right
|
|
243
|
+
elif isinstance(stmt.target, ast.Attribute):
|
|
244
|
+
#TODO: resolve attributes
|
|
245
|
+
pass
|
|
246
|
+
if isinstance(stmt, ast.For):
|
|
247
|
+
# there is a variation of assignment in for loop
|
|
248
|
+
# in the case of : for i in [1,2,3]
|
|
249
|
+
# the element of stmt.iter is the value of this assignment
|
|
250
|
+
if hasattr(stmt.target, "id"):
|
|
251
|
+
left_name = stmt.target.id
|
|
252
|
+
iter_value = stmt.iter
|
|
253
|
+
# make a iter call
|
|
254
|
+
#iter_node = ast.Call(ast.Name("iter", ast.Load()), [stmt.iter], [])
|
|
255
|
+
# make a next call
|
|
256
|
+
#next_call_node = ast.Call(ast.Name("next", ast.Load()), [iter_node], [])
|
|
257
|
+
const_dict[left_name] = iter_value
|
|
258
|
+
|
|
259
|
+
elif isinstance(stmt.target, ast.Tuple):
|
|
260
|
+
# to handle for-loop uch as:
|
|
261
|
+
# for x, y in fun():
|
|
262
|
+
for elt in stmt.target.elts:
|
|
263
|
+
if hasattr(elt, "id"):
|
|
264
|
+
const_dict[elt.id] = stmt.iter
|
|
265
|
+
elif isinstance(stmt.target, ast.Attribute):
|
|
266
|
+
#TODO: resolve attributes
|
|
267
|
+
pass
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
272
|
+
stored_idents.append(stmt.name)
|
|
273
|
+
const_dict[stmt.name] = stmt
|
|
274
|
+
func_names.append(stmt.name)
|
|
275
|
+
new_stmt = stmt
|
|
276
|
+
new_stmt.body = []
|
|
277
|
+
ident_info = get_vars(new_stmt)
|
|
278
|
+
for r in ident_info:
|
|
279
|
+
if r['name'] is None:
|
|
280
|
+
continue
|
|
281
|
+
if r['usage'] == "load":
|
|
282
|
+
loaded_idents.append(r['name'])
|
|
283
|
+
return stored_idents, loaded_idents, func_names
|
|
284
|
+
|
|
285
|
+
if isinstance(stmt, ast.ClassDef):
|
|
286
|
+
stored_idents.append(stmt.name)
|
|
287
|
+
const_dict[stmt.name] = None
|
|
288
|
+
func_names.append(stmt.name)
|
|
289
|
+
return stored_idents, loaded_idents, func_names
|
|
290
|
+
|
|
291
|
+
# if this is control flow statements, we should not visit its body to avoid duplicates
|
|
292
|
+
# as they are already in the next blocks
|
|
293
|
+
if isinstance(stmt, (ast.Import, ast.ImportFrom)):
|
|
294
|
+
for alias in stmt.names:
|
|
295
|
+
if alias.asname is None:
|
|
296
|
+
stored_idents += [alias.name.split('.')[0]]
|
|
297
|
+
else:
|
|
298
|
+
stored_idents += [alias.asname.split('.')[0]]
|
|
299
|
+
return stored_idents, loaded_idents, []
|
|
300
|
+
|
|
301
|
+
if isinstance(stmt, (ast.Try)):
|
|
302
|
+
for handler in stmt.handlers:
|
|
303
|
+
if handler.name is not None:
|
|
304
|
+
stored_idents.append(handler.name)
|
|
305
|
+
|
|
306
|
+
if isinstance(handler.type, ast.Name):
|
|
307
|
+
loaded_idents.append(handler.type.id)
|
|
308
|
+
elif isinstance(handler.type, ast.Attribute) and isinstance(handler.type.value, ast.Name):
|
|
309
|
+
loaded_idents.append(handler.type.value.id)
|
|
310
|
+
return stored_idents, loaded_idents, []
|
|
311
|
+
if isinstance(stmt, ast.Global):
|
|
312
|
+
for name in stmt.names:
|
|
313
|
+
self.global_names.append(name)
|
|
314
|
+
return stored_idents, loaded_idents, []
|
|
315
|
+
|
|
316
|
+
visit_node = stmt
|
|
317
|
+
|
|
318
|
+
if isinstance(visit_node,(ast.If, ast.IfExp)):
|
|
319
|
+
# visit_node.body = []
|
|
320
|
+
# visit_node.orlse=[]
|
|
321
|
+
visit_node = stmt.test
|
|
322
|
+
|
|
323
|
+
elif isinstance(visit_node, (ast.With)):
|
|
324
|
+
visit_node.body = []
|
|
325
|
+
visit_node.orlse=[]
|
|
326
|
+
|
|
327
|
+
elif isinstance(visit_node, (ast.While)):
|
|
328
|
+
visit_node.body = []
|
|
329
|
+
|
|
330
|
+
elif isinstance(visit_node, (ast.For)):
|
|
331
|
+
visit_node.body = []
|
|
332
|
+
|
|
333
|
+
elif isinstance(visit_node, ast.Return):
|
|
334
|
+
# imaginary variable
|
|
335
|
+
stored_idents.append("<ret>")
|
|
336
|
+
const_dict["<ret>"] = visit_node.value
|
|
337
|
+
elif isinstance(visit_node, ast.Yield):
|
|
338
|
+
# imaginary variable
|
|
339
|
+
stored_idents.append("<ret>")
|
|
340
|
+
const_dict["<ret>"] = visit_node.value
|
|
341
|
+
|
|
342
|
+
ident_info = get_vars(visit_node)
|
|
343
|
+
for r in ident_info:
|
|
344
|
+
if r['name'] is None or "_hidden_" in r['name']:
|
|
345
|
+
continue
|
|
346
|
+
if r['usage'] == 'store':
|
|
347
|
+
stored_idents.append(r['name'])
|
|
348
|
+
else:
|
|
349
|
+
loaded_idents.append(r['name'])
|
|
350
|
+
if r['usage'] == 'del':
|
|
351
|
+
del_set.append(r['name'])
|
|
352
|
+
return stored_idents, loaded_idents, []
|
|
353
|
+
|
|
354
|
+
def to_json(self):
|
|
355
|
+
pass
|
|
356
|
+
|
|
357
|
+
def print_block(self, block):
|
|
358
|
+
return block.get_source()
|
|
359
|
+
|
|
360
|
+
# compute the dominators
|
|
361
|
+
def compute_idom(self, ssa_blocks):
|
|
362
|
+
"""
|
|
363
|
+
Compute immediate dominators for each of blocks
|
|
364
|
+
Args:
|
|
365
|
+
ssa_blocks: blocks from a control flow graph.
|
|
366
|
+
"""
|
|
367
|
+
# construct the Graph
|
|
368
|
+
entry_block = ssa_blocks[0]
|
|
369
|
+
G = nx.DiGraph()
|
|
370
|
+
for block in ssa_blocks:
|
|
371
|
+
G.add_node(block.id)
|
|
372
|
+
exits = block.exits
|
|
373
|
+
preds = block.predecessors
|
|
374
|
+
for link in preds+exits:
|
|
375
|
+
G.add_edge(link.source.id, link.target.id)
|
|
376
|
+
# DF = nx.dominance_frontiers(G, entry_block.id)
|
|
377
|
+
idom = nx.immediate_dominators(G, entry_block.id)
|
|
378
|
+
return idom
|
|
379
|
+
|
|
380
|
+
# compute dominance frontiers
|
|
381
|
+
def compute_DF(self, ssa_blocks):
|
|
382
|
+
"""
|
|
383
|
+
Compute dominating frontiers for each of blocks
|
|
384
|
+
Args:
|
|
385
|
+
ssa_blocks: blocks from a control flow graph.
|
|
386
|
+
"""
|
|
387
|
+
# construct the Graph
|
|
388
|
+
entry_block = ssa_blocks[0]
|
|
389
|
+
G = nx.DiGraph()
|
|
390
|
+
for block in ssa_blocks:
|
|
391
|
+
G.add_node(block.id)
|
|
392
|
+
exits = block.exits
|
|
393
|
+
preds = block.predecessors
|
|
394
|
+
for link in preds+exits:
|
|
395
|
+
G.add_edge(link.source.id, link.target.id)
|
|
396
|
+
DF = nx.dominance_frontiers(G, entry_block.id)
|
|
397
|
+
#idom = nx.immediate_dominators(G, entry_block.id)
|
|
398
|
+
return DF
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Static Anaysis for Python Programs
|
|
3
|
+
==================================
|
|
4
|
+
Scalpel is a Python library integrating classical program anaysis algorithms
|
|
5
|
+
with tailored features for Python language. It aims to provide simple and
|
|
6
|
+
efficient solutions to software engineering researchers that are accessible to
|
|
7
|
+
everybody and reusable in various contexts.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__all__ = ["cfg", "call_graph", "SSA", "core", "typeinfer", "import_graph", "rewriter"]
|
|
11
|
+
__version__ = '1.0dev'
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The control-flow graph(CFG) is an essential component in static flow analysis with applications such as program
|
|
3
|
+
optimization and taint analysis.
|
|
4
|
+
scalpel.cfg module is used to construct the control flow graph for given python programs. The basic unit in the CFG,
|
|
5
|
+
Block, contains a list of sequential statements that can be executed in a program without any control jumps. The Blocks
|
|
6
|
+
are linked by Link objects, which represent control flow jumps between two blocks and contain the jump conditions in
|
|
7
|
+
the form of an expression. Please see the example diagram a control flow graph 
|
|
8
|
+
"""
|
|
9
|
+
from .builder import CFGBuilder
|
|
10
|
+
from .model import Block, Link, CFG
|