unidecompiler-plugin-python-pyc 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unidecompiler_plugin_python_pyc/__init__.py +2 -0
- unidecompiler_plugin_python_pyc/lifter.py +1085 -0
- unidecompiler_plugin_python_pyc/plugin.py +47 -0
- unidecompiler_plugin_python_pyc/pyc.py +136 -0
- unidecompiler_plugin_python_pyc/simulation.py +54 -0
- unidecompiler_plugin_python_pyc/support.py +15 -0
- unidecompiler_plugin_python_pyc-0.1.1.dist-info/METADATA +26 -0
- unidecompiler_plugin_python_pyc-0.1.1.dist-info/RECORD +11 -0
- unidecompiler_plugin_python_pyc-0.1.1.dist-info/WHEEL +5 -0
- unidecompiler_plugin_python_pyc-0.1.1.dist-info/entry_points.txt +2 -0
- unidecompiler_plugin_python_pyc-0.1.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1085 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from unidecompiler.core.effects import (
|
|
4
|
+
AssignValue,
|
|
5
|
+
Binary,
|
|
6
|
+
BuildArray,
|
|
7
|
+
BuildConstKeyMap,
|
|
8
|
+
BuildCall,
|
|
9
|
+
BuildMap,
|
|
10
|
+
BuildSet,
|
|
11
|
+
BuildString,
|
|
12
|
+
BuildShapeTest,
|
|
13
|
+
BuildArrayCall,
|
|
14
|
+
CallTopAs,
|
|
15
|
+
Copy,
|
|
16
|
+
DropBelowTop,
|
|
17
|
+
Effect,
|
|
18
|
+
Compare,
|
|
19
|
+
ExceptionMatch,
|
|
20
|
+
Emit,
|
|
21
|
+
ExtendArray,
|
|
22
|
+
FormatTop,
|
|
23
|
+
LoadAttr,
|
|
24
|
+
LoadAttrFromTop,
|
|
25
|
+
LoadSuperAttr,
|
|
26
|
+
LoadItem,
|
|
27
|
+
LoadLocal,
|
|
28
|
+
Invoke,
|
|
29
|
+
InvokeExpanded,
|
|
30
|
+
InvokeMethod,
|
|
31
|
+
InvokeKw,
|
|
32
|
+
Iterate,
|
|
33
|
+
MakeFunctionValue,
|
|
34
|
+
MergeMap,
|
|
35
|
+
Pop,
|
|
36
|
+
Push,
|
|
37
|
+
RaiseTop,
|
|
38
|
+
RaiseWithCause,
|
|
39
|
+
ReraiseTop,
|
|
40
|
+
ReturnTop,
|
|
41
|
+
ReturnVoid,
|
|
42
|
+
StoreAttr,
|
|
43
|
+
StoreLocal,
|
|
44
|
+
StoreMany,
|
|
45
|
+
StoreManyFromPopOrder,
|
|
46
|
+
StoreItemAtDepth,
|
|
47
|
+
StoreItemEffect,
|
|
48
|
+
SetAdd,
|
|
49
|
+
Swap,
|
|
50
|
+
Truthy,
|
|
51
|
+
Unpack,
|
|
52
|
+
UnknownOpcode,
|
|
53
|
+
YieldTop,
|
|
54
|
+
Unary,
|
|
55
|
+
)
|
|
56
|
+
from unidecompiler.core.vm_module import assemble_vm_module
|
|
57
|
+
from unidecompiler.core.ir import (
|
|
58
|
+
Assign,
|
|
59
|
+
BinaryOp,
|
|
60
|
+
CapturedVar,
|
|
61
|
+
Const,
|
|
62
|
+
Expr,
|
|
63
|
+
ExprStmt,
|
|
64
|
+
GetAttr,
|
|
65
|
+
Global,
|
|
66
|
+
MapLiteral,
|
|
67
|
+
Raise,
|
|
68
|
+
SourceRef,
|
|
69
|
+
Var,
|
|
70
|
+
Yield,
|
|
71
|
+
)
|
|
72
|
+
from unidecompiler.core.vm_effect_table import VMEffectRule, VMEffectTable
|
|
73
|
+
from unidecompiler.core.vm_function import (
|
|
74
|
+
VMFunctionSpec,
|
|
75
|
+
recover_vm_function,
|
|
76
|
+
lift_vm_step_function,
|
|
77
|
+
lift_steps,
|
|
78
|
+
)
|
|
79
|
+
from unidecompiler.core.vm_bytecode import VMBytecodeStep
|
|
80
|
+
from unidecompiler.core.vm_hints import VMHint
|
|
81
|
+
from unidecompiler.core.vm_operands import VMDecodedInstruction, VMOperand
|
|
82
|
+
from unidecompiler.core.vm_region import (
|
|
83
|
+
VMLinearState,
|
|
84
|
+
VMRegionCallbacks,
|
|
85
|
+
VMRegionOpcodeClasses,
|
|
86
|
+
VMRegionProfile,
|
|
87
|
+
VMStatefulCallbacks,
|
|
88
|
+
build_hint_region_profile,
|
|
89
|
+
lift_control_region as lift_vm_control_region,
|
|
90
|
+
)
|
|
91
|
+
from unidecompiler.core.vm_structures import (
|
|
92
|
+
vm_unsupported,
|
|
93
|
+
)
|
|
94
|
+
from unidecompiler_plugin_python_pyc.pyc import PycCodeObject, PycExceptionRegion, PycModule
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
BINARY_SYMBOLS = {
|
|
98
|
+
"+": "+",
|
|
99
|
+
"+=": "+",
|
|
100
|
+
"-": "-",
|
|
101
|
+
"-=": "-",
|
|
102
|
+
"*": "*",
|
|
103
|
+
"*=": "*",
|
|
104
|
+
"/": "/",
|
|
105
|
+
"/=": "/",
|
|
106
|
+
"//": "//",
|
|
107
|
+
"//=": "//",
|
|
108
|
+
"%": "%",
|
|
109
|
+
"%=": "%",
|
|
110
|
+
"^": "^",
|
|
111
|
+
"^=": "^",
|
|
112
|
+
"&": "&",
|
|
113
|
+
"&=": "&",
|
|
114
|
+
"|": "|",
|
|
115
|
+
"|=": "|",
|
|
116
|
+
"<<": "<<",
|
|
117
|
+
"<<=": "<<",
|
|
118
|
+
">>": ">>",
|
|
119
|
+
">>=": ">>",
|
|
120
|
+
}
|
|
121
|
+
COMPLEX_OPS = {
|
|
122
|
+
}
|
|
123
|
+
IGNORED_OPS = {
|
|
124
|
+
"RESUME",
|
|
125
|
+
"CACHE",
|
|
126
|
+
"NOP",
|
|
127
|
+
"EXTENDED_ARG",
|
|
128
|
+
"MAKE_CELL",
|
|
129
|
+
"COPY_FREE_VARS",
|
|
130
|
+
}
|
|
131
|
+
PYTHON_REGION_OPCODE_CLASSES = VMRegionOpcodeClasses(
|
|
132
|
+
noise=frozenset({"END_FOR", "POP_TOP", "NOP", "RESUME"}),
|
|
133
|
+
control=frozenset(
|
|
134
|
+
{
|
|
135
|
+
"POP_JUMP_IF_FALSE",
|
|
136
|
+
"POP_JUMP_IF_TRUE",
|
|
137
|
+
"POP_JUMP_IF_NOT_NONE",
|
|
138
|
+
"POP_JUMP_IF_NONE",
|
|
139
|
+
"JUMP_BACKWARD",
|
|
140
|
+
"JUMP_BACKWARD_NO_INTERRUPT",
|
|
141
|
+
"JUMP_FORWARD",
|
|
142
|
+
"FOR_ITER",
|
|
143
|
+
"GET_ITER",
|
|
144
|
+
"GET_AITER",
|
|
145
|
+
"GET_ANEXT",
|
|
146
|
+
"PUSH_EXC_INFO",
|
|
147
|
+
"RERAISE",
|
|
148
|
+
}
|
|
149
|
+
),
|
|
150
|
+
jumps=frozenset({"JUMP_BACKWARD", "JUMP_BACKWARD_NO_INTERRUPT", "JUMP_FORWARD"}),
|
|
151
|
+
forward_jumps=frozenset({"JUMP_FORWARD"}),
|
|
152
|
+
backward_jumps=frozenset({"JUMP_BACKWARD", "JUMP_BACKWARD_NO_INTERRUPT"}),
|
|
153
|
+
iter_starts=frozenset({"GET_ITER"}),
|
|
154
|
+
async_iter_starts=frozenset({"GET_AITER"}),
|
|
155
|
+
conditional_jumps=frozenset(
|
|
156
|
+
{
|
|
157
|
+
"POP_JUMP_IF_FALSE",
|
|
158
|
+
"POP_JUMP_IF_TRUE",
|
|
159
|
+
"POP_JUMP_IF_NOT_NONE",
|
|
160
|
+
"POP_JUMP_IF_NONE",
|
|
161
|
+
}
|
|
162
|
+
),
|
|
163
|
+
cleanup=frozenset({"PUSH_EXC_INFO", "RERAISE", "CHECK_EG_MATCH", "CHECK_EXC_MATCH", "WITH_EXCEPT_START", "END_ASYNC_FOR", "CLEANUP_THROW"}),
|
|
164
|
+
null_jumps=frozenset({"POP_JUMP_IF_NONE"}),
|
|
165
|
+
not_null_jumps=frozenset({"POP_JUMP_IF_NOT_NONE"}),
|
|
166
|
+
truthy_jumps=frozenset({"POP_JUMP_IF_TRUE"}),
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _no_effect(_context, _instruction, _source: SourceRef) -> tuple[Effect, ...]:
|
|
171
|
+
return ()
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _unknown_opcode_effect(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
175
|
+
return (UnknownOpcode(source=source, opcode=str(getattr(instruction, "opname", "")), raw=_raw_instruction_line(instruction)),)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _pop_top(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
179
|
+
return (Pop(source=source, emit_calls=True, allow_missing=True),)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _setup_annotations(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
183
|
+
return (AssignValue(source=source, name="__annotations__", value=MapLiteral(source=source)),)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _load_build_class(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
187
|
+
return (Push(source=source, value=Global(name="build_class", source=source)),)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _import_name(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
191
|
+
return (
|
|
192
|
+
Pop(source=source, count=2, allow_missing=True),
|
|
193
|
+
Push(source=source, value=Global(name=str(instruction.argval), source=source)),
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _import_from(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
198
|
+
return (
|
|
199
|
+
LoadAttrFromTop(
|
|
200
|
+
source=source,
|
|
201
|
+
attr=str(instruction.argval),
|
|
202
|
+
fallback_obj=Global(name="<module>", source=source),
|
|
203
|
+
),
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _load_const(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
208
|
+
return (Push(source=source, value=Const(value=instruction.argval, source=source)),)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _load_name(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
212
|
+
name = str(instruction.argval)
|
|
213
|
+
return (LoadLocal(source=source, name=name, fallback=Global(name=name, source=source)),)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _load_local(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
217
|
+
name = str(instruction.argval)
|
|
218
|
+
return (LoadLocal(source=source, name=name, fallback=Var(name=name, source=source)),)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _load_closure(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
222
|
+
name = str(instruction.argval)
|
|
223
|
+
return (Push(source=source, value=CapturedVar(name=name, source=source)),)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _load_deref(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
227
|
+
name = str(instruction.argval)
|
|
228
|
+
return (Push(source=source, value=CapturedVar(name=name, source=source)),)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _load_fast_pair(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
232
|
+
names = tuple(name for name in str(instruction.argrepr).split(", ") if name)
|
|
233
|
+
return tuple(
|
|
234
|
+
LoadLocal(source=source, name=name, fallback=Var(name=name, source=source))
|
|
235
|
+
for name in names
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _load_global(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
240
|
+
return (Push(source=source, value=Global(name=str(instruction.argval), source=source)),)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _load_from_dict_or_globals(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
244
|
+
name = str(instruction.argval)
|
|
245
|
+
return (
|
|
246
|
+
Pop(source=source, count=1, allow_missing=True),
|
|
247
|
+
Push(source=source, value=Global(name=name, source=source)),
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _make_function(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
252
|
+
return (MakeFunctionValue(source=source, fallback_name=str(instruction.argval or instruction.argrepr or "<function>")),)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _copy(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
256
|
+
return (Copy(source=source, depth=_instruction_count(instruction), allow_missing=True),)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _swap(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
260
|
+
return (Swap(source=source, depth=_instruction_count(instruction), allow_missing=True),)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _load_attr(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
264
|
+
return (LoadAttr(source=source, attr=str(instruction.argval)),)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _load_super_attr(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
268
|
+
attr = str(instruction.argval or instruction.argrepr).split()[0]
|
|
269
|
+
return (LoadSuperAttr(source=source, attr=attr),)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _build_list(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
273
|
+
return (BuildArray(source=source, kind="list", count=_instruction_count(instruction)),)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _build_tuple(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
277
|
+
return (BuildArray(source=source, kind="tuple", count=_instruction_count(instruction)),)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _build_set(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
281
|
+
return (BuildSet(source=source, count=_instruction_count(instruction)),)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _get_iter(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
285
|
+
return (Iterate(source=source),)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _unpack_sequence(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
289
|
+
return (Unpack(source=source, count=_instruction_count(instruction)),)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _compare(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
293
|
+
return (Compare(source=source, op=str(instruction.argval)),)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _contains(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
297
|
+
return (Compare(source=source, op="in", negate=bool(instruction.arg)),)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _is_op(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
301
|
+
return (Compare(source=source, op="is not" if instruction.arg else "is"),)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _binary(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
305
|
+
return (
|
|
306
|
+
Binary(
|
|
307
|
+
source=source,
|
|
308
|
+
op=BINARY_SYMBOLS.get(str(instruction.argrepr), str(instruction.argrepr)),
|
|
309
|
+
semantics="dynamic",
|
|
310
|
+
),
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _unary(op: str):
|
|
315
|
+
def factory(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
316
|
+
return (Unary(source=source, op=op),)
|
|
317
|
+
|
|
318
|
+
return factory
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _store_global(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
322
|
+
name = str(instruction.argval)
|
|
323
|
+
return (StoreLocal(source=source, name=name, target=Var(name=name, source=source), materialize=False),)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _store_deref(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
327
|
+
name = str(instruction.argval)
|
|
328
|
+
return (
|
|
329
|
+
StoreLocal(
|
|
330
|
+
source=source,
|
|
331
|
+
name=name,
|
|
332
|
+
target=CapturedVar(name=name, source=source),
|
|
333
|
+
materialize=True,
|
|
334
|
+
),
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _delete_global(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
339
|
+
return (Pop(source=source, count=1, allow_missing=True),)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _delete_attr(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
343
|
+
return (Pop(source=source, count=1, allow_missing=True),)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _delete_subscr(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
347
|
+
return (Pop(source=source, count=2, allow_missing=True),)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _build_slice(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
351
|
+
return (BuildCall(source=source, arg_count=_instruction_count(instruction), callee=Global(name="slice", source=source)),)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _binary_slice(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
355
|
+
return (
|
|
356
|
+
BuildCall(source=source, arg_count=2, callee=Global(name="slice", source=source)),
|
|
357
|
+
LoadItem(source=source),
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _store_slice(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
362
|
+
return (StoreItemEffect(source=source),)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _list_extend(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
366
|
+
return (ExtendArray(source=source),)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _dict_merge(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
370
|
+
return (MergeMap(source=source, depth=_instruction_count(instruction)),)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _set_update(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
374
|
+
return (InvokeMethod(source=source, attr="update", arg_count=1, depth=1, returns=0),)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _load_locals(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
378
|
+
return (Push(source=source, value=Global(name="locals", source=source)),)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _call_function_ex(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
382
|
+
return (
|
|
383
|
+
InvokeExpanded(
|
|
384
|
+
source=source,
|
|
385
|
+
has_keywords=bool(_instruction.arg),
|
|
386
|
+
),
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _call_intrinsic_2(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
391
|
+
name = str(instruction.argval or instruction.argrepr or "intrinsic")
|
|
392
|
+
return (BuildCall(source=source, arg_count=2, callee=Global(name=name, source=source)),)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _invoke(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
396
|
+
return (Invoke(source=source, arg_count=_instruction_count(instruction)),)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _invoke_kw(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
400
|
+
return (InvokeKw(source=source, arg_count=_instruction_count(instruction)),)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _convert_value(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
404
|
+
return (FormatTop(source=source, converter=str(instruction.argrepr or instruction.argval or "convert")),)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _build_string(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
408
|
+
return (BuildString(source=source, count=_instruction_count(instruction)),)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _call_top_as(name: str):
|
|
412
|
+
def factory(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
413
|
+
return (CallTopAs(source=source, callee_name=name),)
|
|
414
|
+
|
|
415
|
+
return factory
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _build_map(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
419
|
+
return (BuildMap(source=source, count=_instruction_count(instruction)),)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _build_const_key_map(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
423
|
+
return (BuildConstKeyMap(source=source, count=_instruction_count(instruction)),)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _invoke_collection_method(attr: str, *, depth_from_instruction: bool = False):
|
|
427
|
+
def factory(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
428
|
+
return (
|
|
429
|
+
InvokeMethod(
|
|
430
|
+
source=source,
|
|
431
|
+
attr=attr,
|
|
432
|
+
arg_count=1,
|
|
433
|
+
depth=_instruction_count(instruction) if depth_from_instruction else 1,
|
|
434
|
+
returns=0,
|
|
435
|
+
),
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
return factory
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _map_add(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
442
|
+
return (StoreItemAtDepth(source=source, depth=_instruction_count(instruction)),)
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _set_add(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
446
|
+
return (SetAdd(source=source),)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _store_local(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
450
|
+
name = str(instruction.argval)
|
|
451
|
+
return (StoreLocal(source=source, name=name, target=Var(name=name, source=source)),)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _store_many(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
455
|
+
return (StoreManyFromPopOrder(source=source, names=_instruction_names(instruction)),)
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def _store_fast_load_fast(_context, instruction, source: SourceRef) -> tuple[Effect, ...] | None:
|
|
459
|
+
names = _instruction_names(instruction)
|
|
460
|
+
if not names:
|
|
461
|
+
return None
|
|
462
|
+
return (
|
|
463
|
+
StoreMany(source=source, names=(names[0],)),
|
|
464
|
+
*tuple(LoadLocal(source=source, name=name, fallback=Var(name=name, source=source)) for name in names[1:]),
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _return_const(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
469
|
+
return (Push(source=source, value=Const(value=instruction.argval, source=source)), ReturnTop(source=source))
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _raise_varargs(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
473
|
+
if instruction.arg == 0:
|
|
474
|
+
return (ReraiseTop(source=source),)
|
|
475
|
+
if instruction.arg == 2:
|
|
476
|
+
return (RaiseWithCause(source=source),)
|
|
477
|
+
return (RaiseTop(source=source),)
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _yield_value(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
481
|
+
if instruction.arg in {1, 2}:
|
|
482
|
+
return ()
|
|
483
|
+
return (YieldTop(source=source, default=Const(value=None, source=source)),)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _send(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
487
|
+
return (Pop(source=source, count=1, allow_missing=True),)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _unpack_ex(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
|
|
491
|
+
before = instruction.arg if isinstance(instruction.arg, int) else 0
|
|
492
|
+
return (Unpack(source=source, before=before & 0xFF, after=before >> 8),)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
PYTHON_EFFECT_TABLE = VMEffectTable(
|
|
496
|
+
opcode_attr="opname",
|
|
497
|
+
ignored=frozenset(IGNORED_OPS)
|
|
498
|
+
| {
|
|
499
|
+
"RETURN_GENERATOR",
|
|
500
|
+
"PUSH_NULL",
|
|
501
|
+
"END_SEND",
|
|
502
|
+
"INSTRUMENTED_END_SEND",
|
|
503
|
+
"GET_AITER",
|
|
504
|
+
"FOR_ITER",
|
|
505
|
+
"INSTRUMENTED_FOR_ITER",
|
|
506
|
+
"END_FOR",
|
|
507
|
+
"INSTRUMENTED_END_FOR",
|
|
508
|
+
"POP_JUMP_IF_FALSE",
|
|
509
|
+
"POP_JUMP_IF_TRUE",
|
|
510
|
+
"POP_JUMP_IF_NONE",
|
|
511
|
+
"POP_JUMP_IF_NOT_NONE",
|
|
512
|
+
"INSTRUMENTED_POP_JUMP_IF_FALSE",
|
|
513
|
+
"INSTRUMENTED_POP_JUMP_IF_TRUE",
|
|
514
|
+
"INSTRUMENTED_POP_JUMP_IF_NONE",
|
|
515
|
+
"INSTRUMENTED_POP_JUMP_IF_NOT_NONE",
|
|
516
|
+
"JUMP_BACKWARD_NO_INTERRUPT",
|
|
517
|
+
"JUMP_BACKWARD",
|
|
518
|
+
"JUMP_FORWARD",
|
|
519
|
+
"JUMP",
|
|
520
|
+
"JUMP_NO_INTERRUPT",
|
|
521
|
+
"INSTRUMENTED_JUMP_BACKWARD",
|
|
522
|
+
"INSTRUMENTED_JUMP_FORWARD",
|
|
523
|
+
"CALL_INTRINSIC_1",
|
|
524
|
+
"BEFORE_WITH",
|
|
525
|
+
"CLEANUP_THROW",
|
|
526
|
+
"CHECK_EG_MATCH",
|
|
527
|
+
"END_ASYNC_FOR",
|
|
528
|
+
"LOOKUP_METHOD",
|
|
529
|
+
"PUSH_EXC_INFO",
|
|
530
|
+
"WITH_EXCEPT_START",
|
|
531
|
+
"DELETE_FAST",
|
|
532
|
+
"DELETE_NAME",
|
|
533
|
+
"DELETE_DEREF",
|
|
534
|
+
"DELETE_GLOBAL",
|
|
535
|
+
"ENTER_EXECUTOR",
|
|
536
|
+
"EXIT_INIT_CHECK",
|
|
537
|
+
"INSTRUMENTED_INSTRUCTION",
|
|
538
|
+
"INSTRUMENTED_LINE",
|
|
539
|
+
"INSTRUMENTED_RESUME",
|
|
540
|
+
"INTERPRETER_EXIT",
|
|
541
|
+
"POP_BLOCK",
|
|
542
|
+
"RESERVED",
|
|
543
|
+
"SETUP_CLEANUP",
|
|
544
|
+
"SETUP_FINALLY",
|
|
545
|
+
"SETUP_WITH",
|
|
546
|
+
},
|
|
547
|
+
exact={
|
|
548
|
+
"POP_TOP": _pop_top,
|
|
549
|
+
"SETUP_ANNOTATIONS": _setup_annotations,
|
|
550
|
+
"LOAD_BUILD_CLASS": _load_build_class,
|
|
551
|
+
"IMPORT_NAME": _import_name,
|
|
552
|
+
"IMPORT_FROM": _import_from,
|
|
553
|
+
"LOAD_CONST": _load_const,
|
|
554
|
+
"LOAD_NAME": _load_name,
|
|
555
|
+
"LOAD_DEREF": _load_deref,
|
|
556
|
+
"LOAD_CLASSDEREF": _load_deref,
|
|
557
|
+
"LOAD_FROM_DICT_OR_DEREF": _load_deref,
|
|
558
|
+
"LOAD_CLOSURE": _load_closure,
|
|
559
|
+
"LOAD_FAST": _load_local,
|
|
560
|
+
"LOAD_FAST_CHECK": _load_local,
|
|
561
|
+
"LOAD_FAST_AND_CLEAR": _load_local,
|
|
562
|
+
"LOAD_FAST_LOAD_FAST": _load_fast_pair,
|
|
563
|
+
"LOAD_FROM_DICT_OR_GLOBALS": _load_from_dict_or_globals,
|
|
564
|
+
"LOAD_GLOBAL": _load_global,
|
|
565
|
+
"MAKE_FUNCTION": _make_function,
|
|
566
|
+
"STORE_GLOBAL": _store_global,
|
|
567
|
+
"DELETE_GLOBAL": _delete_global,
|
|
568
|
+
"DELETE_ATTR": _delete_attr,
|
|
569
|
+
"DELETE_SUBSCR": _delete_subscr,
|
|
570
|
+
"LOAD_ASSERTION_ERROR": lambda _context, _instruction, source: (Push(source=source, value=Global(name="AssertionError", source=source)),),
|
|
571
|
+
"BUILD_SLICE": _build_slice,
|
|
572
|
+
"STORE_SLICE": _store_slice,
|
|
573
|
+
"LIST_EXTEND": _list_extend,
|
|
574
|
+
"DICT_MERGE": _dict_merge,
|
|
575
|
+
"SET_UPDATE": _set_update,
|
|
576
|
+
"CALL_FUNCTION_EX": _call_function_ex,
|
|
577
|
+
"INSTRUMENTED_CALL_FUNCTION_EX": _call_function_ex,
|
|
578
|
+
"LOAD_LOCALS": _load_locals,
|
|
579
|
+
"SET_FUNCTION_ATTRIBUTE": lambda _context, _instruction, source: (DropBelowTop(source=source, count=1),),
|
|
580
|
+
"COPY": _copy,
|
|
581
|
+
"SWAP": _swap,
|
|
582
|
+
"LOAD_ATTR": _load_attr,
|
|
583
|
+
"LOAD_METHOD": _load_attr,
|
|
584
|
+
"LOAD_SUPER_ATTR": _load_super_attr,
|
|
585
|
+
"LOAD_SUPER_METHOD": _load_super_attr,
|
|
586
|
+
"LOAD_ZERO_SUPER_ATTR": _load_super_attr,
|
|
587
|
+
"LOAD_ZERO_SUPER_METHOD": _load_super_attr,
|
|
588
|
+
"INSTRUMENTED_LOAD_SUPER_ATTR": _load_super_attr,
|
|
589
|
+
"GET_LEN": _call_top_as("len"),
|
|
590
|
+
"MATCH_MAPPING": lambda _context, _instruction, source: (
|
|
591
|
+
BuildShapeTest(source=source, descriptor_count=0),
|
|
592
|
+
),
|
|
593
|
+
"MATCH_SEQUENCE": lambda _context, _instruction, source: (
|
|
594
|
+
BuildShapeTest(source=source, descriptor_count=0),
|
|
595
|
+
),
|
|
596
|
+
"MATCH_KEYS": lambda _context, _instruction, source: (
|
|
597
|
+
BuildShapeTest(source=source, descriptor_count=0),
|
|
598
|
+
),
|
|
599
|
+
"TO_BOOL": lambda _context, _instruction, source: (Truthy(source=source),),
|
|
600
|
+
"MATCH_CLASS": lambda _context, instruction, source: (
|
|
601
|
+
BuildShapeTest(source=source, descriptor_count=_instruction_count(instruction) + 2),
|
|
602
|
+
),
|
|
603
|
+
"BUILD_LIST": _build_list,
|
|
604
|
+
"BUILD_TUPLE": _build_tuple,
|
|
605
|
+
"BUILD_SET": _build_set,
|
|
606
|
+
"SET_ADD": _set_add,
|
|
607
|
+
"GET_ITER": _get_iter,
|
|
608
|
+
"GET_YIELD_FROM_ITER": _get_iter,
|
|
609
|
+
"UNPACK_SEQUENCE": _unpack_sequence,
|
|
610
|
+
"BINARY_SUBSCR": lambda _context, _instruction, source: (LoadItem(source=source),),
|
|
611
|
+
"BINARY_SLICE": _binary_slice,
|
|
612
|
+
"COMPARE_OP": _compare,
|
|
613
|
+
"CONTAINS_OP": _contains,
|
|
614
|
+
"BINARY_OP": _binary,
|
|
615
|
+
"UNARY_NEGATIVE": _unary("-"),
|
|
616
|
+
"UNARY_INVERT": _unary("~"),
|
|
617
|
+
"UNARY_NOT": _unary("not"),
|
|
618
|
+
"CALL": _invoke,
|
|
619
|
+
"INSTRUMENTED_CALL": _invoke,
|
|
620
|
+
"CALL_KW": _invoke_kw,
|
|
621
|
+
"INSTRUMENTED_CALL_KW": _invoke_kw,
|
|
622
|
+
"CALL_INTRINSIC_2": _call_intrinsic_2,
|
|
623
|
+
"FORMAT_SIMPLE": lambda _context, _instruction, source: (FormatTop(source=source),),
|
|
624
|
+
"FORMAT_WITH_SPEC": lambda _context, _instruction, source: (BuildString(source=source, count=2),),
|
|
625
|
+
"CONVERT_VALUE": _convert_value,
|
|
626
|
+
"BUILD_STRING": _build_string,
|
|
627
|
+
"GET_AWAITABLE": _call_top_as("await"),
|
|
628
|
+
"GET_ANEXT": _call_top_as("anext"),
|
|
629
|
+
"BEFORE_ASYNC_WITH": _no_effect,
|
|
630
|
+
"BUILD_MAP": _build_map,
|
|
631
|
+
"BUILD_CONST_KEY_MAP": _build_const_key_map,
|
|
632
|
+
"DICT_UPDATE": _invoke_collection_method("update"),
|
|
633
|
+
"LIST_APPEND": _invoke_collection_method("append", depth_from_instruction=True),
|
|
634
|
+
"MAP_ADD": _map_add,
|
|
635
|
+
"STORE_SUBSCR": lambda _context, _instruction, source: (StoreItemEffect(source=source, order="value-obj-key"),),
|
|
636
|
+
"STORE_ATTR": lambda _context, instruction, source: (StoreAttr(source=source, attr=str(instruction.argval), order="value-obj"),),
|
|
637
|
+
"STORE_FAST": _store_local,
|
|
638
|
+
"STORE_FAST_MAYBE_NULL": _store_local,
|
|
639
|
+
"STORE_NAME": _store_local,
|
|
640
|
+
"STORE_DEREF": _store_deref,
|
|
641
|
+
"STORE_FAST_STORE_FAST": _store_many,
|
|
642
|
+
"STORE_FAST_LOAD_FAST": _store_fast_load_fast,
|
|
643
|
+
"RETURN_VALUE": lambda _context, _instruction, source: (ReturnTop(source=source, empty_is_void=True),),
|
|
644
|
+
"INSTRUMENTED_RETURN_VALUE": lambda _context, _instruction, source: (ReturnTop(source=source, empty_is_void=True),),
|
|
645
|
+
"RETURN_CONST": _return_const,
|
|
646
|
+
"INSTRUMENTED_RETURN_CONST": _return_const,
|
|
647
|
+
"RAISE_VARARGS": _raise_varargs,
|
|
648
|
+
"RERAISE": lambda _context, _instruction, source: (ReraiseTop(source=source),),
|
|
649
|
+
"YIELD_VALUE": _yield_value,
|
|
650
|
+
"INSTRUMENTED_YIELD_VALUE": _yield_value,
|
|
651
|
+
"SEND": _send,
|
|
652
|
+
"INSTRUMENTED_SEND": _send,
|
|
653
|
+
"IS_OP": _is_op,
|
|
654
|
+
"POP_EXCEPT": _no_effect,
|
|
655
|
+
"CHECK_EXC_MATCH": lambda _context, _instruction, source: (ExceptionMatch(source=source),),
|
|
656
|
+
"POP_JUMP_IF_NONE": _no_effect,
|
|
657
|
+
"POP_JUMP_IF_NOT_NONE": _no_effect,
|
|
658
|
+
},
|
|
659
|
+
rules=(
|
|
660
|
+
VMEffectRule(matches=lambda opcode, _instruction: opcode == "UNPACK_EX", factory=_unpack_ex),
|
|
661
|
+
),
|
|
662
|
+
fallback=_unknown_opcode_effect,
|
|
663
|
+
)
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def lift_pyc_module(pyc_module: PycModule, metadata: dict) -> ModuleIR:
|
|
667
|
+
functions = list(_lift_code_object_tree(pyc_module.code))
|
|
668
|
+
return assemble_vm_module(
|
|
669
|
+
name=pyc_module.filename or "<python-pyc>",
|
|
670
|
+
source_language="python",
|
|
671
|
+
functions=tuple(functions),
|
|
672
|
+
metadata={
|
|
673
|
+
"frontend": metadata,
|
|
674
|
+
"bytecode_format": "pyc",
|
|
675
|
+
},
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def lift_code_object(code: PycCodeObject) -> FunctionIR:
|
|
680
|
+
instructions = tuple(code.instructions)
|
|
681
|
+
steps = _python_bytecode_steps(instructions, code.exception_regions)
|
|
682
|
+
function = lift_vm_step_function(
|
|
683
|
+
_python_function_spec(code),
|
|
684
|
+
steps,
|
|
685
|
+
profile=_python_region_profile(steps, instructions),
|
|
686
|
+
callbacks=_python_region_callbacks(code, instructions),
|
|
687
|
+
stateful_callbacks=_python_stateful_callbacks(code, instructions),
|
|
688
|
+
initial_locals=_python_initial_locals(code),
|
|
689
|
+
raw_window=lambda index: _raw_instruction_window(instructions, index),
|
|
690
|
+
)
|
|
691
|
+
return function
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def _lift_code_object_tree(code: PycCodeObject) -> tuple[FunctionIR, ...]:
|
|
695
|
+
spec = _python_function_spec(code)
|
|
696
|
+
functions = [
|
|
697
|
+
recover_vm_function(
|
|
698
|
+
spec,
|
|
699
|
+
lambda: lift_code_object(code),
|
|
700
|
+
raw=tuple(_raw_instruction_line(instruction) for instruction in code.instructions),
|
|
701
|
+
)
|
|
702
|
+
]
|
|
703
|
+
for child in code.children:
|
|
704
|
+
functions.extend(_lift_code_object_tree(child))
|
|
705
|
+
return tuple(functions)
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def _python_low_effects(instruction, source: SourceRef) -> tuple[Effect, ...] | None:
|
|
709
|
+
return PYTHON_EFFECT_TABLE.effects_for(None, instruction, source)
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def _python_bytecode_step(instruction, source: SourceRef | None = None) -> VMBytecodeStep:
|
|
713
|
+
source = source or _python_source(instruction)
|
|
714
|
+
decoded = _python_decoded_instruction(instruction, source)
|
|
715
|
+
return VMBytecodeStep(
|
|
716
|
+
opcode=decoded.opcode,
|
|
717
|
+
source=source,
|
|
718
|
+
effects=_python_low_effects(instruction, source),
|
|
719
|
+
raw=decoded.raw,
|
|
720
|
+
decoded=decoded,
|
|
721
|
+
hints=_python_instruction_hints(instruction, source),
|
|
722
|
+
)
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def _python_source(instruction) -> SourceRef:
|
|
726
|
+
return SourceRef(
|
|
727
|
+
frontend="python-pyc",
|
|
728
|
+
offset=instruction.offset,
|
|
729
|
+
line=instruction.starts_line,
|
|
730
|
+
)
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _python_bytecode_steps(
|
|
734
|
+
instructions: tuple[object, ...],
|
|
735
|
+
exception_regions: tuple[PycExceptionRegion, ...] = (),
|
|
736
|
+
) -> tuple[VMBytecodeStep, ...]:
|
|
737
|
+
steps: list[VMBytecodeStep] = []
|
|
738
|
+
previous_opcode = ""
|
|
739
|
+
for instruction in instructions:
|
|
740
|
+
step = _python_bytecode_step(instruction)
|
|
741
|
+
region_hints = tuple(
|
|
742
|
+
VMHint(
|
|
743
|
+
kind="exception-region",
|
|
744
|
+
source=step.source,
|
|
745
|
+
target=region.target,
|
|
746
|
+
value={
|
|
747
|
+
"start": region.start,
|
|
748
|
+
"end": region.end,
|
|
749
|
+
"target": region.target,
|
|
750
|
+
"depth": region.depth,
|
|
751
|
+
"lasti": region.lasti,
|
|
752
|
+
},
|
|
753
|
+
label="protected-region",
|
|
754
|
+
)
|
|
755
|
+
for region in exception_regions
|
|
756
|
+
if region.start == instruction.offset
|
|
757
|
+
)
|
|
758
|
+
if instruction.opname == "STORE_FAST_STORE_FAST" and previous_opcode in {"UNPACK_SEQUENCE", "UNPACK_EX"}:
|
|
759
|
+
step = VMBytecodeStep(
|
|
760
|
+
opcode=step.opcode,
|
|
761
|
+
source=step.source,
|
|
762
|
+
effects=(StoreMany(source=step.source, names=_instruction_names(instruction)),),
|
|
763
|
+
raw=step.raw,
|
|
764
|
+
decoded=step.decoded,
|
|
765
|
+
hints=(*step.hints, *region_hints),
|
|
766
|
+
)
|
|
767
|
+
if _is_branch_value_start(instructions, len(steps)):
|
|
768
|
+
step = VMBytecodeStep(
|
|
769
|
+
opcode=step.opcode,
|
|
770
|
+
source=step.source,
|
|
771
|
+
effects=step.effects,
|
|
772
|
+
raw=step.raw,
|
|
773
|
+
decoded=step.decoded,
|
|
774
|
+
hints=(*step.hints, *region_hints, VMHint(kind="branch-value", source=step.source, label="short-circuit-value")),
|
|
775
|
+
)
|
|
776
|
+
elif region_hints:
|
|
777
|
+
step = VMBytecodeStep(
|
|
778
|
+
opcode=step.opcode,
|
|
779
|
+
source=step.source,
|
|
780
|
+
effects=step.effects,
|
|
781
|
+
raw=step.raw,
|
|
782
|
+
decoded=step.decoded,
|
|
783
|
+
hints=(*step.hints, *region_hints),
|
|
784
|
+
)
|
|
785
|
+
steps.append(step)
|
|
786
|
+
previous_opcode = instruction.opname
|
|
787
|
+
return tuple(steps)
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def _is_branch_value_start(instructions: tuple[object, ...], index: int) -> bool:
|
|
791
|
+
"""Identify a VM-neutral short-circuit value shape without structuring it."""
|
|
792
|
+
|
|
793
|
+
window = instructions[index : index + 4]
|
|
794
|
+
return (
|
|
795
|
+
len(window) == 4
|
|
796
|
+
and getattr(window[0], "opname", None) == "COPY"
|
|
797
|
+
and getattr(window[1], "opname", None) == "TO_BOOL"
|
|
798
|
+
and getattr(window[2], "opname", None) in {"POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE"}
|
|
799
|
+
and getattr(window[3], "opname", None) == "POP_TOP"
|
|
800
|
+
)
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def _python_decoded_instruction(instruction, source: SourceRef) -> VMDecodedInstruction:
|
|
804
|
+
operands: list[VMOperand] = []
|
|
805
|
+
if instruction.arg is not None:
|
|
806
|
+
operands.append(VMOperand(role="immediate", value=instruction.arg, text=str(instruction.arg)))
|
|
807
|
+
if instruction.argval is not None:
|
|
808
|
+
operands.append(VMOperand(role=_python_operand_role(instruction.opname), value=instruction.argval, text=str(instruction.argrepr or instruction.argval)))
|
|
809
|
+
return VMDecodedInstruction(
|
|
810
|
+
opcode=instruction.opname,
|
|
811
|
+
source=source,
|
|
812
|
+
operands=tuple(operands),
|
|
813
|
+
raw=_raw_instruction_line(instruction),
|
|
814
|
+
)
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
def _python_operand_role(opname: str):
|
|
818
|
+
if "CONST" in opname:
|
|
819
|
+
return "constant"
|
|
820
|
+
if "FAST" in opname or "DEREF" in opname or opname in {"LOAD_NAME", "STORE_NAME"}:
|
|
821
|
+
return "local"
|
|
822
|
+
if "GLOBAL" in opname:
|
|
823
|
+
return "global"
|
|
824
|
+
if "JUMP" in opname or opname == "FOR_ITER":
|
|
825
|
+
return "target"
|
|
826
|
+
if "ATTR" in opname or "METHOD" in opname:
|
|
827
|
+
return "attribute"
|
|
828
|
+
return "raw"
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def _python_instruction_hints(instruction, source: SourceRef) -> tuple[VMHint, ...]:
|
|
832
|
+
if instruction.opname in {"PUSH_EXC_INFO", "CHECK_EXC_MATCH", "WITH_EXCEPT_START"}:
|
|
833
|
+
return (VMHint(kind="exception-region", source=source, label=instruction.opname),)
|
|
834
|
+
if "JUMP" in instruction.opname or instruction.opname == "FOR_ITER":
|
|
835
|
+
target = instruction.argval if isinstance(instruction.argval, int) else None
|
|
836
|
+
label = "loop-backedge" if target is not None and target <= instruction.offset else "branch-target"
|
|
837
|
+
detail = "iter-next-target-if-false" if instruction.opname == "FOR_ITER" else None
|
|
838
|
+
flow = "conditional" if instruction.opname == "FOR_ITER" or "IF" in instruction.opname or instruction.opname.startswith("POP_JUMP") else "unconditional"
|
|
839
|
+
return (VMHint(kind=label, source=source, target=target, label=instruction.opname, detail=detail, flow=flow),)
|
|
840
|
+
return ()
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
def _python_function_spec(
|
|
844
|
+
code: PycCodeObject,
|
|
845
|
+
*,
|
|
846
|
+
local_names: tuple[str, ...] | None = None,
|
|
847
|
+
) -> VMFunctionSpec:
|
|
848
|
+
argument_names = _python_argument_names(code)
|
|
849
|
+
closure_names = (*code.cellvars, *code.freevars)
|
|
850
|
+
locals_for_metadata = local_names if local_names is not None else (*code.varnames, *closure_names)
|
|
851
|
+
return VMFunctionSpec(
|
|
852
|
+
name=code.name,
|
|
853
|
+
params=argument_names,
|
|
854
|
+
frontend="python-pyc",
|
|
855
|
+
instruction_count=len(code.instructions),
|
|
856
|
+
local_names=tuple(name for name in locals_for_metadata if name not in argument_names),
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def _python_argument_names(code: PycCodeObject) -> tuple[str, ...]:
|
|
861
|
+
positional_end = code.argcount
|
|
862
|
+
keyword_only_end = positional_end + code.kwonlyargcount
|
|
863
|
+
names = list(code.varnames[:keyword_only_end])
|
|
864
|
+
cursor = keyword_only_end
|
|
865
|
+
if code.flags & 0x04: # CO_VARARGS
|
|
866
|
+
names.append(code.varnames[cursor])
|
|
867
|
+
cursor += 1
|
|
868
|
+
if code.flags & 0x08: # CO_VARKEYWORDS
|
|
869
|
+
names.append(code.varnames[cursor])
|
|
870
|
+
return tuple(names)
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
def _python_initial_locals(code: PycCodeObject) -> dict[str, Expr]:
|
|
874
|
+
locals_ = {
|
|
875
|
+
name: Var(name=name, source=SourceRef(frontend="python-pyc", detail=f"local:{name}"))
|
|
876
|
+
for name in code.varnames
|
|
877
|
+
}
|
|
878
|
+
for name in (*code.cellvars, *code.freevars):
|
|
879
|
+
locals_.setdefault(name, CapturedVar(name=name, source=SourceRef(frontend="python-pyc", detail=f"capture:{name}")))
|
|
880
|
+
return locals_
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
def _python_region_callbacks(
|
|
884
|
+
code: PycCodeObject,
|
|
885
|
+
instructions: tuple[object, ...],
|
|
886
|
+
) -> VMRegionCallbacks[VMBytecodeStep]:
|
|
887
|
+
return VMRegionCallbacks(
|
|
888
|
+
lift_slice=lambda slice_start, slice_end, stack: _lift_instruction_slice(
|
|
889
|
+
code,
|
|
890
|
+
instructions[slice_start:slice_end],
|
|
891
|
+
[],
|
|
892
|
+
stack,
|
|
893
|
+
),
|
|
894
|
+
lift_expr=lambda slice_start, slice_end, stack: _lift_stack_expr(
|
|
895
|
+
code,
|
|
896
|
+
instructions[slice_start:slice_end],
|
|
897
|
+
[],
|
|
898
|
+
stack,
|
|
899
|
+
),
|
|
900
|
+
lift_iter_loop=lambda _get_iter_index, _iterable: None,
|
|
901
|
+
lift_async_iter_loop=lambda _prefix_start, _get_aiter_index, _region_end: None,
|
|
902
|
+
lift_comprehension=lambda _prefix_start, _get_iter_index, _region_end, _iterable: None,
|
|
903
|
+
)
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
def _python_stateful_callbacks(
|
|
907
|
+
code: PycCodeObject,
|
|
908
|
+
instructions: tuple[object, ...],
|
|
909
|
+
) -> VMStatefulCallbacks[VMBytecodeStep]:
|
|
910
|
+
return VMStatefulCallbacks(
|
|
911
|
+
initial_locals=lambda: _python_initial_locals(code),
|
|
912
|
+
lift_linear=lambda start, end, locals, stack: _python_linear_state(
|
|
913
|
+
instructions[start:end],
|
|
914
|
+
locals,
|
|
915
|
+
stack,
|
|
916
|
+
),
|
|
917
|
+
branch_condition=_python_branch_condition,
|
|
918
|
+
)
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def _python_linear_state(
|
|
922
|
+
instructions: tuple[object, ...],
|
|
923
|
+
initial_locals: dict[str, Expr],
|
|
924
|
+
initial_stack: tuple[Expr, ...],
|
|
925
|
+
) -> VMLinearState | None:
|
|
926
|
+
result = _run_python_stack_slice(
|
|
927
|
+
_PycLinearLocals(tuple(initial_locals)),
|
|
928
|
+
instructions,
|
|
929
|
+
dict(initial_locals),
|
|
930
|
+
initial_stack,
|
|
931
|
+
)
|
|
932
|
+
stopped_at_terminal_end = (
|
|
933
|
+
getattr(result.stopped_at, "source", None) == _python_source(instructions[-1])
|
|
934
|
+
if instructions and result.state.terminator is not None
|
|
935
|
+
else False
|
|
936
|
+
)
|
|
937
|
+
if result.state.diagnostics or (result.stopped_at is not None and not stopped_at_terminal_end):
|
|
938
|
+
return None
|
|
939
|
+
return VMLinearState(
|
|
940
|
+
locals=result.state.locals,
|
|
941
|
+
stack=tuple(result.state.stack),
|
|
942
|
+
statements=tuple(result.state.statements),
|
|
943
|
+
terminator=result.state.terminator,
|
|
944
|
+
)
|
|
945
|
+
|
|
946
|
+
|
|
947
|
+
def _python_branch_condition(branch: VMBytecodeStep, stack: tuple[Expr, ...]) -> Expr | None:
|
|
948
|
+
if not stack:
|
|
949
|
+
return None
|
|
950
|
+
condition = stack[-1]
|
|
951
|
+
source = branch.source
|
|
952
|
+
if branch.opcode == "TO_BOOL":
|
|
953
|
+
return condition
|
|
954
|
+
if branch.opcode == "POP_JUMP_IF_TRUE":
|
|
955
|
+
return BinaryOp(source=source, op="==", left=condition, right=Const(value=False, source=source))
|
|
956
|
+
if branch.opcode == "POP_JUMP_IF_NONE":
|
|
957
|
+
return BinaryOp(source=source, op="!=", left=condition, right=Const(value=None, source=source))
|
|
958
|
+
if branch.opcode == "POP_JUMP_IF_NOT_NONE":
|
|
959
|
+
return BinaryOp(source=source, op="==", left=condition, right=Const(value=None, source=source))
|
|
960
|
+
return condition
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
class _PycLinearLocals:
|
|
964
|
+
def __init__(self, varnames: tuple[str, ...]) -> None:
|
|
965
|
+
self.varnames = varnames
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def _python_region_profile(
|
|
969
|
+
steps: tuple[VMBytecodeStep, ...],
|
|
970
|
+
instructions: tuple[object, ...],
|
|
971
|
+
) -> VMRegionProfile[VMBytecodeStep]:
|
|
972
|
+
return build_hint_region_profile(
|
|
973
|
+
steps,
|
|
974
|
+
frontend="python-pyc",
|
|
975
|
+
opcode_classes=PYTHON_REGION_OPCODE_CLASSES,
|
|
976
|
+
raw_window=lambda index: _raw_instruction_window(instructions, index),
|
|
977
|
+
await_region_end=lambda await_start, await_end: _await_region_end(instructions, await_start, await_end),
|
|
978
|
+
)
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def _await_region_end(instructions: tuple[object, ...], start: int, end: int) -> int | None:
|
|
982
|
+
for index in range(start, min(end, start + 12)):
|
|
983
|
+
if instructions[index].opname != "SEND":
|
|
984
|
+
continue
|
|
985
|
+
cursor = index + 1
|
|
986
|
+
if cursor < end and instructions[cursor].opname == "YIELD_VALUE":
|
|
987
|
+
cursor += 1
|
|
988
|
+
if cursor < end and instructions[cursor].opname == "RESUME":
|
|
989
|
+
cursor += 1
|
|
990
|
+
if cursor < end and instructions[cursor].opname == "JUMP_BACKWARD_NO_INTERRUPT":
|
|
991
|
+
cursor += 1
|
|
992
|
+
if cursor < end and instructions[cursor].opname == "END_SEND":
|
|
993
|
+
cursor += 1
|
|
994
|
+
if cursor < end and instructions[cursor].opname == "POP_TOP":
|
|
995
|
+
cursor += 1
|
|
996
|
+
return cursor
|
|
997
|
+
return None
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
def _lift_instruction_slice(
|
|
1001
|
+
code: PycCodeObject,
|
|
1002
|
+
instructions: tuple[object, ...],
|
|
1003
|
+
assignments: list[Assign],
|
|
1004
|
+
preloaded_stack: tuple[Expr, ...] = (),
|
|
1005
|
+
) -> tuple[object, ...]:
|
|
1006
|
+
if not instructions:
|
|
1007
|
+
return ()
|
|
1008
|
+
initial_locals = {
|
|
1009
|
+
name: Var(name=name, source=SourceRef(frontend="python-pyc", detail=f"local:{name}"))
|
|
1010
|
+
for name in code.varnames
|
|
1011
|
+
}
|
|
1012
|
+
for assignment in assignments:
|
|
1013
|
+
initial_locals[assignment.target.name] = assignment.target
|
|
1014
|
+
result = _run_python_stack_slice(code, instructions, initial_locals, preloaded_stack)
|
|
1015
|
+
if result.state.diagnostics:
|
|
1016
|
+
return ()
|
|
1017
|
+
output: list[object] = list(result.state.statements)
|
|
1018
|
+
if result.stopped_at is not None and result.state.terminator is None:
|
|
1019
|
+
try:
|
|
1020
|
+
stopped_index = instructions.index(result.stopped_at)
|
|
1021
|
+
except ValueError:
|
|
1022
|
+
stopped_index = 0
|
|
1023
|
+
output.append(
|
|
1024
|
+
vm_unsupported(
|
|
1025
|
+
source=result.stopped_at.source,
|
|
1026
|
+
message="unsupported region",
|
|
1027
|
+
detail=f"stopped at {result.stopped_at.opcode}",
|
|
1028
|
+
raw=(result.stopped_at.raw,) if result.stopped_at.raw else _raw_instruction_window(instructions, stopped_index),
|
|
1029
|
+
)
|
|
1030
|
+
)
|
|
1031
|
+
if result.state.terminator is not None:
|
|
1032
|
+
output.append(result.state.terminator)
|
|
1033
|
+
return tuple(output)
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
def _lift_stack_expr(
|
|
1037
|
+
code: PycCodeObject,
|
|
1038
|
+
instructions: tuple[object, ...],
|
|
1039
|
+
assignments: list[Assign],
|
|
1040
|
+
preloaded_stack: tuple[Expr, ...] = (),
|
|
1041
|
+
) -> Expr | None:
|
|
1042
|
+
initial_locals = {
|
|
1043
|
+
name: Var(name=name, source=SourceRef(frontend="python-pyc", detail=f"local:{name}"))
|
|
1044
|
+
for name in code.varnames
|
|
1045
|
+
}
|
|
1046
|
+
for assignment in assignments:
|
|
1047
|
+
initial_locals[assignment.target.name] = assignment.target
|
|
1048
|
+
result = _run_python_stack_slice(code, instructions, initial_locals, preloaded_stack)
|
|
1049
|
+
if result.state.diagnostics or not result.state.stack:
|
|
1050
|
+
return None
|
|
1051
|
+
return result.state.stack[-1]
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
def _run_python_stack_slice(
|
|
1055
|
+
code: PycCodeObject,
|
|
1056
|
+
instructions: tuple[object, ...],
|
|
1057
|
+
initial_locals: dict[str, Expr],
|
|
1058
|
+
initial_stack: tuple[Expr, ...] = (),
|
|
1059
|
+
):
|
|
1060
|
+
steps = _python_bytecode_steps(instructions)
|
|
1061
|
+
return lift_steps(steps, initial_locals=initial_locals, initial_stack=initial_stack)
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def _instruction_count(instruction) -> int:
|
|
1065
|
+
return instruction.arg if isinstance(instruction.arg, int) else 0
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
def _instruction_names(instruction) -> tuple[str, ...]:
|
|
1069
|
+
value = instruction.argval
|
|
1070
|
+
if isinstance(value, tuple):
|
|
1071
|
+
return tuple(str(name) for name in value)
|
|
1072
|
+
return tuple(name for name in str(instruction.argrepr).split(", ") if name)
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
def _raw_instruction_window(instructions: tuple[object, ...], index: int, radius: int = 3) -> tuple[str, ...]:
|
|
1076
|
+
start = max(0, index - radius)
|
|
1077
|
+
end = min(len(instructions), index + radius + 1)
|
|
1078
|
+
return tuple(_raw_instruction_line(instruction) for instruction in instructions[start:end])
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
def _raw_instruction_line(instruction) -> str:
|
|
1082
|
+
arg = getattr(instruction, "argrepr", "")
|
|
1083
|
+
if not arg:
|
|
1084
|
+
arg = repr(getattr(instruction, "argval", ""))
|
|
1085
|
+
return f"@{instruction.offset} {instruction.opname} {arg}".rstrip()
|