cinderx 2026.1.16.2__cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. __static__/__init__.py +641 -0
  2. __static__/compiler_flags.py +8 -0
  3. __static__/enum.py +160 -0
  4. __static__/native_utils.py +77 -0
  5. __static__/type_code.py +48 -0
  6. __strict__/__init__.py +39 -0
  7. _cinderx.so +0 -0
  8. cinderx/__init__.py +577 -0
  9. cinderx/__pycache__/__init__.cpython-314.pyc +0 -0
  10. cinderx/_asyncio.py +156 -0
  11. cinderx/compileall.py +710 -0
  12. cinderx/compiler/__init__.py +40 -0
  13. cinderx/compiler/__main__.py +137 -0
  14. cinderx/compiler/config.py +7 -0
  15. cinderx/compiler/consts.py +72 -0
  16. cinderx/compiler/debug.py +70 -0
  17. cinderx/compiler/dis_stable.py +283 -0
  18. cinderx/compiler/errors.py +151 -0
  19. cinderx/compiler/flow_graph_optimizer.py +1287 -0
  20. cinderx/compiler/future.py +91 -0
  21. cinderx/compiler/misc.py +32 -0
  22. cinderx/compiler/opcode_cinder.py +18 -0
  23. cinderx/compiler/opcode_static.py +100 -0
  24. cinderx/compiler/opcodebase.py +158 -0
  25. cinderx/compiler/opcodes.py +991 -0
  26. cinderx/compiler/optimizer.py +547 -0
  27. cinderx/compiler/pyassem.py +3711 -0
  28. cinderx/compiler/pycodegen.py +7660 -0
  29. cinderx/compiler/pysourceloader.py +62 -0
  30. cinderx/compiler/static/__init__.py +1404 -0
  31. cinderx/compiler/static/compiler.py +629 -0
  32. cinderx/compiler/static/declaration_visitor.py +335 -0
  33. cinderx/compiler/static/definite_assignment_checker.py +280 -0
  34. cinderx/compiler/static/effects.py +160 -0
  35. cinderx/compiler/static/module_table.py +666 -0
  36. cinderx/compiler/static/type_binder.py +2176 -0
  37. cinderx/compiler/static/types.py +10580 -0
  38. cinderx/compiler/static/util.py +81 -0
  39. cinderx/compiler/static/visitor.py +91 -0
  40. cinderx/compiler/strict/__init__.py +69 -0
  41. cinderx/compiler/strict/class_conflict_checker.py +249 -0
  42. cinderx/compiler/strict/code_gen_base.py +409 -0
  43. cinderx/compiler/strict/common.py +507 -0
  44. cinderx/compiler/strict/compiler.py +352 -0
  45. cinderx/compiler/strict/feature_extractor.py +130 -0
  46. cinderx/compiler/strict/flag_extractor.py +97 -0
  47. cinderx/compiler/strict/loader.py +827 -0
  48. cinderx/compiler/strict/preprocessor.py +11 -0
  49. cinderx/compiler/strict/rewriter/__init__.py +5 -0
  50. cinderx/compiler/strict/rewriter/remove_annotations.py +84 -0
  51. cinderx/compiler/strict/rewriter/rewriter.py +975 -0
  52. cinderx/compiler/strict/runtime.py +77 -0
  53. cinderx/compiler/symbols.py +1754 -0
  54. cinderx/compiler/unparse.py +414 -0
  55. cinderx/compiler/visitor.py +194 -0
  56. cinderx/jit.py +230 -0
  57. cinderx/opcode.py +202 -0
  58. cinderx/static.py +113 -0
  59. cinderx/strictmodule.py +6 -0
  60. cinderx/test_support.py +341 -0
  61. cinderx-2026.1.16.2.dist-info/METADATA +15 -0
  62. cinderx-2026.1.16.2.dist-info/RECORD +68 -0
  63. cinderx-2026.1.16.2.dist-info/WHEEL +6 -0
  64. cinderx-2026.1.16.2.dist-info/licenses/LICENSE +21 -0
  65. cinderx-2026.1.16.2.dist-info/top_level.txt +5 -0
  66. opcodes/__init__.py +0 -0
  67. opcodes/assign_opcode_numbers.py +272 -0
  68. opcodes/cinderx_opcodes.py +121 -0
@@ -0,0 +1,3711 @@
1
+ # Portions copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # pyre-strict
3
+
4
+ """A flow graph representation for Python bytecode"""
5
+
6
+ from __future__ import annotations
7
+
8
+ import sys
9
+ from ast import AST
10
+ from collections import defaultdict
11
+ from contextlib import contextmanager, redirect_stdout
12
+ from dataclasses import dataclass
13
+ from enum import IntEnum, IntFlag
14
+
15
+ try:
16
+ # pyre-ignore[21]: No _inline_cache_entries
17
+ from opcode import _inline_cache_entries
18
+ except ImportError:
19
+ _inline_cache_entries = None
20
+ from types import CodeType
21
+ from typing import (
22
+ Callable,
23
+ ClassVar,
24
+ Generator,
25
+ Iterable,
26
+ Iterator,
27
+ Optional,
28
+ Sequence,
29
+ TextIO,
30
+ TypeAlias,
31
+ TypeVar,
32
+ )
33
+
34
+ from .consts import (
35
+ CO_ASYNC_GENERATOR,
36
+ CO_COROUTINE,
37
+ CO_GENERATOR,
38
+ CO_NEWLOCALS,
39
+ CO_OPTIMIZED,
40
+ CO_SUPPRESS_JIT,
41
+ )
42
+ from .debug import dump_graph
43
+ from .flow_graph_optimizer import (
44
+ FlowGraphConstOptimizer314,
45
+ FlowGraphOptimizer,
46
+ FlowGraphOptimizer310,
47
+ FlowGraphOptimizer312,
48
+ FlowGraphOptimizer314,
49
+ )
50
+ from .opcode_cinder import opcode as cinder_opcode
51
+ from .opcodebase import Opcode
52
+ from .opcodes import opcode as opcodes_opcode, STATIC_OPCODES
53
+ from .symbols import ClassScope, Scope
54
+
55
+
56
+ MAX_COPY_SIZE = 4
57
+
58
+
59
+ class ResumeOparg(IntFlag):
60
+ ScopeEntry = 0
61
+ Yield = 1
62
+ YieldFrom = 2
63
+ Await = 3
64
+
65
+ LocationMask = 0x03
66
+ Depth1Mask = 0x04
67
+
68
+
69
+ def sign(a: float) -> float:
70
+ if not isinstance(a, float):
71
+ raise TypeError(f"Must be a real number, not {type(a)}")
72
+ if a != a:
73
+ return 1.0 # NaN case
74
+ return 1.0 if str(a)[0] != "-" else -1.0
75
+
76
+
77
+ def cast_signed_byte_to_unsigned(i: int) -> int:
78
+ if i < 0:
79
+ i = 255 + i + 1
80
+ return i
81
+
82
+
83
+ FVC_MASK = 0x3
84
+ FVC_NONE = 0x0
85
+ FVC_STR = 0x1
86
+ FVC_REPR = 0x2
87
+ FVC_ASCII = 0x3
88
+ FVS_MASK = 0x4
89
+ FVS_HAVE_SPEC = 0x4
90
+
91
+ NO_INPUT_INSTRS = {
92
+ "FORMAT_SIMPLE",
93
+ "GET_ANEXT",
94
+ "GET_LEN",
95
+ "GET_YIELD_FROM_ITER",
96
+ "IMPORT_FROM",
97
+ "MATCH_KEYS",
98
+ "MATCH_MAPPING",
99
+ "MATCH_SEQUENCE",
100
+ "WITH_EXCEPT_START",
101
+ }
102
+ if sys.version_info >= (3, 15):
103
+ NO_INPUT_INSTRS.add("GET_ITER")
104
+
105
+ UNCONDITIONAL_JUMP_OPCODES = (
106
+ "JUMP_ABSOLUTE",
107
+ "JUMP_FORWARD",
108
+ "JUMP",
109
+ "JUMP_BACKWARD",
110
+ "JUMP_NO_INTERRUPT",
111
+ "JUMP_BACKWARD_NO_INTERRUPT",
112
+ )
113
+
114
+
115
+ # TASK(T128853358): The RETURN_PRIMITIVE logic should live in the Static flow graph.
116
+ RETURN_OPCODES = ("RETURN_VALUE", "RETURN_CONST", "RETURN_PRIMITIVE")
117
+
118
+
119
+ # TASK(T128853358): The RETURN_PRIMITIVE logic should live in the Static flow graph.
120
+ SCOPE_EXIT_OPCODES = (
121
+ "RETURN_VALUE",
122
+ "RETURN_CONST",
123
+ "RETURN_PRIMITIVE",
124
+ "RAISE_VARARGS",
125
+ "RERAISE",
126
+ )
127
+
128
+ SETUP_OPCODES = ("SETUP_ASYNC_WITH", "SETUP_FINALLY", "SETUP_WITH", "SETUP_CLEANUP")
129
+
130
+
131
+ @dataclass(frozen=True, slots=True)
132
+ class SrcLocation:
133
+ lineno: int
134
+ end_lineno: int
135
+ col_offset: int
136
+ end_col_offset: int
137
+
138
+ def __repr__(self) -> str:
139
+ return f"SrcLocation({self.lineno}, {self.end_lineno}, {self.col_offset}, {self.end_col_offset})"
140
+
141
+
142
+ NO_LOCATION = SrcLocation(-1, -1, -1, -1)
143
+ NEXT_LOCATION = SrcLocation(-2, -2, -2, -2)
144
+
145
+
146
+ class Instruction:
147
+ __slots__ = ("opname", "oparg", "target", "ioparg", "loc", "exc_handler")
148
+
149
+ def __init__(
150
+ self,
151
+ opname: str,
152
+ oparg: object,
153
+ ioparg: int = 0,
154
+ loc: AST | SrcLocation = NO_LOCATION,
155
+ target: Block | None = None,
156
+ exc_handler: Block | None = None,
157
+ ) -> None:
158
+ self.opname = opname
159
+ self.oparg = oparg
160
+ self.loc = loc
161
+ self.ioparg = ioparg
162
+ self.target = target
163
+ self.exc_handler = exc_handler
164
+
165
+ @property
166
+ def lineno(self) -> int:
167
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute `lineno`.
168
+ return self.loc.lineno
169
+
170
+ def __repr__(self) -> str:
171
+ args = [
172
+ f"{self.opname!r}",
173
+ f"{self.oparg!r}",
174
+ f"{self.ioparg!r}",
175
+ f"{self.loc!r}",
176
+ ]
177
+ if self.target is not None:
178
+ args.append(f"{self.target!r}")
179
+ if self.exc_handler is not None:
180
+ args.append(f"{self.exc_handler!r}")
181
+
182
+ return f"Instruction({', '.join(args)})"
183
+
184
+ def is_jump(self, opcode: Opcode) -> bool:
185
+ op = opcode.opmap[self.opname]
186
+ return opcode.has_jump(op)
187
+
188
+ def set_to_nop(self) -> None:
189
+ self.opname = "NOP"
190
+ self.oparg = self.ioparg = 0
191
+ self.target = None
192
+
193
+ def set_to_nop_no_loc(self) -> None:
194
+ self.opname = "NOP"
195
+ self.oparg = self.ioparg = 0
196
+ self.target = None
197
+ self.loc = NO_LOCATION
198
+
199
+ @property
200
+ def stores_to(self) -> str | None:
201
+ if self.opname == "STORE_FAST" or self.opname == "STORE_FAST_MAYBE_NULL":
202
+ assert isinstance(self.oparg, str)
203
+ return self.oparg
204
+ return None
205
+
206
+ def copy(self) -> Instruction:
207
+ return Instruction(
208
+ self.opname,
209
+ self.oparg,
210
+ self.ioparg,
211
+ self.loc,
212
+ self.target,
213
+ self.exc_handler,
214
+ )
215
+
216
+
217
+ class CompileScope:
218
+ START_MARKER = "compile-scope-start-marker"
219
+ __slots__ = "blocks"
220
+
221
+ def __init__(self, blocks: list[Block]) -> None:
222
+ self.blocks: list[Block] = blocks
223
+
224
+
225
+ class FlowGraph:
226
+ def __init__(self) -> None:
227
+ self.next_block_id = 0
228
+ # List of blocks in the order they should be output for linear
229
+ # code. As we deal with structured code, this order corresponds
230
+ # to the order of source level constructs. (The original
231
+ # implementation from Python2 used a complex ordering algorithm
232
+ # which more buggy and erratic than useful.)
233
+ self.ordered_blocks: list[Block] = []
234
+ # Current block being filled in with instructions.
235
+ self.current: Block | None = None
236
+ self.entry = Block("entry")
237
+ self.startBlock(self.entry)
238
+
239
+ # Source line number to use for next instruction.
240
+ self.loc: AST | SrcLocation = SrcLocation(0, 0, 0, 0)
241
+ # First line of this code block. This field is expected to be set
242
+ # externally and serve as a reference for generating all other
243
+ # line numbers in the code block. (If it's not set, it will be
244
+ # deduced).
245
+ self.firstline = 0
246
+ # Line number of first instruction output. Used to deduce .firstline
247
+ # if it's not set explicitly.
248
+ self.first_inst_lineno = 0
249
+ # If non-zero, do not emit bytecode
250
+ self.do_not_emit_bytecode = 0
251
+ self.annotations_block: Block | None = None
252
+
253
+ def fetch_current(self) -> Block:
254
+ current = self.current
255
+ assert current
256
+ return current
257
+
258
+ def get_new_block_id(self) -> int:
259
+ ret = self.next_block_id
260
+ self.next_block_id += 1
261
+ return ret
262
+
263
+ def blocks_in_reverse_allocation_order(self) -> Iterable[Block]:
264
+ yield from sorted(self.ordered_blocks, key=lambda b: b.alloc_id, reverse=True)
265
+
266
+ @contextmanager
267
+ def new_compile_scope(self) -> Generator[CompileScope, None, None]:
268
+ prev_current = self.current
269
+ prev_ordered_blocks = self.ordered_blocks
270
+ prev_line_no = self.first_inst_lineno
271
+ try:
272
+ self.ordered_blocks = []
273
+ self.current = self.newBlock(CompileScope.START_MARKER)
274
+ yield CompileScope(self.ordered_blocks)
275
+ finally:
276
+ self.current = prev_current
277
+ self.ordered_blocks = prev_ordered_blocks
278
+ self.first_inst_lineno = prev_line_no
279
+
280
+ def apply_from_scope(self, scope: CompileScope) -> None:
281
+ # link current block with the block from out of order result
282
+ block: Block = scope.blocks[0]
283
+ assert block.prev is not None
284
+ assert block.prev.label == CompileScope.START_MARKER
285
+ block.prev = None
286
+
287
+ self.fetch_current().connect_next(block)
288
+ self.ordered_blocks.extend(scope.blocks)
289
+ self.current = scope.blocks[-1]
290
+
291
+ def startBlock(self, block: Block) -> None:
292
+ """Add `block` to ordered_blocks and set it as the current block."""
293
+ if self._debug:
294
+ current = self.current
295
+ if current:
296
+ print("end", repr(current))
297
+ print(" next", current.next)
298
+ print(" prev", current.prev)
299
+ print(" ", current.get_outgoing())
300
+ print(repr(block))
301
+ block.bid = self.get_new_block_id()
302
+ assert block not in self.ordered_blocks
303
+ self.ordered_blocks.append(block)
304
+ self.current = block
305
+
306
+ def nextBlock(self, block: Block | None = None, label: str = "") -> None:
307
+ """Connect `block` as current.next, then set it as the new `current`
308
+
309
+ Create a new block if needed.
310
+ """
311
+ if self.do_not_emit_bytecode:
312
+ return
313
+ # XXX think we need to specify when there is implicit transfer
314
+ # from one block to the next. might be better to represent this
315
+ # with explicit JUMP_ABSOLUTE instructions that are optimized
316
+ # out when they are unnecessary.
317
+ #
318
+ # I think this strategy works: each block has a child
319
+ # designated as "next" which is returned as the last of the
320
+ # children. because the nodes in a graph are emitted in
321
+ # reverse post order, the "next" block will always be emitted
322
+ # immediately after its parent.
323
+ # Worry: maintaining this invariant could be tricky
324
+ if block is None:
325
+ block = self.newBlock(label=label)
326
+
327
+ # Note: If the current block ends with an unconditional control
328
+ # transfer, then it is technically incorrect to add an implicit
329
+ # transfer to the block graph. Doing so results in code generation
330
+ # for unreachable blocks. That doesn't appear to be very common
331
+ # with Python code and since the built-in compiler doesn't optimize
332
+ # it out we don't either.
333
+ self.fetch_current().connect_next(block)
334
+ self.startBlock(block)
335
+
336
+ def newBlock(self, label: str = "") -> Block:
337
+ """Creates a new Block object, but does not add it to the graph."""
338
+ return Block(label)
339
+
340
+ _debug = 0
341
+
342
+ def _enable_debug(self) -> None:
343
+ self._debug = 1
344
+
345
+ def _disable_debug(self) -> None:
346
+ self._debug = 0
347
+
348
+ def emit_with_loc(self, opcode: str, oparg: object, loc: AST | SrcLocation) -> None:
349
+ if isinstance(oparg, Block):
350
+ if not self.do_not_emit_bytecode:
351
+ current = self.fetch_current()
352
+ current.add_out_edge(oparg)
353
+ current.emit(Instruction(opcode, 0, 0, loc, target=oparg))
354
+ oparg.is_jump_target = True
355
+ return
356
+
357
+ ioparg = self.convertArg(opcode, oparg)
358
+
359
+ if not self.do_not_emit_bytecode:
360
+ self.fetch_current().emit(Instruction(opcode, oparg, ioparg, loc))
361
+
362
+ def emit(self, opcode: str, oparg: object = 0) -> None:
363
+ self.emit_with_loc(opcode, oparg, self.loc)
364
+
365
+ def emit_noline(self, opcode: str, oparg: object = 0) -> None:
366
+ self.emit_with_loc(opcode, oparg, NO_LOCATION)
367
+
368
+ def emitWithBlock(self, opcode: str, oparg: object, target: Block) -> None:
369
+ if not self.do_not_emit_bytecode:
370
+ current = self.fetch_current()
371
+ current.add_out_edge(target)
372
+ current.emit(Instruction(opcode, oparg, target=target))
373
+
374
+ def set_pos(self, node: AST | SrcLocation) -> None:
375
+ if not self.first_inst_lineno:
376
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
377
+ # `lineno`.
378
+ self.first_inst_lineno = node.lineno
379
+ self.loc = node
380
+
381
+ def convertArg(self, opcode: str, oparg: object) -> int:
382
+ if isinstance(oparg, int):
383
+ return oparg
384
+ raise ValueError(f"invalid oparg {oparg!r} for {opcode!r}")
385
+
386
+ def getBlocksInOrder(self) -> list[Block]:
387
+ """Return the blocks in the order they should be output."""
388
+ return self.ordered_blocks
389
+
390
+ def getBlocks(self) -> list[Block]:
391
+ return self.ordered_blocks
392
+
393
+ def getRoot(self) -> Block:
394
+ """Return nodes appropriate for use with dominator"""
395
+ return self.entry
396
+
397
+ def getContainedGraphs(self) -> list[PyFlowGraph]:
398
+ result = []
399
+ for b in self.getBlocks():
400
+ result.extend(b.getContainedGraphs())
401
+ return result
402
+
403
+
404
+ class Block:
405
+ allocated_block_count: ClassVar[int] = 0
406
+
407
+ def __init__(self, label: str = "") -> None:
408
+ self.insts: list[Instruction] = []
409
+ self.out_edges: set[Block] = set()
410
+ self.label: str = label
411
+ self.is_jump_target = False
412
+ self.bid: int | None = None # corresponds to b_label.id in cpython
413
+ self.next: Block | None = None
414
+ self.prev: Block | None = None
415
+ self.returns: bool = False
416
+ self.offset: int = 0
417
+ self.is_exc_handler: bool = False # used in reachability computations
418
+ self.preserve_lasti: bool = False # used if block is an exception handler
419
+ self.seen: bool = False # visited during stack depth calculation
420
+ self.startdepth: int = -1
421
+ self.is_exit: bool = False
422
+ self.has_fallthrough: bool = True
423
+ self.num_predecessors: int = 0
424
+ self.alloc_id: int = Block.allocated_block_count
425
+ # for 3.12
426
+ self.except_stack: list[Block] = []
427
+
428
+ Block.allocated_block_count += 1
429
+
430
+ def __repr__(self) -> str:
431
+ data = []
432
+ data.append(f"id={self.bid}")
433
+ data.append(f"startdepth={self.startdepth}")
434
+ if self.next:
435
+ data.append(f"next={self.next.bid}")
436
+ if self.is_exc_handler:
437
+ data.append("EH")
438
+ extras = ", ".join(data)
439
+ if self.label:
440
+ return f"<block {self.label} {extras}>"
441
+ else:
442
+ return f"<block {extras}>"
443
+
444
+ def __str__(self) -> str:
445
+ insts = map(str, self.insts)
446
+ insts = "\n".join(insts)
447
+ return f"<block label={self.label} bid={self.bid} startdepth={self.startdepth}: {insts}>"
448
+
449
+ def emit(self, instr: Instruction) -> None:
450
+ if instr.opname in RETURN_OPCODES:
451
+ self.returns = True
452
+
453
+ self.insts.append(instr)
454
+
455
+ def getInstructions(self) -> list[Instruction]:
456
+ return self.insts
457
+
458
+ def add_out_edge(self, block: Block) -> None:
459
+ self.out_edges.add(block)
460
+
461
+ def connect_next(self, block: Block) -> None:
462
+ """Connect `block` as self.next."""
463
+ assert self.next is None, self.next
464
+ self.next = block
465
+ assert block.prev is None, block.prev
466
+ block.prev = self
467
+
468
+ def insert_next(self, block: Block) -> None:
469
+ """Insert `block` as self.next in the linked list."""
470
+ assert block.prev is None, block.prev
471
+ block.next = self.next
472
+ block.prev = self
473
+ self.next = block
474
+
475
+ def get_outgoing(self) -> list[Block]:
476
+ """Get the list of blocks this block can transfer control to."""
477
+ return list(self.out_edges) + ([self.next] if self.next is not None else [])
478
+
479
+ def getContainedGraphs(self) -> list[PyFlowGraph]:
480
+ """Return all graphs contained within this block.
481
+
482
+ For example, a MAKE_FUNCTION block will contain a reference to
483
+ the graph for the function body.
484
+ """
485
+ contained = []
486
+ for inst in self.insts:
487
+ # pyre-fixme[6] It is not clear what the type of inst is.
488
+ if len(inst) == 1:
489
+ continue
490
+
491
+ # pyre-fixme[16] It is not clear what the type of inst is.
492
+ op = inst[1]
493
+ if hasattr(op, "graph"):
494
+ contained.append(op.graph)
495
+ return contained
496
+
497
+ def append_instr(
498
+ self,
499
+ opname: str,
500
+ oparg: object,
501
+ ioparg: int = 0,
502
+ loc: AST | SrcLocation = NO_LOCATION,
503
+ target: Block | None = None,
504
+ exc_handler: Block | None = None,
505
+ ) -> None:
506
+ self.insts.append(Instruction(opname, oparg, ioparg, loc, target, exc_handler))
507
+
508
+ def has_no_lineno(self) -> bool:
509
+ for inst in self.insts:
510
+ if inst.lineno >= 0:
511
+ return False
512
+ return True
513
+
514
+ @property
515
+ def exits(self) -> bool:
516
+ return len(self.insts) > 0 and self.insts[-1].opname in SCOPE_EXIT_OPCODES
517
+
518
+ def copy(self) -> Block:
519
+ # Cannot copy block if it has fallthrough, since a block can have only one
520
+ # fallthrough predecessor
521
+ assert not self.has_fallthrough
522
+ result = Block()
523
+ result.insts = [instr.copy() for instr in self.insts]
524
+ result.is_exit = self.is_exit
525
+ result.has_fallthrough = False
526
+ return result
527
+
528
+
529
+ # flags for code objects
530
+
531
+ # the FlowGraph is transformed in place; it exists in one of these states
532
+ ACTIVE = "ACTIVE" # accepting calls to .emit()
533
+ CLOSED = "CLOSED" # closed to new instructions
534
+ CONSTS_CLOSED = "CONSTS_CLOSED" # closed to new consts
535
+ OPTIMIZED = "OPTIMIZED" # optimizations have been run
536
+ ORDERED = "ORDERED" # basic block ordering is set
537
+ FINAL = "FINAL" # all optimization and normalization of flow graph is done
538
+ FLAT = "FLAT" # flattened
539
+ DONE = "DONE"
540
+
541
+
542
+ class IndexedSet:
543
+ """Container that behaves like a `set` that assigns stable dense indexes
544
+ to each element. Put another way: This behaves like a `list` where you
545
+ check `x in <list>` before doing any insertion to avoid duplicates. But
546
+ contrary to the list this does not require an O(n) member check."""
547
+
548
+ __delitem__: None = None
549
+
550
+ def __init__(self, iterable: Iterable[str] = ()) -> None:
551
+ self.keys: dict[str, int] = {}
552
+ for item in iterable:
553
+ self.get_index(item)
554
+
555
+ def __add__(self, iterable: Iterable[str]) -> IndexedSet:
556
+ result = IndexedSet()
557
+ for item in self.keys.keys():
558
+ result.get_index(item)
559
+ for item in iterable:
560
+ result.get_index(item)
561
+ return result
562
+
563
+ def __contains__(self, item: str) -> bool:
564
+ return item in self.keys
565
+
566
+ def __iter__(self) -> Iterator[str]:
567
+ # This relies on `dict` maintaining insertion order.
568
+ return iter(self.keys.keys())
569
+
570
+ def __len__(self) -> int:
571
+ return len(self.keys)
572
+
573
+ def get_index(self, item: str) -> int:
574
+ """Return index of name in collection, appending if necessary"""
575
+ assert type(item) is str
576
+ idx = self.keys.get(item)
577
+ if idx is not None:
578
+ return idx
579
+ idx = len(self.keys)
580
+ self.keys[item] = idx
581
+ return idx
582
+
583
+ def index(self, item: str) -> int:
584
+ assert type(item) is str
585
+ idx = self.keys.get(item)
586
+ if idx is not None:
587
+ return idx
588
+ raise ValueError()
589
+
590
+ def update(self, iterable: Iterable[str]) -> None:
591
+ for item in iterable:
592
+ self.get_index(item)
593
+
594
+
595
+ TConstValue = TypeVar("TConstValue", bound=object)
596
+ TConstKey: TypeAlias = (
597
+ tuple[type[TConstValue], TConstValue] | tuple[type[TConstValue], TConstValue, ...]
598
+ )
599
+
600
+
601
+ class PyFlowGraph(FlowGraph):
602
+ super_init: Callable[[FlowGraph], None] = FlowGraph.__init__
603
+ flow_graph_optimizer: type[FlowGraphOptimizer] = FlowGraphOptimizer
604
+ opcode: Opcode = opcodes_opcode
605
+
606
+ def __init__(
607
+ self,
608
+ name: str,
609
+ filename: str,
610
+ scope: Scope | None,
611
+ flags: int = 0,
612
+ args: Sequence[str] = (),
613
+ kwonlyargs: Sequence[str] = (),
614
+ starargs: Sequence[str] = (),
615
+ optimized: int = 0,
616
+ klass: bool = False,
617
+ docstring: str | None = None,
618
+ firstline: int = 0,
619
+ posonlyargs: int = 0,
620
+ suppress_default_const: bool = False,
621
+ ) -> None:
622
+ self.super_init()
623
+ self.name = name
624
+ self.filename = filename
625
+ self.scope = scope
626
+ self.docstring = None
627
+ self.args = args
628
+ self.kwonlyargs = kwonlyargs
629
+ self.posonlyargs = posonlyargs
630
+ self.starargs = starargs
631
+ self.klass = klass
632
+ self.stacksize = 0
633
+ self.docstring = docstring
634
+ self.flags = flags
635
+ if optimized:
636
+ self.setFlag(CO_OPTIMIZED | CO_NEWLOCALS)
637
+ self.consts: dict[TConstKey, int] = {}
638
+ self.names = IndexedSet()
639
+ # Free variables found by the symbol table scan, including
640
+ # variables used only in nested scopes, are included here.
641
+ if scope is not None:
642
+ self.freevars: IndexedSet = IndexedSet(scope.get_free_vars())
643
+ self.cellvars: IndexedSet = IndexedSet(scope.get_cell_vars())
644
+ else:
645
+ self.freevars = IndexedSet([])
646
+ self.cellvars = IndexedSet([])
647
+ # The closure list is used to track the order of cell
648
+ # variables and free variables in the resulting code object.
649
+ # The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both
650
+ # kinds of variables.
651
+ self.closure: IndexedSet = self.cellvars + self.freevars
652
+ varnames = IndexedSet()
653
+ varnames.update(args)
654
+ varnames.update(kwonlyargs)
655
+ varnames.update(starargs)
656
+ self.varnames: IndexedSet = varnames
657
+ self.stage: str = ACTIVE
658
+ self.firstline = firstline
659
+ self.first_inst_lineno = 0
660
+ # Add any extra consts that were requested to the const pool
661
+ self.extra_consts: list[object] = []
662
+ if not suppress_default_const:
663
+ self.initializeConsts()
664
+ self.fast_vars: set[object] = set()
665
+ self.gen_kind: int | None = None
666
+ self.insts: list[Instruction] = []
667
+ if flags & CO_COROUTINE:
668
+ self.gen_kind = 1
669
+ elif flags & CO_ASYNC_GENERATOR:
670
+ self.gen_kind = 2
671
+ elif flags & CO_GENERATOR:
672
+ self.gen_kind = 0
673
+
674
+ def emit_gen_start(self) -> None:
675
+ if self.gen_kind is not None:
676
+ self.emit_noline("GEN_START", self.gen_kind)
677
+
678
+ # These are here rather than in CodeGenerator because we need to emit jumps
679
+ # while doing block operations in the flowgraph
680
+ def emit_jump_forward(self, target: Block) -> None:
681
+ raise NotImplementedError()
682
+
683
+ def emit_jump_forward_noline(self, target: Block) -> None:
684
+ raise NotImplementedError()
685
+
686
+ def emit_call_one_arg(self) -> None:
687
+ raise NotImplementedError()
688
+
689
+ def emit_super_call(self, name: str, is_zero: bool) -> None:
690
+ raise NotImplementedError()
691
+
692
+ def emit_load_method(self, name: str) -> None:
693
+ raise NotImplementedError()
694
+
695
+ def emit_call_method(self, argcnt: int) -> None:
696
+ raise NotImplementedError()
697
+
698
+ def emit_prologue(self) -> None:
699
+ raise NotImplementedError()
700
+
701
+ def emit_format_value(self, format: int = -1) -> None:
702
+ if format == -1:
703
+ format = FVC_NONE
704
+ self.emit("FORMAT_VALUE", format)
705
+
706
+ def setFlag(self, flag: int) -> None:
707
+ self.flags |= flag
708
+
709
+ def checkFlag(self, flag: int) -> int | None:
710
+ if self.flags & flag:
711
+ return 1
712
+
713
+ def initializeConsts(self) -> None:
714
+ # Docstring is first entry in co_consts for normal functions
715
+ # (Other types of code objects deal with docstrings in different
716
+ # manner, e.g. lambdas and comprehensions don't have docstrings,
717
+ # classes store them as __doc__ attribute.
718
+ if self.name == "<lambda>":
719
+ self.consts[self.get_const_key(None)] = 0
720
+ elif not self.name.startswith("<") and not self.klass:
721
+ if self.docstring is not None:
722
+ self.consts[self.get_const_key(self.docstring)] = 0
723
+ else:
724
+ self.consts[self.get_const_key(None)] = 0
725
+
726
+ def convertArg(self, opcode: str, oparg: object) -> int:
727
+ assert self.stage in {ACTIVE, CLOSED}, self.stage
728
+
729
+ if self.do_not_emit_bytecode and opcode in self._quiet_opcodes:
730
+ # return -1 so this errors if it ever ends up in non-dead-code due
731
+ # to a bug.
732
+ return -1
733
+
734
+ conv = self._converters.get(opcode)
735
+ if conv is not None:
736
+ return conv(self, oparg)
737
+
738
+ return super().convertArg(opcode, oparg)
739
+
740
+ def finalize(self) -> None:
741
+ """Perform final optimizations and normalization of flow graph."""
742
+ assert self.stage == ACTIVE, self.stage
743
+ self.stage = CLOSED
744
+
745
+ for block in self.ordered_blocks:
746
+ self.normalize_basic_block(block)
747
+
748
+ self.optimizeCFG()
749
+
750
+ self.stage = CONSTS_CLOSED
751
+ self.trim_unused_consts()
752
+ self.duplicate_exits_without_lineno()
753
+ self.propagate_line_numbers()
754
+ self.firstline = self.firstline or self.first_inst_lineno or 1
755
+ self.guarantee_lineno_for_exits()
756
+
757
+ self.stage = ORDERED
758
+ self.normalize_jumps()
759
+ self.stage = FINAL
760
+
761
+ def assemble_final_code(self) -> None:
762
+ """Finish assembling code object components from the final graph."""
763
+ raise NotImplementedError()
764
+
765
+ def getCode(self) -> CodeType:
766
+ """Get a Python code object"""
767
+ raise NotImplementedError()
768
+
769
+ def dump(self, io: TextIO | None = None, stack_effect: bool = False) -> None:
770
+ if io:
771
+ with redirect_stdout(io):
772
+ dump_graph(self, stack_effect)
773
+ else:
774
+ dump_graph(self, stack_effect)
775
+
776
+ def push_block(self, worklist: list[Block], block: Block, depth: int) -> None:
777
+ assert block.startdepth < 0 or block.startdepth >= depth, (
778
+ f"{block!r}: {block.startdepth} vs {depth}"
779
+ )
780
+ if block.startdepth < depth:
781
+ block.startdepth = depth
782
+ worklist.append(block)
783
+
784
+ def get_stack_effects(self, opname: str, oparg: object, jump: bool) -> int:
785
+ return self.opcode.stack_effect_raw(opname, oparg, jump)
786
+
787
+ @property
788
+ def initial_stack_depth(self) -> int:
789
+ return 0 if self.gen_kind is None else 1
790
+
791
+ def stackdepth_walk(self, block: Block) -> int:
792
+ # see flowgraph.c :: _PyCfg_Stackdepth()
793
+ maxdepth = 0
794
+ worklist = []
795
+ self.push_block(worklist, block, self.initial_stack_depth)
796
+ while worklist:
797
+ block = worklist.pop()
798
+ next = block.next
799
+ depth = block.startdepth
800
+ assert depth >= 0
801
+
802
+ for instr in block.getInstructions():
803
+ delta = self.get_stack_effects(instr.opname, instr.oparg, False)
804
+ new_depth = depth + delta
805
+ if new_depth > maxdepth:
806
+ maxdepth = new_depth
807
+
808
+ assert new_depth >= 0, (instr, self.dump())
809
+
810
+ op = self.opcode.opmap[instr.opname]
811
+ if (
812
+ self.opcode.has_jump(op) or instr.opname in SETUP_OPCODES
813
+ ) and instr.opname != "END_ASYNC_FOR":
814
+ delta = self.get_stack_effects(instr.opname, instr.oparg, True)
815
+
816
+ target_depth = depth + delta
817
+ if target_depth > maxdepth:
818
+ maxdepth = target_depth
819
+
820
+ assert target_depth >= 0
821
+
822
+ self.push_block(worklist, instr.target, target_depth)
823
+
824
+ depth = new_depth
825
+
826
+ if (
827
+ instr.opname in SCOPE_EXIT_OPCODES
828
+ or instr.opname in UNCONDITIONAL_JUMP_OPCODES
829
+ ):
830
+ # Remaining code is dead
831
+ next = None
832
+ break
833
+
834
+ # Consider saving the delta we came up with here and reapplying it on
835
+ # subsequent walks rather than having to walk all of the instructions again.
836
+ if next:
837
+ self.push_block(worklist, next, depth)
838
+
839
+ return maxdepth
840
+
841
+ def compute_stack_depth(self) -> None:
842
+ """Compute the max stack depth.
843
+
844
+ Find the flow path that needs the largest stack. We assume that
845
+ cycles in the flow graph have no net effect on the stack depth.
846
+ """
847
+ assert self.stage == FINAL, self.stage
848
+ for block in self.getBlocksInOrder():
849
+ # We need to get to the first block which actually has instructions
850
+ if block.getInstructions():
851
+ self.stacksize = self.stackdepth_walk(block)
852
+ break
853
+
854
+ def instrsize(self, instr: Instruction, oparg: int) -> int:
855
+ if oparg <= 0xFF:
856
+ return 1
857
+ elif oparg <= 0xFFFF:
858
+ return 2
859
+ elif oparg <= 0xFFFFFF:
860
+ return 3
861
+ else:
862
+ return 4
863
+
864
+ def flatten_jump(self, inst: Instruction, pc: int) -> int:
865
+ target = inst.target
866
+ assert target is not None
867
+
868
+ offset = target.offset
869
+ if self.opcode.opmap[inst.opname] in self.opcode.hasjrel:
870
+ offset -= pc
871
+
872
+ return abs(offset)
873
+
874
+ def flatten_graph(self) -> None:
875
+ """Arrange the blocks in order and resolve jumps"""
876
+ assert self.stage == FINAL, self.stage
877
+ # This is an awful hack that could hurt performance, but
878
+ # on the bright side it should work until we come up
879
+ # with a better solution.
880
+ #
881
+ # The issue is that in the first loop blocksize() is called
882
+ # which calls instrsize() which requires i_oparg be set
883
+ # appropriately. There is a bootstrap problem because
884
+ # i_oparg is calculated in the second loop.
885
+ #
886
+ # So we loop until we stop seeing new EXTENDED_ARGs.
887
+ # The only EXTENDED_ARGs that could be popping up are
888
+ # ones in jump instructions. So this should converge
889
+ # fairly quickly.
890
+ extended_arg_recompile = True
891
+ while extended_arg_recompile:
892
+ extended_arg_recompile = False
893
+ self.insts = insts = []
894
+ pc = 0
895
+ for b in self.getBlocksInOrder():
896
+ b.offset = pc
897
+
898
+ for inst in b.getInstructions():
899
+ insts.append(inst)
900
+ pc += self.instrsize(inst, inst.ioparg)
901
+
902
+ pc = 0
903
+ for inst in insts:
904
+ pc += self.instrsize(inst, inst.ioparg)
905
+ op = self.opcode.opmap[inst.opname]
906
+ if self.opcode.has_jump(op):
907
+ offset = self.flatten_jump(inst, pc)
908
+
909
+ if self.instrsize(inst, inst.ioparg) != self.instrsize(
910
+ inst, offset
911
+ ):
912
+ extended_arg_recompile = True
913
+
914
+ inst.ioparg = offset
915
+
916
+ self.stage = FLAT
917
+
918
+ # TASK(T128853358): pull out all converters for static opcodes into
919
+ # StaticPyFlowGraph
920
+
921
+ def _convert_LOAD_CONST(self, arg: object) -> int:
922
+ getCode = getattr(arg, "getCode", None)
923
+ if getCode is not None:
924
+ arg = getCode()
925
+ key = self.get_const_key(arg)
926
+ res = self.consts.get(key, self)
927
+ if res is self:
928
+ res = self.consts[key] = len(self.consts)
929
+ return res
930
+
931
+ def get_const_key(self, value: TConstValue) -> TConstKey:
932
+ if isinstance(value, float):
933
+ return type(value), value, sign(value)
934
+ elif isinstance(value, complex):
935
+ return type(value), value, sign(value.real), sign(value.imag)
936
+ elif isinstance(value, (tuple, frozenset)):
937
+ return (
938
+ type(value),
939
+ value,
940
+ type(value)(self.get_const_key(const) for const in value),
941
+ )
942
+ elif isinstance(value, slice):
943
+ return (
944
+ type(value),
945
+ value,
946
+ type(value.start),
947
+ type(value.step),
948
+ type(value.stop),
949
+ )
950
+
951
+ return type(value), value
952
+
953
+ def _convert_LOAD_FAST(self, arg: object) -> int:
954
+ self.fast_vars.add(arg)
955
+ if isinstance(arg, int):
956
+ return arg
957
+
958
+ assert isinstance(arg, str)
959
+ return self.varnames.get_index(arg)
960
+
961
+ def _convert_LOAD_LOCAL(self, arg: object) -> int:
962
+ self.fast_vars.add(arg)
963
+ assert isinstance(arg, tuple), "invalid oparg {arg!r}"
964
+ return self._convert_LOAD_CONST((self.varnames.get_index(arg[0]), arg[1]))
965
+
966
+ def _convert_NAME(self, arg: object) -> int:
967
+ assert isinstance(arg, str)
968
+ return self.names.get_index(arg)
969
+
970
+ def _convert_LOAD_SUPER(self, arg: object) -> int:
971
+ assert isinstance(arg, tuple), "invalid oparg {arg!r}"
972
+ return self._convert_LOAD_CONST((self._convert_NAME(arg[0]), arg[1]))
973
+
974
+ def _convert_LOAD_SUPER_ATTR(self, arg: object) -> int:
975
+ assert isinstance(arg, tuple), "invalid oparg {arg!r}"
976
+ op, name, zero_args = arg
977
+ name_idx = self._convert_NAME(name)
978
+ mask = {
979
+ "LOAD_SUPER_ATTR": 2,
980
+ "LOAD_ZERO_SUPER_ATTR": 0,
981
+ "LOAD_SUPER_METHOD": 3,
982
+ "LOAD_ZERO_SUPER_METHOD": 1,
983
+ }
984
+ return (name_idx << 2) | ((not zero_args) << 1) | mask[op]
985
+
986
+ def _convert_DEREF(self, arg: object) -> int:
987
+ # Sometimes, both cellvars and freevars may contain the same var
988
+ # (e.g., for class' __class__). In this case, prefer freevars.
989
+ assert isinstance(arg, str)
990
+ if arg in self.freevars:
991
+ return self.freevars.get_index(arg) + len(self.cellvars)
992
+ return self.closure.get_index(arg)
993
+
994
+ # similarly for other opcodes...
995
+ _converters: dict[str, Callable[[PyFlowGraph, object], int]] = {
996
+ "LOAD_CLASS": _convert_LOAD_CONST,
997
+ "LOAD_CONST": _convert_LOAD_CONST,
998
+ "INVOKE_FUNCTION": _convert_LOAD_CONST,
999
+ "INVOKE_METHOD": _convert_LOAD_CONST,
1000
+ "LOAD_METHOD_STATIC": _convert_LOAD_CONST,
1001
+ "INVOKE_NATIVE": _convert_LOAD_CONST,
1002
+ "LOAD_FIELD": _convert_LOAD_CONST,
1003
+ "STORE_FIELD": _convert_LOAD_CONST,
1004
+ "CAST": _convert_LOAD_CONST,
1005
+ "TP_ALLOC": _convert_LOAD_CONST,
1006
+ "BUILD_CHECKED_MAP": _convert_LOAD_CONST,
1007
+ "BUILD_CHECKED_LIST": _convert_LOAD_CONST,
1008
+ "PRIMITIVE_LOAD_CONST": _convert_LOAD_CONST,
1009
+ "LOAD_FAST": _convert_LOAD_FAST,
1010
+ "LOAD_FAST_AND_CLEAR": _convert_LOAD_FAST,
1011
+ "STORE_FAST": _convert_LOAD_FAST,
1012
+ "STORE_FAST_MAYBE_NULL": _convert_LOAD_FAST,
1013
+ "DELETE_FAST": _convert_LOAD_FAST,
1014
+ "LOAD_LOCAL": _convert_LOAD_LOCAL,
1015
+ "STORE_LOCAL": _convert_LOAD_LOCAL,
1016
+ "LOAD_NAME": _convert_NAME,
1017
+ "LOAD_FROM_DICT_OR_DEREF": _convert_DEREF,
1018
+ "LOAD_FROM_DICT_OR_GLOBALS": _convert_NAME,
1019
+ "LOAD_CLOSURE": lambda self, arg: self.closure.get_index(arg),
1020
+ "COMPARE_OP": lambda self, arg: self.opcode.CMP_OP.index(arg),
1021
+ "LOAD_GLOBAL": _convert_NAME,
1022
+ "STORE_GLOBAL": _convert_NAME,
1023
+ "DELETE_GLOBAL": _convert_NAME,
1024
+ "CONVERT_NAME": _convert_NAME,
1025
+ "STORE_NAME": _convert_NAME,
1026
+ "STORE_ANNOTATION": _convert_NAME,
1027
+ "DELETE_NAME": _convert_NAME,
1028
+ "IMPORT_NAME": _convert_NAME,
1029
+ "IMPORT_FROM": _convert_NAME,
1030
+ "STORE_ATTR": _convert_NAME,
1031
+ "LOAD_ATTR": _convert_NAME,
1032
+ "DELETE_ATTR": _convert_NAME,
1033
+ "LOAD_METHOD": _convert_NAME,
1034
+ "LOAD_DEREF": _convert_DEREF,
1035
+ "STORE_DEREF": _convert_DEREF,
1036
+ "DELETE_DEREF": _convert_DEREF,
1037
+ "LOAD_CLASSDEREF": _convert_DEREF,
1038
+ "REFINE_TYPE": _convert_LOAD_CONST,
1039
+ "LOAD_METHOD_SUPER": _convert_LOAD_SUPER,
1040
+ "LOAD_ATTR_SUPER": _convert_LOAD_SUPER,
1041
+ "LOAD_SUPER_ATTR": _convert_LOAD_SUPER_ATTR,
1042
+ "LOAD_ZERO_SUPER_ATTR": _convert_LOAD_SUPER_ATTR,
1043
+ "LOAD_SUPER_METHOD": _convert_LOAD_SUPER_ATTR,
1044
+ "LOAD_ZERO_SUPER_METHOD": _convert_LOAD_SUPER_ATTR,
1045
+ "LOAD_TYPE": _convert_LOAD_CONST,
1046
+ }
1047
+
1048
+ # Converters which add an entry to co_consts
1049
+ _const_converters: set[Callable[[PyFlowGraph, object], int]] = {
1050
+ _convert_LOAD_CONST,
1051
+ _convert_LOAD_LOCAL,
1052
+ _convert_LOAD_SUPER,
1053
+ _convert_LOAD_SUPER_ATTR,
1054
+ }
1055
+ # Opcodes which reference an entry in co_consts
1056
+ _const_opcodes: set[str] = set()
1057
+ for op, converter in _converters.items():
1058
+ if converter in _const_converters:
1059
+ _const_opcodes.add(op)
1060
+
1061
+ # Opcodes which do not add names to co_consts/co_names/co_varnames in dead code (self.do_not_emit_bytecode)
1062
+ _quiet_opcodes = {
1063
+ "LOAD_GLOBAL",
1064
+ "LOAD_CONST",
1065
+ "IMPORT_NAME",
1066
+ "STORE_ATTR",
1067
+ "LOAD_ATTR",
1068
+ "DELETE_ATTR",
1069
+ "LOAD_METHOD",
1070
+ "STORE_FAST",
1071
+ "LOAD_FAST",
1072
+ }
1073
+
1074
+ def make_byte_code(self) -> bytes:
1075
+ assert self.stage == FLAT, self.stage
1076
+
1077
+ code: bytearray = bytearray()
1078
+
1079
+ def addCode(opcode: int, oparg: int) -> None:
1080
+ assert opcode < 256, opcode
1081
+ code.append(opcode)
1082
+ code.append(oparg)
1083
+
1084
+ for t in self.insts:
1085
+ oparg = t.ioparg
1086
+ assert 0 <= oparg <= 0xFFFFFFFF, oparg
1087
+ if oparg > 0xFFFFFF:
1088
+ addCode(self.opcode.EXTENDED_ARG, (oparg >> 24) & 0xFF)
1089
+ if oparg > 0xFFFF:
1090
+ addCode(self.opcode.EXTENDED_ARG, (oparg >> 16) & 0xFF)
1091
+ if oparg > 0xFF:
1092
+ addCode(self.opcode.EXTENDED_ARG, (oparg >> 8) & 0xFF)
1093
+ addCode(self.opcode.opmap[t.opname], oparg & 0xFF)
1094
+ self.emit_inline_cache(t.opname, addCode)
1095
+
1096
+ self.stage = DONE
1097
+ return bytes(code)
1098
+
1099
+ def emit_inline_cache(
1100
+ self, opcode: str, addCode: Callable[[int, int], None]
1101
+ ) -> None:
1102
+ pass
1103
+
1104
+ def make_line_table(self) -> bytes:
1105
+ lnotab = LineAddrTable()
1106
+ lnotab.setFirstLine(self.firstline)
1107
+
1108
+ prev_offset = offset = 0
1109
+ for t in self.insts:
1110
+ if lnotab.current_line != t.lineno and t.lineno:
1111
+ lnotab.nextLine(t.lineno, prev_offset, offset)
1112
+ prev_offset = offset
1113
+
1114
+ offset += self.instrsize(t, t.ioparg) * self.opcode.CODEUNIT_SIZE
1115
+
1116
+ # Since the linetable format writes the end offset of bytecodes, we can't commit the
1117
+ # last write until all the instructions are iterated over.
1118
+ lnotab.emitCurrentLine(prev_offset, offset)
1119
+ return lnotab.getTable()
1120
+
1121
+ def getConsts(self) -> tuple[object]:
1122
+ """Return a tuple for the const slot of the code object"""
1123
+ # Just return the constant value, removing the type portion. Order by const index.
1124
+ return tuple(
1125
+ const[1] for const, idx in sorted(self.consts.items(), key=lambda o: o[1])
1126
+ )
1127
+
1128
+ def propagate_line_numbers(self) -> None:
1129
+ """Propagate line numbers to instructions without."""
1130
+ for block in self.ordered_blocks:
1131
+ if not block.insts:
1132
+ continue
1133
+ prev_loc = NO_LOCATION
1134
+ for instr in block.insts:
1135
+ if instr.lineno == NO_LOCATION.lineno:
1136
+ instr.loc = prev_loc
1137
+ else:
1138
+ prev_loc = instr.loc
1139
+ if block.has_fallthrough:
1140
+ next = block.next
1141
+ assert next
1142
+ if next.num_predecessors == 1 and next.insts:
1143
+ next_instr = next.insts[0]
1144
+ if next_instr.lineno == -1:
1145
+ next_instr.loc = prev_loc
1146
+ last_instr = block.insts[-1]
1147
+ if (
1148
+ last_instr.is_jump(self.opcode)
1149
+ and last_instr.opname not in SETUP_OPCODES
1150
+ ):
1151
+ # Only actual jumps, not exception handlers
1152
+ target = last_instr.target
1153
+ assert target
1154
+ # CPython doesn't check target.insts and instead can write
1155
+ # to random memory...
1156
+ if target.num_predecessors == 1 and target.insts:
1157
+ next_instr = target.insts[0]
1158
+ if next_instr.lineno == NO_LOCATION.lineno:
1159
+ next_instr.loc = prev_loc
1160
+
1161
+ def guarantee_lineno_for_exits(self) -> None:
1162
+ assert self.firstline > 0
1163
+ loc = SrcLocation(self.firstline, self.firstline, 0, 0)
1164
+ for block in self.ordered_blocks:
1165
+ if not block.insts:
1166
+ continue
1167
+ last_instr = block.insts[-1]
1168
+ if last_instr.lineno < 0:
1169
+ if last_instr.opname == "RETURN_VALUE":
1170
+ for instr in block.insts:
1171
+ assert instr.loc == NO_LOCATION
1172
+ instr.loc = loc
1173
+ else:
1174
+ loc = last_instr.loc
1175
+
1176
+ def is_exit_without_line_number(self, target: Block) -> bool:
1177
+ raise NotImplementedError()
1178
+
1179
+ def get_duplicate_exit_visitation_order(self) -> Iterable[Block]:
1180
+ raise NotImplementedError()
1181
+
1182
+ def get_target_block_for_exit(self, target: Block) -> Block:
1183
+ assert target.insts
1184
+ return target
1185
+
1186
+ def duplicate_exits_without_lineno(self) -> None:
1187
+ """
1188
+ PEP 626 mandates that the f_lineno of a frame is correct
1189
+ after a frame terminates. It would be prohibitively expensive
1190
+ to continuously update the f_lineno field at runtime,
1191
+ so we make sure that all exiting instruction (raises and returns)
1192
+ have a valid line number, allowing us to compute f_lineno lazily.
1193
+ We can do this by duplicating the exit blocks without line number
1194
+ so that none have more than one predecessor. We can then safely
1195
+ copy the line number from the sole predecessor block.
1196
+ """
1197
+ # Copy all exit blocks without line number that are targets of a jump.
1198
+ append_after = {}
1199
+ for block in self.get_duplicate_exit_visitation_order():
1200
+ if block.insts and (last := block.insts[-1]).is_jump(self.opcode):
1201
+ if last.opname in SETUP_OPCODES:
1202
+ continue
1203
+ target = last.target
1204
+ assert target
1205
+ target = self.get_target_block_for_exit(target)
1206
+ if target.insts[0].opname == "SETUP_CLEANUP":
1207
+ # We have wrapped this block in a stopiteration handler.
1208
+ # The SETUP_CLEANUP is a pseudo-op which will be removed in
1209
+ # a later pass, so it does not need a line number,
1210
+ continue
1211
+
1212
+ if (
1213
+ self.is_exit_without_line_number(target)
1214
+ and target.num_predecessors > 1
1215
+ ):
1216
+ new_target = target.copy()
1217
+ new_target.bid = self.get_new_block_id()
1218
+ new_target.insts[0].loc = last.loc
1219
+ last.target = new_target
1220
+ target.num_predecessors -= 1
1221
+ new_target.num_predecessors = 1
1222
+ target.insert_next(new_target)
1223
+ append_after.setdefault(target, []).append(new_target)
1224
+ for after, to_append in append_after.items():
1225
+ idx = self.ordered_blocks.index(after) + 1
1226
+ self.ordered_blocks[idx:idx] = reversed(to_append)
1227
+
1228
+ for block in self.ordered_blocks:
1229
+ if block.has_fallthrough and block.next and block.insts:
1230
+ if self.is_exit_without_line_number(block.next):
1231
+ block.next.insts[0].loc = block.insts[-1].loc
1232
+
1233
+ def normalize_jumps(self) -> None:
1234
+ assert self.stage == ORDERED, self.stage
1235
+
1236
+ seen_blocks = set()
1237
+
1238
+ for block in self.ordered_blocks:
1239
+ seen_blocks.add(block.bid)
1240
+ if not block.insts:
1241
+ continue
1242
+ last = block.insts[-1]
1243
+ target = last.target
1244
+
1245
+ if last.opname == "JUMP_ABSOLUTE":
1246
+ assert target
1247
+ if target.bid not in seen_blocks:
1248
+ last.opname = "JUMP_FORWARD"
1249
+ elif last.opname == "JUMP_FORWARD":
1250
+ assert target
1251
+ if target.bid in seen_blocks:
1252
+ last.opname = "JUMP_ABSOLUTE"
1253
+
1254
+ def remove_redundant_nops(self, optimizer: FlowGraphOptimizer) -> bool:
1255
+ prev_block = None
1256
+ cleaned = False
1257
+ for block in self.ordered_blocks:
1258
+ prev_lineno = -1
1259
+ if prev_block and prev_block.insts:
1260
+ prev_lineno = prev_block.insts[-1].lineno
1261
+ cleaned |= optimizer.clean_basic_block(block, prev_lineno)
1262
+ prev_block = block if block.has_fallthrough else None
1263
+ return cleaned
1264
+
1265
+ def remove_redundant_jumps(
1266
+ self, optimizer: FlowGraphOptimizer, clean: bool = True
1267
+ ) -> bool:
1268
+ # Delete jump instructions made redundant by previous step. If a non-empty
1269
+ # block ends with a jump instruction, check if the next non-empty block
1270
+ # reached through normal flow control is the target of that jump. If it
1271
+ # is, then the jump instruction is redundant and can be deleted.
1272
+ maybe_empty_blocks = False
1273
+ for block in self.ordered_blocks:
1274
+ if not block.insts:
1275
+ continue
1276
+ last = block.insts[-1]
1277
+ if last.opname not in UNCONDITIONAL_JUMP_OPCODES:
1278
+ continue
1279
+ if last.target == block.next:
1280
+ block.has_fallthrough = True
1281
+ last.set_to_nop()
1282
+ if clean:
1283
+ optimizer.clean_basic_block(block, -1)
1284
+ maybe_empty_blocks = True
1285
+ return maybe_empty_blocks
1286
+
1287
+ def optimizeCFG(self) -> None:
1288
+ """Optimize a well-formed CFG."""
1289
+ raise NotImplementedError()
1290
+
1291
+ def eliminate_empty_basic_blocks(self) -> None:
1292
+ for block in self.ordered_blocks:
1293
+ next_block = block.next
1294
+ if next_block:
1295
+ while not next_block.insts and next_block.next:
1296
+ next_block = next_block.next
1297
+ block.next = next_block
1298
+ for block in self.ordered_blocks:
1299
+ if not block.insts:
1300
+ continue
1301
+ last = block.insts[-1]
1302
+ if last.is_jump(self.opcode) and last.opname != "END_ASYNC_FOR":
1303
+ target = last.target
1304
+ while not target.insts and target.next:
1305
+ target = target.next
1306
+ last.target = target
1307
+ self.ordered_blocks = [block for block in self.ordered_blocks if block.insts]
1308
+
1309
+ def unlink_unreachable_basic_blocks(self, reachable_blocks: set[int]) -> None:
1310
+ self.ordered_blocks = [
1311
+ block
1312
+ for block in self.ordered_blocks
1313
+ if block.bid in reachable_blocks or block.is_exc_handler
1314
+ ]
1315
+ prev = None
1316
+ for block in self.ordered_blocks:
1317
+ block.prev = prev
1318
+ if prev is not None:
1319
+ prev.next = block
1320
+ prev = block
1321
+
1322
+ def remove_unreachable_basic_blocks(self) -> None:
1323
+ # mark all reachable blocks
1324
+ for block in self.getBlocks():
1325
+ block.num_predecessors = 0
1326
+
1327
+ reachable_blocks = set()
1328
+ worklist = [self.entry]
1329
+ self.entry.num_predecessors = 1
1330
+ while worklist:
1331
+ entry = worklist.pop()
1332
+ if entry.bid in reachable_blocks:
1333
+ continue
1334
+ reachable_blocks.add(entry.bid)
1335
+ for instruction in entry.getInstructions():
1336
+ target = instruction.target
1337
+ if target is not None:
1338
+ worklist.append(target)
1339
+ target.num_predecessors += 1
1340
+
1341
+ if entry.has_fallthrough:
1342
+ next = entry.next
1343
+ assert next
1344
+
1345
+ worklist.append(next)
1346
+ next.num_predecessors += 1
1347
+
1348
+ self.unlink_unreachable_basic_blocks(reachable_blocks)
1349
+
1350
+ def normalize_basic_block(self, block: Block) -> None:
1351
+ """Sets the `fallthrough` and `exit` properties of a block, and ensures that the targets of
1352
+ any jumps point to non-empty blocks by following the next pointer of empty blocks.
1353
+ """
1354
+ for instr in block.getInstructions():
1355
+ if instr.opname in SCOPE_EXIT_OPCODES:
1356
+ block.is_exit = True
1357
+ block.has_fallthrough = False
1358
+ continue
1359
+ elif instr.opname in UNCONDITIONAL_JUMP_OPCODES:
1360
+ block.has_fallthrough = False
1361
+ elif not instr.is_jump(self.opcode):
1362
+ continue
1363
+
1364
+ # pyre-fixme[16] instr.target can be None
1365
+ while not instr.target.insts:
1366
+ # pyre-fixme[16] instr.target can be None
1367
+ instr.target = instr.target.next
1368
+
1369
+ def should_inline_block(self, block: Block) -> bool:
1370
+ return block.is_exit and len(block.insts) <= MAX_COPY_SIZE
1371
+
1372
+ def extend_block(self, block: Block) -> bool:
1373
+ """If this block ends with an unconditional jump to an exit block,
1374
+ then remove the jump and extend this block with the target.
1375
+ """
1376
+ if len(block.insts) == 0:
1377
+ return False
1378
+ last = block.insts[-1]
1379
+ if last.opname not in UNCONDITIONAL_JUMP_OPCODES:
1380
+ return False
1381
+ target = last.target
1382
+ assert target is not None
1383
+ if not self.should_inline_block(target):
1384
+ return False
1385
+ last = block.insts[-1]
1386
+ last.set_to_nop()
1387
+ for instr in target.insts:
1388
+ block.insts.append(instr.copy())
1389
+
1390
+ block.next = None
1391
+ block.is_exit = True
1392
+ block.has_fallthrough = False
1393
+ return True
1394
+
1395
+ def trim_unused_consts(self) -> None:
1396
+ """Remove trailing unused constants."""
1397
+ assert self.stage == CONSTS_CLOSED, self.stage
1398
+
1399
+ max_const_index = 0
1400
+ for block in self.ordered_blocks:
1401
+ for instr in block.insts:
1402
+ if (
1403
+ instr.opname in self._const_opcodes
1404
+ and instr.ioparg > max_const_index
1405
+ ):
1406
+ max_const_index = instr.ioparg
1407
+
1408
+ self.consts = {
1409
+ key: index for key, index in self.consts.items() if index <= max_const_index
1410
+ }
1411
+
1412
+
1413
+ class PyFlowGraph310(PyFlowGraph):
1414
+ flow_graph_optimizer = FlowGraphOptimizer310
1415
+
1416
+ # pyre-ignore[2] Forwarding all of the arguments up to the super __init__.
1417
+ def __init__(self, *args, **kwargs) -> None:
1418
+ super().__init__(*args, **kwargs)
1419
+ # Final assembled code objects
1420
+ self.bytecode: bytes | None = None
1421
+ self.line_table: bytes | None = None
1422
+
1423
+ def emit_call_one_arg(self) -> None:
1424
+ self.emit("CALL_FUNCTION", 1)
1425
+
1426
+ def emit_load_method(self, name: str) -> None:
1427
+ self.emit("LOAD_METHOD", name)
1428
+
1429
+ def emit_call_method(self, argcnt: int) -> None:
1430
+ self.emit("CALL_METHOD", argcnt)
1431
+
1432
+ def emit_prologue(self) -> None:
1433
+ pass
1434
+
1435
+ def is_exit_without_line_number(self, target: Block) -> bool:
1436
+ return target.is_exit and target.insts[0].lineno < 0
1437
+
1438
+ def get_duplicate_exit_visitation_order(self) -> Iterable[Block]:
1439
+ return self.blocks_in_reverse_allocation_order()
1440
+
1441
+ def emit_jump_forward(self, target: Block) -> None:
1442
+ self.emit("JUMP_FORWARD", target)
1443
+
1444
+ def emit_jump_forward_noline(self, target: Block) -> None:
1445
+ self.emit_noline("JUMP_FORWARD", target)
1446
+
1447
+ def optimizeCFG(self) -> None:
1448
+ """Optimize a well-formed CFG."""
1449
+ for block in self.blocks_in_reverse_allocation_order():
1450
+ self.extend_block(block)
1451
+
1452
+ assert self.stage == CLOSED, self.stage
1453
+
1454
+ optimizer = self.flow_graph_optimizer(self)
1455
+ for block in self.ordered_blocks:
1456
+ optimizer.optimize_basic_block(block)
1457
+ optimizer.clean_basic_block(block, -1)
1458
+
1459
+ for block in self.blocks_in_reverse_allocation_order():
1460
+ self.extend_block(block)
1461
+
1462
+ self.remove_redundant_nops(optimizer)
1463
+
1464
+ self.eliminate_empty_basic_blocks()
1465
+ self.remove_unreachable_basic_blocks()
1466
+
1467
+ maybe_empty_blocks = self.remove_redundant_jumps(optimizer)
1468
+
1469
+ if maybe_empty_blocks:
1470
+ self.eliminate_empty_basic_blocks()
1471
+
1472
+ self.stage = OPTIMIZED
1473
+
1474
+ def assemble_final_code(self) -> None:
1475
+ """Finish assembling code object components from the final graph."""
1476
+ self.finalize()
1477
+ assert self.stage == FINAL, self.stage
1478
+
1479
+ self.compute_stack_depth()
1480
+ self.flatten_graph()
1481
+
1482
+ assert self.stage == FLAT, self.stage
1483
+ self.bytecode = self.make_byte_code()
1484
+ self.line_table = self.make_line_table()
1485
+
1486
+ def getCode(self) -> CodeType:
1487
+ """Get a Python code object"""
1488
+ self.assemble_final_code()
1489
+ bytecode = self.bytecode
1490
+ assert bytecode is not None
1491
+ line_table = self.line_table
1492
+ assert line_table is not None
1493
+ assert self.stage == DONE, self.stage
1494
+ return self.new_code_object(bytecode, line_table)
1495
+
1496
+ def new_code_object(self, code: bytes, lnotab: bytes) -> CodeType:
1497
+ assert self.stage == DONE, self.stage
1498
+ if (self.flags & CO_NEWLOCALS) == 0:
1499
+ nlocals = len(self.fast_vars)
1500
+ else:
1501
+ nlocals = len(self.varnames)
1502
+
1503
+ firstline = self.firstline
1504
+ # For module, .firstline is initially not set, and should be first
1505
+ # line with actual bytecode instruction (skipping docstring, optimized
1506
+ # out instructions, etc.)
1507
+ if not firstline:
1508
+ firstline = self.first_inst_lineno
1509
+ # If no real instruction, fallback to 1
1510
+ if not firstline:
1511
+ firstline = 1
1512
+
1513
+ consts = self.getConsts()
1514
+ consts = consts + tuple(self.extra_consts)
1515
+ return self.make_code(nlocals, code, consts, firstline, lnotab)
1516
+
1517
+ def make_code(
1518
+ self,
1519
+ nlocals: int,
1520
+ code: bytes,
1521
+ consts: tuple[object, ...],
1522
+ firstline: int,
1523
+ lnotab: bytes,
1524
+ ) -> CodeType:
1525
+ return CodeType(
1526
+ len(self.args),
1527
+ self.posonlyargs,
1528
+ len(self.kwonlyargs),
1529
+ nlocals,
1530
+ self.stacksize,
1531
+ self.flags,
1532
+ code,
1533
+ consts,
1534
+ tuple(self.names),
1535
+ tuple(self.varnames),
1536
+ self.filename,
1537
+ self.name,
1538
+ # pyre-fixme[6]: For 13th argument expected `str` but got `int`.
1539
+ firstline,
1540
+ # pyre-fixme[6]: For 14th argument expected `int` but got `bytes`.
1541
+ lnotab,
1542
+ # pyre-fixme[6]: For 15th argument expected `bytes` but got `tuple[str,
1543
+ # ...]`.
1544
+ tuple(self.freevars),
1545
+ # pyre-fixme[6]: For 16th argument expected `bytes` but got `tuple[str,
1546
+ # ...]`.
1547
+ tuple(self.cellvars),
1548
+ )
1549
+
1550
+
1551
+ class PyFlowGraphCinderMixin(PyFlowGraph):
1552
+ opcode: Opcode = cinder_opcode
1553
+
1554
+ def emit_super_call(self, name: str, is_zero: bool) -> None:
1555
+ self.emit("LOAD_METHOD_SUPER", (name, is_zero))
1556
+
1557
+ def make_code(
1558
+ self,
1559
+ nlocals: int,
1560
+ code: bytes,
1561
+ consts: tuple[object, ...],
1562
+ firstline: int,
1563
+ lnotab: bytes,
1564
+ ) -> CodeType:
1565
+ if self.scope is not None and self.scope.suppress_jit:
1566
+ self.setFlag(CO_SUPPRESS_JIT)
1567
+ # pyre-ignore[16]: `PyFlowGraph` has no attribute `make_code`
1568
+ return super().make_code(nlocals, code, consts, firstline, lnotab)
1569
+
1570
+
1571
+ class PyFlowGraphCinder310(PyFlowGraphCinderMixin, PyFlowGraph310):
1572
+ pass
1573
+
1574
+
1575
+ class PyFlowGraph312(PyFlowGraph):
1576
+ flow_graph_optimizer = FlowGraphOptimizer312
1577
+
1578
+ def __init__(
1579
+ self,
1580
+ name: str,
1581
+ filename: str,
1582
+ scope: Scope | None,
1583
+ flags: int = 0,
1584
+ args: Sequence[str] = (),
1585
+ kwonlyargs: Sequence[str] = (),
1586
+ starargs: Sequence[str] = (),
1587
+ optimized: int = 0,
1588
+ klass: bool = False,
1589
+ docstring: Optional[str] = None,
1590
+ firstline: int = 0,
1591
+ posonlyargs: int = 0,
1592
+ qualname: Optional[str] = None,
1593
+ suppress_default_const: bool = False,
1594
+ ) -> None:
1595
+ super().__init__(
1596
+ name,
1597
+ filename,
1598
+ scope,
1599
+ flags,
1600
+ args,
1601
+ kwonlyargs,
1602
+ starargs,
1603
+ optimized,
1604
+ klass,
1605
+ docstring,
1606
+ firstline,
1607
+ posonlyargs,
1608
+ suppress_default_const,
1609
+ )
1610
+ self.qualname: str = qualname or name
1611
+ # Final assembled code objects
1612
+ self.bytecode: bytes | None = None
1613
+ self.line_table: bytes | None = None
1614
+ self.exception_table: bytes | None = None
1615
+
1616
+ def emit_call_one_arg(self) -> None:
1617
+ self.emit("CALL", 0)
1618
+
1619
+ def emit_load_method(self, name: str) -> None:
1620
+ self.emit("LOAD_ATTR", (name, 1))
1621
+
1622
+ def emit_call_method(self, argcnt: int) -> None:
1623
+ self.emit("CALL", argcnt)
1624
+
1625
+ def emit_super_call(self, name: str, is_zero: bool) -> None:
1626
+ op = "LOAD_ZERO_SUPER_METHOD" if is_zero else "LOAD_SUPER_METHOD"
1627
+ self.emit("LOAD_SUPER_ATTR", (op, name, is_zero))
1628
+
1629
+ def emit_prologue(self) -> None:
1630
+ self.emit("RESUME", 0)
1631
+
1632
+ def is_exit_without_line_number(self, target: Block) -> bool:
1633
+ if not target.is_exit:
1634
+ return False
1635
+
1636
+ for inst in target.insts:
1637
+ if inst.lineno >= 0:
1638
+ return False
1639
+
1640
+ return True
1641
+
1642
+ def get_duplicate_exit_visitation_order(self) -> Iterable[Block]:
1643
+ return self.ordered_blocks
1644
+
1645
+ def emit_gen_start(self) -> None:
1646
+ # This is handled with the prefix instructions in finalize
1647
+ pass
1648
+
1649
+ def emit_jump_forward(self, target: Block) -> None:
1650
+ self.emit("JUMP", target)
1651
+
1652
+ def emit_jump_forward_noline(self, target: Block) -> None:
1653
+ self.emit_noline("JUMP", target)
1654
+
1655
+ def flatten_jump(self, inst: Instruction, pc: int) -> int:
1656
+ target = inst.target
1657
+ assert target is not None, inst.opname
1658
+
1659
+ offset = target.offset - pc
1660
+
1661
+ return abs(offset)
1662
+
1663
+ def push_except_block(self, except_stack: list[Block], instr: Instruction) -> Block:
1664
+ target = instr.target
1665
+ assert target is not None, instr
1666
+ if instr.opname in ("SETUP_WITH", "SETUP_CLEANUP"):
1667
+ target.preserve_lasti = True
1668
+ except_stack.append(target)
1669
+ return target
1670
+
1671
+ def label_exception_targets(self) -> None:
1672
+ def push_todo_block(block: Block) -> None:
1673
+ todo_stack.append(block)
1674
+ visited.add(block)
1675
+
1676
+ todo_stack: list[Block] = [self.entry]
1677
+ visited: set[Block] = {self.entry}
1678
+ except_stack = []
1679
+ self.entry.except_stack = except_stack
1680
+
1681
+ while todo_stack:
1682
+ block = todo_stack.pop()
1683
+ assert block in visited
1684
+ except_stack = block.except_stack
1685
+ block.except_stack = []
1686
+ handler = except_stack[-1] if except_stack else None
1687
+ for instr in block.insts:
1688
+ if instr.opname in SETUP_OPCODES:
1689
+ target = instr.target
1690
+ assert target, instr
1691
+ if target not in visited:
1692
+ # Copy the current except stack into the target's except stack
1693
+ target.except_stack = list(except_stack)
1694
+ push_todo_block(target)
1695
+ handler = self.push_except_block(except_stack, instr)
1696
+ elif instr.opname == "POP_BLOCK":
1697
+ except_stack.pop()
1698
+ handler = except_stack[-1] if except_stack else None
1699
+ elif instr.is_jump(self.opcode):
1700
+ instr.exc_handler = handler
1701
+ if instr.target not in visited:
1702
+ target = instr.target
1703
+ assert target
1704
+ if block.has_fallthrough:
1705
+ # Copy the current except stack into the block's except stack
1706
+ target.except_stack = list(except_stack)
1707
+ else:
1708
+ # Move the current except stack to the block and start a new one
1709
+ target.except_stack = except_stack
1710
+ except_stack = []
1711
+ push_todo_block(target)
1712
+ else:
1713
+ if instr.opname == "YIELD_VALUE":
1714
+ assert except_stack is not None
1715
+ instr.ioparg = len(except_stack)
1716
+ instr.exc_handler = handler
1717
+
1718
+ if block.has_fallthrough and block.next and block.next not in visited:
1719
+ assert except_stack is not None
1720
+ block.next.except_stack = except_stack
1721
+ push_todo_block(block.next)
1722
+
1723
+ def compute_except_handlers(self) -> set[Block]:
1724
+ except_handlers: set[Block] = set()
1725
+ for block in self.ordered_blocks:
1726
+ for instr in block.insts:
1727
+ if instr.opname in SETUP_OPCODES:
1728
+ target = instr.target
1729
+ assert target is not None, "SETUP_* opcodes all have targets"
1730
+ except_handlers.add(target)
1731
+ break
1732
+
1733
+ return except_handlers
1734
+
1735
+ def make_explicit_jump_block(self) -> Block:
1736
+ return self.newBlock("explicit_jump")
1737
+
1738
+ def remove_redundant_nops_and_jumps(self, optimizer: FlowGraphOptimizer) -> None:
1739
+ self.remove_redundant_jumps(optimizer, False)
1740
+
1741
+ def push_cold_blocks_to_end(
1742
+ self, except_handlers: set[Block], optimizer: FlowGraphOptimizer
1743
+ ) -> None:
1744
+ warm = self.compute_warm()
1745
+
1746
+ # If we have a cold block with fallthrough to a warm block, add
1747
+ # an explicit jump instead of fallthrough
1748
+ for block in list(self.ordered_blocks):
1749
+ if block not in warm and block.has_fallthrough and block.next in warm:
1750
+ explicit_jump = self.make_explicit_jump_block()
1751
+ explicit_jump.bid = self.get_new_block_id()
1752
+ self.current = explicit_jump
1753
+
1754
+ next_block = block.next
1755
+ assert next_block is not None
1756
+
1757
+ self.emit_jump_forward_noline(next_block)
1758
+ self.ordered_blocks.insert(
1759
+ self.ordered_blocks.index(block) + 1, explicit_jump
1760
+ )
1761
+
1762
+ explicit_jump.next = block.next
1763
+ explicit_jump.has_fallthrough = False
1764
+ block.next = explicit_jump
1765
+
1766
+ new_ordered = []
1767
+ to_end = []
1768
+ prev = None
1769
+ for block in self.ordered_blocks:
1770
+ if block in warm:
1771
+ new_ordered.append(block)
1772
+ if prev is not None:
1773
+ prev.next = block
1774
+ block.prev = prev
1775
+ prev = block
1776
+ else:
1777
+ to_end.append(block)
1778
+
1779
+ for block in to_end:
1780
+ prev.next = block
1781
+ block.prev = prev
1782
+ prev = block
1783
+
1784
+ block.next = None
1785
+
1786
+ self.ordered_blocks = new_ordered + to_end
1787
+ if to_end:
1788
+ self.remove_redundant_nops_and_jumps(optimizer)
1789
+
1790
+ def compute_warm(self) -> set[Block]:
1791
+ """Compute the set of 'warm' blocks, which are blocks that are reachable
1792
+ through normal control flow. 'Cold' blocks are those that are not
1793
+ directly reachable and may be moved to the end of the block list
1794
+ for optimization purposes."""
1795
+
1796
+ stack = [self.entry]
1797
+ visited = set(stack)
1798
+ warm: set[Block] = set()
1799
+
1800
+ while stack:
1801
+ cur = stack.pop()
1802
+ warm.add(cur)
1803
+ next = cur.next
1804
+ if next is not None and cur.has_fallthrough and next not in visited:
1805
+ stack.append(next)
1806
+ visited.add(next)
1807
+
1808
+ for instr in cur.insts:
1809
+ target = instr.target
1810
+ if (
1811
+ target is not None
1812
+ and instr.is_jump(self.opcode)
1813
+ and target not in visited
1814
+ ):
1815
+ stack.append(target)
1816
+ visited.add(target)
1817
+ return warm
1818
+
1819
+ def remove_unused_consts(self) -> None:
1820
+ nconsts = len(self.consts)
1821
+ if not nconsts:
1822
+ return
1823
+
1824
+ index_map = [-1] * nconsts
1825
+ index_map[0] = 0 # The first constant may be docstring; keep it always.
1826
+
1827
+ # mark used consts
1828
+ for block in self.ordered_blocks:
1829
+ for instr in block.insts:
1830
+ if instr.opname in self.opcode.hasconst:
1831
+ index_map[instr.ioparg] = instr.ioparg
1832
+
1833
+ used_consts = [x for x in index_map if x > -1]
1834
+ if len(used_consts) == nconsts:
1835
+ return
1836
+
1837
+ reverse_index_mapping = {
1838
+ old_index: index for index, old_index in enumerate(used_consts)
1839
+ }
1840
+
1841
+ # generate the updated consts mapping and the mapping from
1842
+ # the old index to the new index
1843
+ new_consts = {
1844
+ key: reverse_index_mapping[old_index]
1845
+ for key, old_index in self.consts.items()
1846
+ if old_index in reverse_index_mapping
1847
+ }
1848
+ self.consts = new_consts
1849
+
1850
+ # now update the existing opargs
1851
+ for block in self.ordered_blocks:
1852
+ for instr in block.insts:
1853
+ if instr.opname in self.opcode.hasconst:
1854
+ instr.ioparg = reverse_index_mapping[instr.ioparg]
1855
+
1856
+ def add_checks_for_loads_of_uninitialized_variables(self) -> None:
1857
+ UninitializedVariableChecker(self).check()
1858
+
1859
+ def inline_small_exit_blocks(self) -> None:
1860
+ for block in self.ordered_blocks:
1861
+ self.extend_block(block)
1862
+
1863
+ def is_redundant_pair(
1864
+ self, prev_instr: Instruction | None, instr: Instruction
1865
+ ) -> bool:
1866
+ if prev_instr is not None and instr.opname == "POP_TOP":
1867
+ if (
1868
+ prev_instr.opname == "LOAD_CONST"
1869
+ or prev_instr.opname == "LOAD_SMALL_INT"
1870
+ ):
1871
+ return True
1872
+ elif prev_instr.opname == "COPY" and prev_instr.oparg == 1:
1873
+ return True
1874
+ return False
1875
+
1876
+ def remove_redundant_nops_and_pairs(self, optimizer: FlowGraphOptimizer) -> None:
1877
+ done = False
1878
+ while not done:
1879
+ done = True
1880
+ instr: Instruction | None = None
1881
+ for block in self.ordered_blocks:
1882
+ optimizer.clean_basic_block(block, -1)
1883
+ if block.is_jump_target:
1884
+ instr = None
1885
+
1886
+ for cur_instr in block.insts:
1887
+ prev_instr = instr
1888
+ instr = cur_instr
1889
+
1890
+ if self.is_redundant_pair(prev_instr, instr):
1891
+ instr.set_to_nop()
1892
+ assert prev_instr is not None
1893
+ prev_instr.set_to_nop()
1894
+ done = False
1895
+
1896
+ if (instr and instr.is_jump(self.opcode)) or not block.has_fallthrough:
1897
+ instr = None
1898
+
1899
+ def optimizeCFG(self) -> None:
1900
+ """Optimize a well-formed CFG."""
1901
+ except_handlers = self.compute_except_handlers()
1902
+
1903
+ self.label_exception_targets()
1904
+
1905
+ assert self.stage == CLOSED, self.stage
1906
+
1907
+ self.eliminate_empty_basic_blocks()
1908
+
1909
+ self.inline_small_exit_blocks()
1910
+
1911
+ optimizer = self.flow_graph_optimizer(self)
1912
+ for block in self.ordered_blocks:
1913
+ optimizer.optimize_basic_block(block)
1914
+
1915
+ self.remove_redundant_nops_and_pairs(optimizer)
1916
+
1917
+ self.inline_small_exit_blocks()
1918
+
1919
+ for block in self.ordered_blocks:
1920
+ # remove redundant nops
1921
+ optimizer.clean_basic_block(block, -1)
1922
+
1923
+ self.remove_unreachable_basic_blocks()
1924
+ self.eliminate_empty_basic_blocks()
1925
+
1926
+ self.remove_redundant_jumps(optimizer, False)
1927
+
1928
+ self.stage = OPTIMIZED
1929
+
1930
+ self.remove_unused_consts()
1931
+ self.add_checks_for_loads_of_uninitialized_variables()
1932
+ self.insert_superinstructions()
1933
+ self.push_cold_blocks_to_end(except_handlers, optimizer)
1934
+
1935
+ def insert_superinstructions(self) -> None:
1936
+ # No super instructions on 3.12
1937
+ pass
1938
+
1939
+ def build_cell_fixed_offsets(self) -> list[int]:
1940
+ nlocals = len(self.varnames)
1941
+ ncellvars = len(self.cellvars)
1942
+ nfreevars = len(self.freevars)
1943
+
1944
+ noffsets = ncellvars + nfreevars
1945
+ fixed = [nlocals + i for i in range(noffsets)]
1946
+
1947
+ for varname in self.cellvars:
1948
+ cellindex = self.cellvars.get_index(varname)
1949
+ if varname in self.varnames:
1950
+ varindex = self.varnames.index(varname)
1951
+ fixed[cellindex] = varindex
1952
+
1953
+ return fixed
1954
+
1955
+ def get_generator_prefix(self) -> list[Instruction]:
1956
+ firstline = self.firstline or self.first_inst_lineno or 1
1957
+ loc = SrcLocation(firstline, firstline, -1, -1)
1958
+ return [
1959
+ Instruction("RETURN_GENERATOR", 0, loc=loc),
1960
+ Instruction("POP_TOP", 0),
1961
+ ]
1962
+
1963
+ def insert_prefix_instructions(self, fixed_map: list[int]) -> None:
1964
+ to_insert = []
1965
+
1966
+ if self.freevars:
1967
+ n_freevars = len(self.freevars)
1968
+ to_insert.append(
1969
+ Instruction("COPY_FREE_VARS", n_freevars, ioparg=n_freevars)
1970
+ )
1971
+
1972
+ if self.cellvars:
1973
+ # self.cellvars has the cells out of order so we sort them before
1974
+ # adding the MAKE_CELL instructions. Note that we adjust for arg
1975
+ # cells, which come first.
1976
+ nvars = len(self.cellvars) + len(self.varnames)
1977
+ sorted = [0] * nvars
1978
+ for i in range(len(self.cellvars)):
1979
+ sorted[fixed_map[i]] = i + 1
1980
+
1981
+ # varnames come first
1982
+ n_used = i = 0
1983
+ while n_used < len(self.cellvars):
1984
+ index = sorted[i] - 1
1985
+ if index != -1:
1986
+ to_insert.append(Instruction("MAKE_CELL", index, ioparg=index))
1987
+ n_used += 1
1988
+ i += 1
1989
+
1990
+ if self.gen_kind is not None:
1991
+ to_insert.extend(self.get_generator_prefix())
1992
+
1993
+ if to_insert:
1994
+ self.entry.insts[0:0] = to_insert
1995
+
1996
+ def fix_cell_offsets(self, fixed_map: list[int]) -> int:
1997
+ nlocals = len(self.varnames)
1998
+ ncellvars = len(self.cellvars)
1999
+ nfreevars = len(self.freevars)
2000
+ noffsets = ncellvars + nfreevars
2001
+
2002
+ # First deal with duplicates (arg cells).
2003
+ num_dropped = 0
2004
+ for i in range(noffsets):
2005
+ if fixed_map[i] == i + nlocals:
2006
+ fixed_map[i] -= num_dropped
2007
+ else:
2008
+ # It was a duplicate (cell/arg).
2009
+ num_dropped += 1
2010
+
2011
+ # Then update offsets, either relative to locals or by cell2arg.
2012
+ for b in self.ordered_blocks:
2013
+ for instr in b.insts:
2014
+ if instr.opname in (
2015
+ "MAKE_CELL",
2016
+ "LOAD_CLOSURE",
2017
+ "LOAD_DEREF",
2018
+ "STORE_DEREF",
2019
+ "DELETE_DEREF",
2020
+ "LOAD_FROM_DICT_OR_DEREF",
2021
+ ):
2022
+ oldoffset = instr.ioparg
2023
+ assert oldoffset >= 0
2024
+ assert oldoffset < noffsets, (
2025
+ f"{instr.opname} {self.name} {self.firstline} {instr.oparg}"
2026
+ )
2027
+ assert fixed_map[oldoffset] >= 0
2028
+ if isinstance(instr.oparg, int):
2029
+ # Only update oparg here if it's already in the int-form
2030
+ # so that we can assert on the string form in the sbs
2031
+ # tests.
2032
+ instr.oparg = fixed_map[oldoffset]
2033
+ instr.ioparg = fixed_map[oldoffset]
2034
+ return num_dropped
2035
+
2036
+ def prepare_localsplus(self) -> int:
2037
+ nlocals = len(self.varnames)
2038
+ ncellvars = len(self.cellvars)
2039
+ nfreevars = len(self.freevars)
2040
+ nlocalsplus = nlocals + ncellvars + nfreevars
2041
+ fixed_map = self.build_cell_fixed_offsets()
2042
+ # This must be called before fix_cell_offsets().
2043
+ self.insert_prefix_instructions(fixed_map)
2044
+ num_dropped = self.fix_cell_offsets(fixed_map)
2045
+ assert num_dropped >= 0
2046
+ nlocalsplus -= num_dropped
2047
+ assert nlocalsplus >= 0
2048
+ return nlocalsplus
2049
+
2050
+ def instrsize(self, instr: Instruction, oparg: int) -> int:
2051
+ opname = instr.opname
2052
+ opcode_index = opcodes_opcode.opmap[opname]
2053
+ if opcode_index >= len(_inline_cache_entries):
2054
+ # T190611021: This should never happen as we should remove pseudo
2055
+ # instructions, but we are still missing some functionality
2056
+ # like zero-cost exceptions so we emit things like END_FINALLY
2057
+ base_size = 0
2058
+ else:
2059
+ base_size = _inline_cache_entries[opcode_index]
2060
+ if oparg <= 0xFF:
2061
+ return 1 + base_size
2062
+ elif oparg <= 0xFFFF:
2063
+ return 2 + base_size
2064
+ elif oparg <= 0xFFFFFF:
2065
+ return 3 + base_size
2066
+ else:
2067
+ return 4 + base_size
2068
+
2069
+ _reversed_jumps: dict[str, str] = {
2070
+ "POP_JUMP_IF_NOT_NONE": "POP_JUMP_IF_NONE",
2071
+ "POP_JUMP_IF_NONE": "POP_JUMP_IF_NOT_NONE",
2072
+ "POP_JUMP_IF_FALSE": "POP_JUMP_IF_TRUE",
2073
+ "POP_JUMP_IF_TRUE": "POP_JUMP_IF_FALSE",
2074
+ "POP_JUMP_IF_NONZERO": "POP_JUMP_IF_ZERO",
2075
+ "POP_JUMP_IF_ZERO": "POP_JUMP_IF_NONZERO",
2076
+ "JUMP_IF_TRUE": "JUMP_IF_FALSE",
2077
+ "JUMP_IF_FALSE": "JUMP_IF_TRUE",
2078
+ }
2079
+
2080
+ def normalize_jumps_in_block(
2081
+ self, block: Block, seen_blocks: set[Block]
2082
+ ) -> Block | None:
2083
+ last = block.insts[-1]
2084
+ if not last.is_jump(self.opcode):
2085
+ return
2086
+ target = last.target
2087
+ assert target is not None
2088
+ is_forward = target.bid not in seen_blocks
2089
+ assert last.opname != "JUMP_FOWARD"
2090
+ if last.opname == "JUMP":
2091
+ last.opname = "JUMP_FORWARD" if is_forward else "JUMP_BACKWARD"
2092
+ return
2093
+ elif last.opname == "JUMP_NO_INTERRUPT":
2094
+ last.opname = "JUMP_FORWARD" if is_forward else "JUMP_BACKWARD_NO_INTERRUPT"
2095
+ return
2096
+
2097
+ if is_forward:
2098
+ return
2099
+
2100
+ # transform 'conditional jump T' to
2101
+ # 'reversed_jump b_next' followed by 'jump_backwards T'
2102
+ backwards_jump = self.newBlock("backwards_jump")
2103
+ backwards_jump.bid = self.get_new_block_id()
2104
+ self.current = backwards_jump
2105
+ # cpython has `JUMP(target)` here, but it will always get inserted as
2106
+ # the next block in the loop and then transformed to JUMP_BACKWARD by
2107
+ # the above code, since is_forward(target) won't have changed.
2108
+ self.emit_with_loc("JUMP_BACKWARD", target, last.loc)
2109
+ last.opname = self._reversed_jumps[last.opname]
2110
+ last.target = block.next
2111
+ block.insert_next(backwards_jump)
2112
+ return backwards_jump
2113
+
2114
+ def normalize_jumps(self) -> None:
2115
+ seen_blocks = set()
2116
+ new_blocks = {}
2117
+ for block in self.ordered_blocks:
2118
+ seen_blocks.add(block.bid)
2119
+ if block.insts:
2120
+ ret = self.normalize_jumps_in_block(block, seen_blocks)
2121
+ new_blocks[block] = ret
2122
+ new_ordered = []
2123
+ for block in self.ordered_blocks:
2124
+ new_ordered.append(block)
2125
+ if to_add := new_blocks.get(block):
2126
+ new_ordered.append(to_add)
2127
+ self.ordered_blocks = new_ordered
2128
+
2129
+ def emit_inline_cache(
2130
+ self, opcode: str, addCode: Callable[[int, int], None]
2131
+ ) -> None:
2132
+ opcode_index = opcodes_opcode.opmap[opcode]
2133
+ if opcode_index < len(_inline_cache_entries):
2134
+ base_size = _inline_cache_entries[opcode_index]
2135
+ else:
2136
+ base_size = 0
2137
+ for _i in range(base_size):
2138
+ addCode(0, 0)
2139
+
2140
+ def make_line_table(self) -> bytes:
2141
+ lpostab = LinePositionTable(self.firstline)
2142
+
2143
+ loc = NO_LOCATION
2144
+ size = 0
2145
+ prev = None
2146
+ for t in reversed(self.insts):
2147
+ if t.loc is NEXT_LOCATION:
2148
+ if t.is_jump(self.opcode) or t.opname in SCOPE_EXIT_OPCODES:
2149
+ t.loc = NO_LOCATION
2150
+ else:
2151
+ assert prev is not None
2152
+ t.loc = prev.loc
2153
+ prev = t
2154
+
2155
+ for t in self.insts:
2156
+ if t.loc != loc:
2157
+ lpostab.emit_location(loc, size)
2158
+ loc = t.loc
2159
+ size = 0
2160
+
2161
+ # The size is in terms of code units
2162
+ size += self.instrsize(t, t.ioparg)
2163
+
2164
+ # Since the linetable format writes the end offset of bytecodes, we can't commit the
2165
+ # last write until all the instructions are iterated over.
2166
+ lpostab.emit_location(loc, size)
2167
+ return lpostab.getTable()
2168
+
2169
+ def make_exception_table(self) -> bytes:
2170
+ exception_table = ExceptionTable()
2171
+ ioffset = 0
2172
+ handler = None
2173
+ start = -1
2174
+ for instr in self.insts:
2175
+ if handler != instr.exc_handler:
2176
+ # We have hit a boundary where instr.exc_handler has changed
2177
+ if handler:
2178
+ exception_table.emit_entry(start, ioffset, handler)
2179
+ start = ioffset
2180
+ handler = instr.exc_handler
2181
+ ioffset += self.instrsize(instr, instr.ioparg)
2182
+ if handler:
2183
+ exception_table.emit_entry(start, ioffset, handler)
2184
+ return exception_table.getTable()
2185
+
2186
+ def convert_pseudo_ops(self) -> None:
2187
+ # TASK(T190611021): The graph is actually in the FINAL stage, which
2188
+ # seems wrong because we're still modifying the opcodes.
2189
+ # assert self.stage == ACTIVE, self.stage
2190
+
2191
+ for block in self.ordered_blocks:
2192
+ for instr in block.insts:
2193
+ if instr.opname in SETUP_OPCODES or instr.opname == "POP_BLOCK":
2194
+ instr.set_to_nop()
2195
+ elif instr.opname == "STORE_FAST_MAYBE_NULL":
2196
+ instr.opname = "STORE_FAST"
2197
+
2198
+ # Remove redundant NOPs added in the previous pass
2199
+ optimizer = self.flow_graph_optimizer(self)
2200
+ for block in self.ordered_blocks:
2201
+ optimizer.clean_basic_block(block, -1)
2202
+
2203
+ def assemble_final_code(self) -> None:
2204
+ """Finish assembling code object components from the final graph."""
2205
+ # see compile.c :: optimize_and_assemble_code_unit()
2206
+ self.finalize()
2207
+ assert self.stage == FINAL, self.stage
2208
+
2209
+ # TASK(T206903352): We need to pass the return value to make_code at
2210
+ # some point.
2211
+ self.prepare_localsplus()
2212
+ self.compute_stack_depth()
2213
+ self.convert_pseudo_ops()
2214
+
2215
+ self.flatten_graph()
2216
+ assert self.stage == FLAT, self.stage
2217
+ # see assemble.c :: _PyAssemble_MakeCodeObject()
2218
+ self.bytecode = self.make_byte_code()
2219
+ self.line_table = self.make_line_table()
2220
+ self.exception_table = self.make_exception_table()
2221
+
2222
+ def getCode(self) -> CodeType:
2223
+ """Get a Python code object"""
2224
+ self.assemble_final_code()
2225
+ bytecode = self.bytecode
2226
+ assert bytecode is not None
2227
+ line_table = self.line_table
2228
+ assert line_table is not None
2229
+ exception_table = self.exception_table
2230
+ assert exception_table is not None
2231
+ assert self.stage == DONE, self.stage
2232
+ return self.new_code_object(bytecode, line_table, exception_table)
2233
+
2234
+ def new_code_object(
2235
+ self, code: bytes, lnotab: bytes, exception_table: bytes
2236
+ ) -> CodeType:
2237
+ assert self.stage == DONE, self.stage
2238
+
2239
+ nlocals = len(self.varnames)
2240
+ firstline = self.firstline
2241
+ # For module, .firstline is initially not set, and should be first
2242
+ # line with actual bytecode instruction (skipping docstring, optimized
2243
+ # out instructions, etc.)
2244
+ if not firstline:
2245
+ firstline = self.first_inst_lineno
2246
+ # If no real instruction, fallback to 1
2247
+ if not firstline:
2248
+ firstline = 1
2249
+
2250
+ consts = self.getConsts()
2251
+ consts = consts + tuple(self.extra_consts)
2252
+ return self.make_code(nlocals, code, consts, firstline, lnotab, exception_table)
2253
+
2254
+ def make_code(
2255
+ self,
2256
+ nlocals: int,
2257
+ code: bytes,
2258
+ consts: tuple[object, ...],
2259
+ firstline: int,
2260
+ lnotab: bytes,
2261
+ exception_table: bytes,
2262
+ ) -> CodeType:
2263
+ # pyre-ignore[19]: Too many arguments (this is right for 3.12)
2264
+ return CodeType(
2265
+ len(self.args),
2266
+ self.posonlyargs,
2267
+ len(self.kwonlyargs),
2268
+ nlocals,
2269
+ self.stacksize,
2270
+ self.flags,
2271
+ code,
2272
+ consts,
2273
+ tuple(self.names),
2274
+ tuple(self.varnames),
2275
+ self.filename,
2276
+ self.name,
2277
+ self.qualname,
2278
+ firstline,
2279
+ lnotab,
2280
+ exception_table,
2281
+ tuple(self.freevars),
2282
+ tuple(self.cellvars),
2283
+ )
2284
+
2285
+ def _convert_LOAD_ATTR(self: PyFlowGraph, arg: object) -> int:
2286
+ # 3.12 uses the low-bit to indicate that the LOAD_ATTR is
2287
+ # part of a LOAD_ATTR/CALL sequence which loads two values,
2288
+ # the first being NULL or the object instance and the 2nd
2289
+ # being the method to be called.
2290
+ if isinstance(arg, tuple):
2291
+ return (self.names.get_index(arg[0]) << 1) | arg[1]
2292
+
2293
+ assert isinstance(arg, str)
2294
+ return self.names.get_index(arg) << 1
2295
+
2296
+ def _convert_LOAD_GLOBAL(self: PyFlowGraph, arg: object) -> int:
2297
+ assert isinstance(arg, str)
2298
+ return self.names.get_index(arg) << 1
2299
+
2300
+ COMPARISON_UNORDERED = 1
2301
+ COMPARISON_LESS_THAN = 2
2302
+ COMPARISON_GREATER_THAN = 4
2303
+ COMPARISON_EQUALS = 8
2304
+ COMPARISON_NOT_EQUALS: int = (
2305
+ COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN
2306
+ )
2307
+
2308
+ COMPARE_MASKS: dict[str, int] = {
2309
+ "<": COMPARISON_LESS_THAN,
2310
+ "<=": COMPARISON_LESS_THAN | COMPARISON_EQUALS,
2311
+ "==": COMPARISON_EQUALS,
2312
+ "!=": COMPARISON_NOT_EQUALS,
2313
+ ">": COMPARISON_GREATER_THAN,
2314
+ ">=": COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
2315
+ }
2316
+
2317
+ def _convert_COMAPRE_OP(self: PyFlowGraph, arg: object) -> int:
2318
+ assert isinstance(arg, str)
2319
+ return self.opcode.CMP_OP.index(arg) << 4 | PyFlowGraph312.COMPARE_MASKS[arg]
2320
+
2321
+ def _convert_LOAD_CLOSURE(self: PyFlowGraph, oparg: object) -> int:
2322
+ # __class__ and __classdict__ are special cased to be cell vars in classes
2323
+ # in get_ref_type in compile.c
2324
+ if isinstance(self.scope, ClassScope) and oparg in (
2325
+ "__class__",
2326
+ "__classdict__",
2327
+ ):
2328
+ return self.closure.get_index(oparg)
2329
+
2330
+ assert isinstance(oparg, str)
2331
+ if oparg in self.freevars:
2332
+ return self.freevars.get_index(oparg) + len(self.cellvars)
2333
+ return self.closure.get_index(oparg)
2334
+
2335
+ _converters: dict[str, Callable[[PyFlowGraph, object], int]] = {
2336
+ **PyFlowGraph._converters,
2337
+ "LOAD_ATTR": _convert_LOAD_ATTR,
2338
+ "LOAD_GLOBAL": _convert_LOAD_GLOBAL,
2339
+ "COMPARE_OP": _convert_COMAPRE_OP,
2340
+ "KW_NAMES": PyFlowGraph._convert_LOAD_CONST,
2341
+ "EAGER_IMPORT_NAME": PyFlowGraph._convert_NAME,
2342
+ "LOAD_CLOSURE": _convert_LOAD_CLOSURE,
2343
+ }
2344
+
2345
+ _const_opcodes: set[str] = set(PyFlowGraph._const_opcodes)
2346
+ _const_opcodes.add("RETURN_CONST")
2347
+ _const_opcodes.add("KW_NAMES")
2348
+
2349
+
2350
+ class PyFlowGraphCinder312(PyFlowGraphCinderMixin, PyFlowGraph312):
2351
+ pass
2352
+
2353
+
2354
+ class PyFlowGraph314(PyFlowGraph312):
2355
+ flow_graph_optimizer = FlowGraphOptimizer314
2356
+ _constant_idx: dict[object, int] = {
2357
+ AssertionError: 0,
2358
+ NotImplementedError: 1,
2359
+ tuple: 2,
2360
+ all: 3,
2361
+ any: 4,
2362
+ list: 5,
2363
+ set: 6,
2364
+ }
2365
+ _load_special_idx = {
2366
+ "__enter__": 0,
2367
+ "__exit__": 1,
2368
+ "__aenter__": 2,
2369
+ "__aexit__": 3,
2370
+ }
2371
+
2372
+ END_SEND_OFFSET = 5
2373
+
2374
+ def emit_format_value(self, format: int = -1) -> None:
2375
+ if format != -1:
2376
+ self.emit("CONVERT_VALUE", format)
2377
+ self.emit("FORMAT_SIMPLE")
2378
+
2379
+ def get_stack_effects(self, opname: str, oparg: object, jump: bool) -> int:
2380
+ res = self.opcode.stack_effect_raw(opname, oparg, jump)
2381
+ if opname in SETUP_OPCODES and not jump:
2382
+ return 0
2383
+
2384
+ return res
2385
+
2386
+ def flatten_jump(self, inst: Instruction, pc: int) -> int:
2387
+ res = super().flatten_jump(inst, pc)
2388
+ if inst.opname == "END_ASYNC_FOR":
2389
+ # sys.monitoring needs to be able to find the matching END_SEND
2390
+ # but the target is the SEND, so we adjust it here.
2391
+ res -= self.END_SEND_OFFSET
2392
+
2393
+ return res
2394
+
2395
+ def mark_except_handlers(self) -> None:
2396
+ for block in self.ordered_blocks:
2397
+ for instr in block.insts:
2398
+ if instr.opname in SETUP_OPCODES:
2399
+ target = instr.target
2400
+ assert target is not None, "SETUP_* opcodes all have targets"
2401
+ target.is_exc_handler = True
2402
+ break
2403
+
2404
+ def push_cold_blocks_to_end(
2405
+ self, except_handlers: set[Block] | None, optimizer: FlowGraphOptimizer
2406
+ ) -> None:
2407
+ warm = self.compute_warm()
2408
+ cold = self.compute_cold(warm)
2409
+
2410
+ # If we have a cold block with fallthrough to a warm block, add
2411
+ # an explicit jump instead of fallthrough
2412
+ for block in list(self.ordered_blocks):
2413
+ if block in cold and block.has_fallthrough and block.next in warm:
2414
+ explicit_jump = self.make_explicit_jump_block()
2415
+ explicit_jump.bid = self.get_new_block_id()
2416
+ explicit_jump.num_predecessors = 1
2417
+ cold.add(explicit_jump)
2418
+ self.current = explicit_jump
2419
+
2420
+ next_block = block.next
2421
+ assert next_block is not None
2422
+
2423
+ self.emit_jump_forward_noline(next_block)
2424
+ self.ordered_blocks.insert(
2425
+ self.ordered_blocks.index(block) + 1, explicit_jump
2426
+ )
2427
+
2428
+ explicit_jump.next = block.next
2429
+ explicit_jump.has_fallthrough = False
2430
+ block.next = explicit_jump
2431
+
2432
+ new_ordered = []
2433
+ to_end = []
2434
+ prev = None
2435
+ for block in self.ordered_blocks:
2436
+ if block in warm:
2437
+ new_ordered.append(block)
2438
+ if prev is not None:
2439
+ prev.next = block
2440
+ block.prev = prev
2441
+ prev = block
2442
+ else:
2443
+ to_end.append(block)
2444
+
2445
+ for block in to_end:
2446
+ prev.next = block
2447
+ block.prev = prev
2448
+ prev = block
2449
+
2450
+ block.next = None
2451
+
2452
+ self.ordered_blocks = new_ordered + to_end
2453
+ if to_end:
2454
+ self.remove_redundant_nops_and_jumps(optimizer)
2455
+
2456
+ def has_fallthrough(self, block: Block) -> bool:
2457
+ if not block.insts:
2458
+ return True
2459
+
2460
+ last = block.insts[-1]
2461
+ return (
2462
+ last.opname not in UNCONDITIONAL_JUMP_OPCODES
2463
+ and last.opname not in SCOPE_EXIT_OPCODES
2464
+ )
2465
+
2466
+ def compute_cold(self, warm: set[Block]) -> set[Block]:
2467
+ stack = []
2468
+ cold = set()
2469
+ visited = set()
2470
+
2471
+ for block in self.ordered_blocks:
2472
+ if block.is_exc_handler:
2473
+ assert block not in warm
2474
+ stack.append(block)
2475
+
2476
+ for block in stack:
2477
+ cold.add(block)
2478
+ next = block.next
2479
+ if next is not None and self.has_fallthrough(block):
2480
+ if next not in warm and next not in visited:
2481
+ stack.append(next)
2482
+ visited.add(next)
2483
+
2484
+ for instr in block.insts:
2485
+ if instr.is_jump(self.opcode):
2486
+ target = instr.target
2487
+ if target not in warm and target not in visited:
2488
+ stack.append(target)
2489
+ visited.add(target)
2490
+ return cold
2491
+
2492
+ def flatten_graph(self) -> None:
2493
+ # Init the ioparg of jump instructions to their index into
2494
+ # the instruction sequence. This matches CPython's behavior
2495
+ # in resolve_jump_offsets where i_target is initialized from
2496
+ # i_oparg but i_oparg is not reset to zero.
2497
+ offset = 0
2498
+ label_map: dict[Block, int] = {}
2499
+ for b in self.getBlocksInOrder():
2500
+ label_map[b] = offset
2501
+ offset += len(b.insts)
2502
+
2503
+ for b in self.getBlocksInOrder():
2504
+ for inst in b.getInstructions():
2505
+ if inst.is_jump(self.opcode):
2506
+ assert inst.target is not None, inst
2507
+ inst.ioparg = label_map[inst.target]
2508
+
2509
+ super().flatten_graph()
2510
+
2511
+ # The following bits are chosen so that the value of
2512
+ # COMPARSION_BIT(left, right)
2513
+ # masked by the values below will be non-zero if the
2514
+ # comparison is true, and zero if it is false
2515
+
2516
+ # This is for values that are unordered, ie. NaN, not types that are unordered, e.g. sets
2517
+ COMPARISON_UNORDERED = 1
2518
+
2519
+ COMPARISON_LESS_THAN = 2
2520
+ COMPARISON_GREATER_THAN = 4
2521
+ COMPARISON_EQUALS = 8
2522
+
2523
+ COMPARISON_NOT_EQUALS: int = (
2524
+ COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN
2525
+ )
2526
+
2527
+ COMPARE_MASKS: dict[str, int] = {
2528
+ "<": COMPARISON_LESS_THAN,
2529
+ "<=": COMPARISON_LESS_THAN | COMPARISON_EQUALS,
2530
+ "==": COMPARISON_EQUALS,
2531
+ "!=": COMPARISON_NOT_EQUALS,
2532
+ ">": COMPARISON_GREATER_THAN,
2533
+ ">=": COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
2534
+ }
2535
+
2536
+ def propagate_line_numbers(self) -> None:
2537
+ self.duplicate_exits_without_lineno()
2538
+ super().propagate_line_numbers()
2539
+
2540
+ def has_eval_break(self, target: Block) -> bool:
2541
+ for inst in target.insts:
2542
+ if inst.opname in ["JUMP", "RESUME", "CALL", "JUMP_BACKWARD"]:
2543
+ return True
2544
+ return False
2545
+
2546
+ def is_exit_without_line_number(self, target: Block) -> bool:
2547
+ if target.exits or self.has_eval_break(target):
2548
+ return target.has_no_lineno()
2549
+
2550
+ return False
2551
+
2552
+ @property
2553
+ def initial_stack_depth(self) -> int:
2554
+ return 0
2555
+
2556
+ def initializeConsts(self) -> None:
2557
+ # Docstring is first entry in co_consts for normal functions
2558
+ # (Other types of code objects deal with docstrings in different
2559
+ # manner, e.g. lambdas and comprehensions don't have docstrings,
2560
+ # classes store them as __doc__ attribute.
2561
+ if self.docstring is not None and not self.klass:
2562
+ self.consts[self.get_const_key(self.docstring)] = 0
2563
+
2564
+ def _convert_compare_op(self: PyFlowGraph, arg: object) -> int:
2565
+ # cmp goes in top three bits of the oparg, while the low four bits are used
2566
+ # by quickened versions of this opcode to store the comparison mask. The
2567
+ # fifth-lowest bit indicates whether the result should be converted to bool
2568
+ # and is set later):
2569
+ assert isinstance(arg, str)
2570
+ return self.opcode.CMP_OP.index(arg) << 5 | PyFlowGraph314.COMPARE_MASKS[arg]
2571
+
2572
+ _converters: dict[str, Callable[[PyFlowGraph, object], int]] = {
2573
+ **PyFlowGraph312._converters,
2574
+ "LOAD_COMMON_CONSTANT": lambda self, val: PyFlowGraph314._constant_idx[val],
2575
+ "LOAD_SPECIAL": lambda self, val: PyFlowGraph314._load_special_idx[val],
2576
+ "COMPARE_OP": _convert_compare_op,
2577
+ }
2578
+
2579
+ def get_ext_oparg(self, inst: Instruction) -> int:
2580
+ pushed = self.opcode.get_num_pushed(inst.opname, inst.oparg)
2581
+ popped = self.opcode.get_num_popped(inst.opname, inst.oparg)
2582
+ assert pushed < 4, pushed
2583
+ return popped << 2 | pushed
2584
+
2585
+ def instrsize(self, instr: Instruction, oparg: int) -> int:
2586
+ opname = instr.opname
2587
+ base_size = _inline_cache_entries.get(opname, 0)
2588
+ if opname in STATIC_OPCODES:
2589
+ # extended opcode
2590
+ base_size += 1
2591
+ extoparg = self.get_ext_oparg(instr)
2592
+ while extoparg >= 256:
2593
+ extoparg >>= 8
2594
+ base_size += 1
2595
+
2596
+ if oparg <= 0xFF:
2597
+ return 1 + base_size
2598
+ elif oparg <= 0xFFFF:
2599
+ return 2 + base_size
2600
+ elif oparg <= 0xFFFFFF:
2601
+ return 3 + base_size
2602
+ else:
2603
+ return 4 + base_size
2604
+
2605
+ def make_byte_code(self) -> bytes:
2606
+ assert self.stage == FLAT, self.stage
2607
+
2608
+ code: bytearray = bytearray()
2609
+
2610
+ def addCode(opcode: int, oparg: int) -> None:
2611
+ assert opcode < 256, opcode
2612
+ assert oparg < 256, oparg
2613
+ code.append(opcode)
2614
+ code.append(oparg)
2615
+
2616
+ for t in self.insts:
2617
+ if t.opname in STATIC_OPCODES:
2618
+ extoparg = self.get_ext_oparg(t)
2619
+
2620
+ if extoparg > 0xFFFFFF:
2621
+ addCode(self.opcode.EXTENDED_ARG, (extoparg >> 24) & 0xFF)
2622
+ if extoparg > 0xFFFF:
2623
+ addCode(self.opcode.EXTENDED_ARG, (extoparg >> 16) & 0xFF)
2624
+ if extoparg > 0xFF:
2625
+ addCode(self.opcode.EXTENDED_ARG, (extoparg >> 8) & 0xFF)
2626
+ addCode(self.opcode.EXTENDED_OPCODE, extoparg & 0xFF)
2627
+
2628
+ oparg = t.ioparg
2629
+ assert 0 <= oparg <= 0xFFFFFFFF, oparg
2630
+ if oparg > 0xFFFFFF:
2631
+ addCode(self.opcode.EXTENDED_ARG, (oparg >> 24) & 0xFF)
2632
+ if oparg > 0xFFFF:
2633
+ addCode(self.opcode.EXTENDED_ARG, (oparg >> 16) & 0xFF)
2634
+ if oparg > 0xFF:
2635
+ addCode(self.opcode.EXTENDED_ARG, (oparg >> 8) & 0xFF)
2636
+ addCode(self.opcode.opmap[t.opname], oparg & 0xFF)
2637
+ self.emit_inline_cache(t.opname, addCode)
2638
+
2639
+ self.stage = DONE
2640
+ return bytes(code)
2641
+
2642
+ def emit_inline_cache(
2643
+ self, opcode: str, addCode: Callable[[int, int], None]
2644
+ ) -> None:
2645
+ base_size = _inline_cache_entries.get(opcode, 0)
2646
+ for _i in range(base_size):
2647
+ addCode(0, 0)
2648
+
2649
+ def convert_pseudo_conditional_jumps(self) -> None:
2650
+ for block in self.getBlocksInOrder():
2651
+ for i, inst in enumerate(block.insts):
2652
+ if inst.opname == "JUMP_IF_FALSE" or inst.opname == "JUMP_IF_TRUE":
2653
+ inst.opname = (
2654
+ "POP_JUMP_IF_FALSE"
2655
+ if inst.opname == "JUMP_IF_FALSE"
2656
+ else "POP_JUMP_IF_TRUE"
2657
+ )
2658
+ block.insts[i:i] = [
2659
+ Instruction(
2660
+ "COPY", 1, 1, inst.loc, exc_handler=inst.exc_handler
2661
+ ),
2662
+ Instruction(
2663
+ "TO_BOOL", 0, 0, inst.loc, exc_handler=inst.exc_handler
2664
+ ),
2665
+ ]
2666
+
2667
+ def unlink_unreachable_basic_blocks(self, reachable_blocks: set[int]) -> None:
2668
+ for block in self.ordered_blocks:
2669
+ if block.num_predecessors == 0:
2670
+ del block.insts[:]
2671
+ block.is_exc_handler = False
2672
+ block.has_fallthrough = True
2673
+
2674
+ def get_target_block_for_exit(self, target: Block) -> Block:
2675
+ while not target.insts:
2676
+ assert target.next is not None
2677
+ target = target.next
2678
+ return target
2679
+
2680
+ def inline_small_exit_blocks(self) -> None:
2681
+ changes = True
2682
+ # every change removes a jump, ensuring convergence
2683
+ while changes:
2684
+ changes = False
2685
+ for block in self.ordered_blocks:
2686
+ if self.extend_block(block):
2687
+ changes = True
2688
+
2689
+ def should_inline_block(self, block: Block) -> bool:
2690
+ if block.exits and len(block.insts) <= MAX_COPY_SIZE:
2691
+ return True
2692
+
2693
+ if block.has_fallthrough:
2694
+ return False
2695
+
2696
+ # We can only inline blocks with no line numbers.
2697
+ for inst in block.insts:
2698
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute `lineno`.
2699
+ if inst.loc.lineno >= 0:
2700
+ return False
2701
+ return True
2702
+
2703
+ def extend_block(self, block: Block) -> bool:
2704
+ """If this block ends with an unconditional jump to an exit block,
2705
+ then remove the jump and extend this block with the target.
2706
+ """
2707
+ if len(block.insts) == 0:
2708
+ return False
2709
+ last = block.insts[-1]
2710
+ if last.opname not in UNCONDITIONAL_JUMP_OPCODES:
2711
+ return False
2712
+ target = last.target
2713
+ assert target is not None
2714
+ small_exit_block = target.exits and len(target.insts) <= MAX_COPY_SIZE
2715
+ no_line_no_fallthrough = not target.has_fallthrough and target.has_no_lineno()
2716
+ if not small_exit_block and not no_line_no_fallthrough:
2717
+ return False
2718
+ last = block.insts[-1]
2719
+ removed = last.opname
2720
+ last.set_to_nop()
2721
+ for instr in target.insts:
2722
+ block.insts.append(instr.copy())
2723
+
2724
+ last = block.insts[-1]
2725
+ if last.opname in UNCONDITIONAL_JUMP_OPCODES and removed == "JUMP":
2726
+ # Make sure we don't lose eval breaker checks
2727
+ last.opname = "JUMP"
2728
+
2729
+ block.next = None
2730
+ block.is_exit = True
2731
+ block.has_fallthrough = False
2732
+ return True
2733
+
2734
+ def finalize(self) -> None:
2735
+ """Perform final optimizations and normalization of flow graph."""
2736
+ assert self.stage == ACTIVE, self.stage
2737
+ self.stage = CLOSED
2738
+
2739
+ self.eliminate_empty_basic_blocks()
2740
+
2741
+ block = self.ordered_blocks[0]
2742
+ for i, inst in enumerate(block.insts):
2743
+ if inst.opname == "ANNOTATIONS_PLACEHOLDER":
2744
+ if self.annotations_block is not None:
2745
+ block.insts[i : i + 1] = self.annotations_block.insts
2746
+ else:
2747
+ inst.set_to_nop_no_loc()
2748
+ break
2749
+
2750
+ for block in self.ordered_blocks:
2751
+ self.normalize_basic_block(block)
2752
+
2753
+ self.optimizeCFG()
2754
+
2755
+ self.stage = CONSTS_CLOSED
2756
+ self.trim_unused_consts()
2757
+ self.duplicate_exits_without_lineno()
2758
+ self.propagate_line_numbers()
2759
+ self.firstline = self.firstline or self.first_inst_lineno or 1
2760
+
2761
+ self.stage = ORDERED
2762
+ self.stage = FINAL
2763
+
2764
+ def assemble_final_code(self) -> None:
2765
+ """Finish assembling code object components from the final graph."""
2766
+ # see compile.c :: optimize_and_assemble_code_unit()
2767
+ self.finalize()
2768
+ assert self.stage == FINAL, self.stage
2769
+
2770
+ self.convert_pseudo_conditional_jumps()
2771
+ self.compute_stack_depth()
2772
+
2773
+ # TASK(T206903352): We need to pass the return value to make_code at
2774
+ # some point.
2775
+ self.prepare_localsplus()
2776
+ self.convert_pseudo_ops()
2777
+
2778
+ self.normalize_jumps()
2779
+
2780
+ self.resolve_unconditional_jumps()
2781
+
2782
+ self.optimize_load_fast()
2783
+
2784
+ self.flatten_graph()
2785
+ assert self.stage == FLAT, self.stage
2786
+ # see assemble.c :: _PyAssemble_MakeCodeObject()
2787
+ self.bytecode = self.make_byte_code()
2788
+ self.line_table = self.make_line_table()
2789
+ self.exception_table = self.make_exception_table()
2790
+
2791
+ def convert_pseudo_ops(self) -> None:
2792
+ # TASK(T190611021): The graph is actually in the FINAL stage, which
2793
+ # seems wrong because we're still modifying the opcodes.
2794
+ # assert self.stage == ACTIVE, self.stage
2795
+
2796
+ for block in self.ordered_blocks:
2797
+ for instr in block.insts:
2798
+ if instr.opname in SETUP_OPCODES or instr.opname == "POP_BLOCK":
2799
+ instr.set_to_nop()
2800
+ elif instr.opname == "STORE_FAST_MAYBE_NULL":
2801
+ instr.opname = "STORE_FAST"
2802
+ elif instr.opname == "LOAD_CLOSURE":
2803
+ instr.opname = "LOAD_FAST"
2804
+
2805
+ # Remove redundant NOPs added in the previous pass
2806
+ optimizer = self.flow_graph_optimizer(self)
2807
+ for block in self.ordered_blocks:
2808
+ optimizer.clean_basic_block(block, -1)
2809
+
2810
+ def normalize_jumps_in_block(
2811
+ self, block: Block, seen_blocks: set[Block]
2812
+ ) -> Block | None:
2813
+ last = block.insts[-1]
2814
+ if last.opname not in self._reversed_jumps:
2815
+ return
2816
+ target = last.target
2817
+ assert target is not None
2818
+ is_forward = target.bid not in seen_blocks
2819
+ if is_forward:
2820
+ block.insts.append(Instruction("NOT_TAKEN", 0, 0, last.loc))
2821
+ return
2822
+
2823
+ # transform 'conditional jump T' to
2824
+ # 'reversed_jump b_next' followed by 'jump_backwards T'
2825
+ backwards_jump = self.newBlock("backwards_jump")
2826
+ backwards_jump.bid = self.get_new_block_id()
2827
+ self.current = backwards_jump
2828
+ # cpython has `JUMP(target)` here, but it will always get inserted as
2829
+ # the next block in the loop and then transformed to JUMP_BACKWARD by
2830
+ # the above code, since is_forward(target) won't have changed.
2831
+ self.fetch_current().emit(Instruction("NOT_TAKEN", 0, 0, last.loc))
2832
+ self.emit_with_loc("JUMP_BACKWARD", target, last.loc)
2833
+ backwards_jump.startdepth = target.startdepth
2834
+
2835
+ last.opname = self._reversed_jumps[last.opname]
2836
+ last.target = block.next
2837
+
2838
+ block.insert_next(backwards_jump)
2839
+ return backwards_jump
2840
+
2841
+ def resolve_unconditional_jumps(self) -> None:
2842
+ seen_blocks = set()
2843
+
2844
+ for b in self.getBlocksInOrder():
2845
+ seen_blocks.add(b.bid)
2846
+ for inst in b.getInstructions():
2847
+ if inst.opname == "JUMP":
2848
+ assert inst.target is not None
2849
+ is_forward = inst.target.bid not in seen_blocks
2850
+ inst.opname = "JUMP_FORWARD" if is_forward else "JUMP_BACKWARD"
2851
+ elif inst.opname == "JUMP_NO_INTERRUPT":
2852
+ assert inst.target is not None
2853
+ is_forward = inst.target.bid not in seen_blocks
2854
+ inst.opname = (
2855
+ "JUMP_FORWARD" if is_forward else "JUMP_BACKWARD_NO_INTERRUPT"
2856
+ )
2857
+
2858
+ # Helper functions for optimize_load_fast
2859
+ def kill_local(
2860
+ self, instr_flags: dict[int, int], refs: list[Ref], local: int
2861
+ ) -> None:
2862
+ """Mark references to a local as SUPPORT_KILLED."""
2863
+ for r in refs:
2864
+ if r.local == local:
2865
+ assert r.instr >= 0
2866
+ instr_flags[r.instr] |= SUPPORT_KILLED
2867
+
2868
+ def store_local(
2869
+ self, instr_flags: dict[int, int], refs: list[Ref], local: int, r: Ref
2870
+ ) -> None:
2871
+ """Kill a local and mark a reference as STORED_AS_LOCAL."""
2872
+ self.kill_local(instr_flags, refs, local)
2873
+ if r.instr != DUMMY_INSTR:
2874
+ instr_flags[r.instr] |= STORED_AS_LOCAL
2875
+
2876
+ def load_fast_push_block(
2877
+ self, stack: list[Block], target: Block, start_depth: int, visited: set[Block]
2878
+ ) -> None:
2879
+ """Add a block to the worklist for processing."""
2880
+ assert target.startdepth >= 0 and target.startdepth == start_depth, (
2881
+ f"{target.startdepth} {start_depth} {self.firstline}"
2882
+ )
2883
+ if target not in visited:
2884
+ visited.add(target)
2885
+ stack.append(target)
2886
+
2887
+ def optimize_load_fast(self) -> None:
2888
+ """
2889
+ Strength reduce LOAD_FAST{_LOAD_FAST} instructions into faster variants that
2890
+ load borrowed references onto the operand stack.
2891
+
2892
+ This is only safe when we can prove that the reference in the frame outlives
2893
+ the borrowed reference produced by the instruction.
2894
+ """
2895
+ stack = [self.entry]
2896
+ visited: set[Block] = set()
2897
+
2898
+ self.entry.startdepth = 0
2899
+ refs: list[Ref] = []
2900
+ instr_flags = defaultdict(lambda: 0) # Maps instruction index to flags
2901
+ while stack:
2902
+ block = stack.pop()
2903
+ visited.add(block)
2904
+
2905
+ # Reset per-block state
2906
+ instr_flags.clear()
2907
+
2908
+ # Reset the stack of refs. We don't track references on the stack
2909
+ # across basic blocks, but the bytecode will expect their presence.
2910
+ # Add dummy references as necessary.
2911
+ refs.clear()
2912
+ for _ in range(block.startdepth):
2913
+ refs.append(Ref(DUMMY_INSTR, NOT_LOCAL))
2914
+ for i, instr in enumerate(block.insts):
2915
+ opcode = instr.opname
2916
+ ioparg = instr.ioparg
2917
+ oparg = instr.oparg
2918
+
2919
+ # Process instruction based on opcode
2920
+ if opcode == "DELETE_FAST":
2921
+ self.kill_local(instr_flags, refs, ioparg)
2922
+
2923
+ elif opcode == "LOAD_FAST":
2924
+ refs.append(Ref(i, ioparg))
2925
+
2926
+ elif opcode == "LOAD_FAST_AND_CLEAR":
2927
+ self.kill_local(instr_flags, refs, ioparg)
2928
+ refs.append(Ref(i, ioparg))
2929
+
2930
+ elif opcode == "LOAD_FAST_LOAD_FAST":
2931
+ # Extract the two locals from the combined oparg
2932
+ local1 = ioparg >> 4
2933
+ local2 = ioparg & 0xF
2934
+ refs.append(Ref(i, local1))
2935
+ refs.append(Ref(i, local2))
2936
+
2937
+ elif opcode == "STORE_FAST":
2938
+ r = refs.pop()
2939
+ self.store_local(instr_flags, refs, ioparg, r)
2940
+
2941
+ elif opcode == "STORE_FAST_LOAD_FAST":
2942
+ # STORE_FAST
2943
+ r = refs.pop()
2944
+ local1 = ioparg >> 4
2945
+ local2 = ioparg & 15
2946
+ self.store_local(instr_flags, refs, local1, r)
2947
+ # LOAD_FAST
2948
+ refs.append(Ref(i, local2))
2949
+
2950
+ elif opcode == "STORE_FAST_STORE_FAST":
2951
+ # STORE_FAST
2952
+ r = refs.pop()
2953
+ local1 = ioparg >> 4
2954
+ self.store_local(instr_flags, refs, local1, r)
2955
+ # STORE_FAST
2956
+ r = refs.pop()
2957
+ local2 = ioparg & 15
2958
+ self.store_local(instr_flags, refs, local2, r)
2959
+
2960
+ # Handle stack manipulation opcodes
2961
+ elif opcode == "COPY":
2962
+ assert ioparg > 0
2963
+ idx = len(refs) - ioparg
2964
+ r = refs[idx]
2965
+ refs.append(Ref(r.instr, r.local))
2966
+
2967
+ elif opcode == "SWAP":
2968
+ assert ioparg >= 2
2969
+ idx = len(refs) - ioparg
2970
+ refs[idx], refs[-1] = refs[-1], refs[idx]
2971
+
2972
+ # We treat opcodes that do not consume all of their inputs on
2973
+ # a case by case basis, as we have no generic way of knowing
2974
+ # how many inputs should be left on the stack.
2975
+
2976
+ # Opcodes that consume no inputs
2977
+ elif opcode in NO_INPUT_INSTRS:
2978
+ delta = self.opcode.stack_effect_raw(opcode, oparg, False)
2979
+ assert delta >= 0
2980
+ for _ in range(delta):
2981
+ refs.append(Ref(i, NOT_LOCAL))
2982
+
2983
+ # Opcodes that consume some inputs and push no new values
2984
+ elif opcode in (
2985
+ "DICT_MERGE",
2986
+ "DICT_UPDATE",
2987
+ "LIST_APPEND",
2988
+ "LIST_EXTEND",
2989
+ "MAP_ADD",
2990
+ "RERAISE",
2991
+ "SET_ADD",
2992
+ "SET_UPDATE",
2993
+ ):
2994
+ num_popped = self.opcode.get_num_popped(opcode, oparg)
2995
+ num_pushed = self.opcode.get_num_pushed(opcode, oparg)
2996
+ net_popped = num_popped - num_pushed
2997
+ assert net_popped > 0
2998
+ for _ in range(net_popped):
2999
+ refs.pop()
3000
+
3001
+ elif opcode in ("END_SEND", "SET_FUNCTION_ATTRIBUTE"):
3002
+ assert self.opcode.stack_effect_raw(opcode, oparg, False) == -1
3003
+ tos = refs.pop()
3004
+ refs.pop() # Pop the second item
3005
+ refs.append(Ref(tos.instr, tos.local)) # Push back the top item
3006
+
3007
+ # Handle opcodes that consume some inputs and push new values
3008
+ elif opcode == "CHECK_EXC_MATCH":
3009
+ refs.pop()
3010
+ refs.append(Ref(i, NOT_LOCAL))
3011
+
3012
+ elif opcode == "FOR_ITER":
3013
+ if instr.target:
3014
+ self.load_fast_push_block(
3015
+ stack, instr.target, len(refs) + 1, visited
3016
+ )
3017
+ refs.append(Ref(i, NOT_LOCAL))
3018
+
3019
+ elif opcode in ("LOAD_ATTR", "LOAD_SUPER_ATTR"):
3020
+ self_ref = refs.pop()
3021
+ if opcode == "LOAD_SUPER_ATTR":
3022
+ refs.pop() # Pop super type
3023
+ refs.pop() # Pop super object
3024
+ refs.append(Ref(i, NOT_LOCAL))
3025
+ if ioparg & 1:
3026
+ # A method call; conservatively assume self is pushed back
3027
+ refs.append(Ref(self_ref.instr, self_ref.local))
3028
+
3029
+ elif opcode in ("LOAD_SPECIAL", "PUSH_EXC_INFO"):
3030
+ tos = refs.pop()
3031
+ refs.append(Ref(i, NOT_LOCAL))
3032
+ refs.append(Ref(tos.instr, tos.local))
3033
+
3034
+ elif opcode == "SEND":
3035
+ assert instr.target
3036
+ self.load_fast_push_block(stack, instr.target, len(refs), visited)
3037
+ refs.pop()
3038
+ refs.append(Ref(i, NOT_LOCAL))
3039
+
3040
+ # Opcodes that consume all of their inputs
3041
+ else:
3042
+ num_popped = self.opcode.get_num_popped(opcode, oparg)
3043
+ num_pushed = self.opcode.get_num_pushed(opcode, oparg)
3044
+
3045
+ if instr.target and self.opcode.has_jump(self.opcode.opmap[opcode]):
3046
+ self.load_fast_push_block(
3047
+ stack,
3048
+ instr.target,
3049
+ len(refs) - num_popped + num_pushed,
3050
+ visited,
3051
+ )
3052
+
3053
+ if opcode not in SETUP_OPCODES:
3054
+ # Block push opcodes only affect the stack when jumping to the target
3055
+ for _ in range(num_popped):
3056
+ refs.pop()
3057
+ for _ in range(num_pushed):
3058
+ refs.append(Ref(i, NOT_LOCAL))
3059
+
3060
+ # Push fallthrough block
3061
+ if block.has_fallthrough and block.next:
3062
+ self.load_fast_push_block(stack, block.next, len(refs), visited)
3063
+
3064
+ # Mark instructions that produce values that are on the stack at the end of the basic block
3065
+ for r in refs:
3066
+ if r.instr != -1:
3067
+ instr_flags[r.instr] = instr_flags.get(r.instr, 0) | REF_UNCONSUMED
3068
+
3069
+ # Optimize instructions
3070
+ for i, instr in enumerate(block.insts):
3071
+ if i not in instr_flags:
3072
+ if instr.opname == "LOAD_FAST":
3073
+ instr.opname = "LOAD_FAST_BORROW"
3074
+ elif instr.opname == "LOAD_FAST_LOAD_FAST":
3075
+ instr.opname = "LOAD_FAST_BORROW_LOAD_FAST_BORROW"
3076
+
3077
+ def get_generator_prefix(self) -> list[Instruction]:
3078
+ firstline = self.firstline or self.first_inst_lineno or 1
3079
+ loc = SrcLocation(firstline, firstline, -1, -1)
3080
+ return [
3081
+ Instruction("RETURN_GENERATOR", 0, loc=loc),
3082
+ Instruction("POP_TOP", 0, loc=loc),
3083
+ ]
3084
+
3085
+ def emit_jump_forward_noline(self, target: Block) -> None:
3086
+ self.emit_noline("JUMP_NO_INTERRUPT", target)
3087
+
3088
+ def make_explicit_jump_block(self) -> Block:
3089
+ res = super().make_explicit_jump_block()
3090
+ res.num_predecessors = 1
3091
+ return res
3092
+
3093
+ def label_exception_targets(self) -> None:
3094
+ def push_todo_block(block: Block) -> None:
3095
+ todo_stack.append(block)
3096
+ visited.add(block)
3097
+
3098
+ todo_stack: list[Block] = [self.entry]
3099
+ visited: set[Block] = {self.entry}
3100
+ except_stack = []
3101
+ self.entry.except_stack = except_stack
3102
+
3103
+ while todo_stack:
3104
+ block = todo_stack.pop()
3105
+ assert block in visited
3106
+ except_stack = block.except_stack
3107
+ block.except_stack = []
3108
+ handler = except_stack[-1] if except_stack else None
3109
+ last_yield_except_depth = -1
3110
+ for instr in block.insts:
3111
+ if instr.opname in SETUP_OPCODES:
3112
+ target = instr.target
3113
+ assert target, instr
3114
+ if target not in visited:
3115
+ # Copy the current except stack into the target's except stack
3116
+ target.except_stack = list(except_stack)
3117
+ push_todo_block(target)
3118
+ handler = self.push_except_block(except_stack, instr)
3119
+ elif instr.opname == "POP_BLOCK":
3120
+ except_stack.pop()
3121
+ handler = except_stack[-1] if except_stack else None
3122
+ instr.set_to_nop()
3123
+ elif instr.is_jump(self.opcode) and instr.opname != "END_ASYNC_FOR":
3124
+ instr.exc_handler = handler
3125
+ if instr.target not in visited:
3126
+ target = instr.target
3127
+ assert target
3128
+ if block.has_fallthrough:
3129
+ # Copy the current except stack into the block's except stack
3130
+ target.except_stack = list(except_stack)
3131
+ else:
3132
+ # Move the current except stack to the block and start a new one
3133
+ target.except_stack = except_stack
3134
+ except_stack = []
3135
+ push_todo_block(target)
3136
+ elif instr.opname == "YIELD_VALUE":
3137
+ last_yield_except_depth = len(except_stack)
3138
+ instr.exc_handler = handler
3139
+ elif instr.opname == "RESUME":
3140
+ instr.exc_handler = handler
3141
+ if instr.oparg != ResumeOparg.ScopeEntry:
3142
+ if last_yield_except_depth == 1:
3143
+ instr.ioparg |= ResumeOparg.Depth1Mask
3144
+ last_yield_except_depth = -1
3145
+ else:
3146
+ instr.exc_handler = handler
3147
+
3148
+ if block.has_fallthrough and block.next and block.next not in visited:
3149
+ assert except_stack is not None
3150
+ block.next.except_stack = except_stack
3151
+ push_todo_block(block.next)
3152
+
3153
+ def make_super_instruction(
3154
+ self, inst1: Instruction, inst2: Instruction, super_op: str
3155
+ ) -> None:
3156
+ # pyre-ignore[16]: lineno is maybe not defined on AST
3157
+ line1 = inst1.loc.lineno
3158
+ # pyre-ignore[16]: lineno is maybe not defined on AST
3159
+ line2 = inst2.loc.lineno
3160
+ # Skip if instructions are on different lines
3161
+ if line1 >= 0 and line2 >= 0 and line1 != line2:
3162
+ return
3163
+
3164
+ if inst1.ioparg >= 16 or inst2.ioparg >= 16:
3165
+ return
3166
+
3167
+ inst1.opname = super_op
3168
+ inst1.ioparg = (inst1.ioparg << 4) | inst2.ioparg
3169
+ inst2.set_to_nop()
3170
+
3171
+ def insert_superinstructions(self) -> None:
3172
+ for block in self.ordered_blocks:
3173
+ for i, instr in enumerate(block.insts):
3174
+ if i + 1 == len(block.insts):
3175
+ break
3176
+
3177
+ next_instr = block.insts[i + 1]
3178
+ if instr.opname == "LOAD_FAST":
3179
+ if next_instr.opname == "LOAD_FAST":
3180
+ self.make_super_instruction(
3181
+ instr, next_instr, "LOAD_FAST_LOAD_FAST"
3182
+ )
3183
+ elif instr.opname == "STORE_FAST":
3184
+ if next_instr.opname == "LOAD_FAST":
3185
+ self.make_super_instruction(
3186
+ instr, next_instr, "STORE_FAST_LOAD_FAST"
3187
+ )
3188
+ elif next_instr.opname == "STORE_FAST":
3189
+ self.make_super_instruction(
3190
+ instr, next_instr, "STORE_FAST_STORE_FAST"
3191
+ )
3192
+
3193
+ def remove_redundant_jumps(
3194
+ self, optimizer: FlowGraphOptimizer, clean: bool = True
3195
+ ) -> bool:
3196
+ # Delete jump instructions made redundant by previous step. If a non-empty
3197
+ # block ends with a jump instruction, check if the next non-empty block
3198
+ # reached through normal flow control is the target of that jump. If it
3199
+ # is, then the jump instruction is redundant and can be deleted.
3200
+ maybe_empty_blocks = False
3201
+ for block in self.ordered_blocks:
3202
+ if not block.insts:
3203
+ continue
3204
+ last = block.insts[-1]
3205
+ if last.opname not in UNCONDITIONAL_JUMP_OPCODES:
3206
+ continue
3207
+ target = last.target
3208
+ while not target.insts:
3209
+ target = target.next
3210
+ next = block.next
3211
+ while next and not next.insts:
3212
+ next = next.next
3213
+ if target == next:
3214
+ block.has_fallthrough = True
3215
+ last.set_to_nop()
3216
+ maybe_empty_blocks = True
3217
+ return maybe_empty_blocks
3218
+
3219
+ def remove_redundant_nops_and_jumps(self, optimizer: FlowGraphOptimizer) -> None:
3220
+ while True:
3221
+ removed = False
3222
+ for block in self.ordered_blocks:
3223
+ removed |= optimizer.clean_basic_block(block, -1)
3224
+
3225
+ removed |= self.remove_redundant_jumps(optimizer, False)
3226
+ if not removed:
3227
+ break
3228
+
3229
+ def optimizeCFG(self) -> None:
3230
+ """Optimize a well-formed CFG."""
3231
+ self.mark_except_handlers()
3232
+
3233
+ self.label_exception_targets()
3234
+
3235
+ assert self.stage == CLOSED, self.stage
3236
+
3237
+ self.inline_small_exit_blocks()
3238
+
3239
+ self.remove_unreachable_basic_blocks()
3240
+ self.propagate_line_numbers()
3241
+
3242
+ const_optimizer = FlowGraphConstOptimizer314(self)
3243
+ for block in self.ordered_blocks:
3244
+ const_optimizer.optimize_basic_block(block)
3245
+
3246
+ optimizer = self.flow_graph_optimizer(self)
3247
+ for block in self.ordered_blocks:
3248
+ optimizer.optimize_basic_block(block)
3249
+
3250
+ self.remove_redundant_nops_and_pairs(optimizer)
3251
+
3252
+ for block in self.ordered_blocks:
3253
+ # remove redundant nops
3254
+ optimizer.clean_basic_block(block, -1)
3255
+
3256
+ self.remove_redundant_nops_and_pairs(optimizer)
3257
+ self.remove_unreachable_basic_blocks()
3258
+ self.remove_redundant_nops_and_jumps(optimizer)
3259
+
3260
+ self.stage = OPTIMIZED
3261
+
3262
+ self.remove_unused_consts()
3263
+ self.add_checks_for_loads_of_uninitialized_variables()
3264
+ self.insert_superinstructions()
3265
+ self.push_cold_blocks_to_end(None, optimizer)
3266
+ self.propagate_line_numbers()
3267
+
3268
+ _const_opcodes: set[str] = set(PyFlowGraph312._const_opcodes) | {"LOAD_SMALL_INT"}
3269
+
3270
+
3271
+ class PyFlowGraph315(PyFlowGraph314):
3272
+ def maybe_propagate_location(
3273
+ self, instr: Instruction, loc: AST | SrcLocation
3274
+ ) -> None:
3275
+ if instr.lineno == NO_LOCATION.lineno:
3276
+ instr.loc = loc
3277
+
3278
+ def propagate_line_numbers(self) -> None:
3279
+ """Propagate line numbers to instructions without."""
3280
+ self.duplicate_exits_without_lineno()
3281
+ for block in self.ordered_blocks:
3282
+ if not block.insts:
3283
+ continue
3284
+ prev_loc = NO_LOCATION
3285
+ for instr in block.insts:
3286
+ self.maybe_propagate_location(instr, prev_loc)
3287
+ prev_loc = instr.loc
3288
+ if block.has_fallthrough:
3289
+ next = block.next
3290
+ assert next
3291
+ if next.num_predecessors == 1 and next.insts:
3292
+ self.maybe_propagate_location(next.insts[0], prev_loc)
3293
+ last_instr = block.insts[-1]
3294
+ if (
3295
+ last_instr.is_jump(self.opcode)
3296
+ and last_instr.opname not in SETUP_OPCODES
3297
+ ):
3298
+ # Only actual jumps, not exception handlers
3299
+ target = last_instr.target
3300
+ assert target
3301
+ while not target.insts and target.num_predecessors == 1:
3302
+ target = target.next
3303
+ assert target
3304
+ # CPython doesn't check target.insts and instead can write
3305
+ # to random memory...
3306
+ if target.num_predecessors == 1 and target.insts:
3307
+ self.maybe_propagate_location(target.insts[0], prev_loc)
3308
+
3309
+
3310
+ # Constants for reference tracking flags
3311
+ SUPPORT_KILLED = (
3312
+ 1 # The loaded reference is still on the stack when the local is killed
3313
+ )
3314
+ STORED_AS_LOCAL = 2 # The loaded reference is stored into a local
3315
+ REF_UNCONSUMED = (
3316
+ 4 # The loaded reference is still on the stack at the end of the basic block
3317
+ )
3318
+
3319
+ # Special values for reference tracking
3320
+ NOT_LOCAL = -1 # Reference does not refer to a local
3321
+ DUMMY_INSTR = -1 # Reference was not produced by an instruction
3322
+
3323
+
3324
+ @dataclass
3325
+ class Ref:
3326
+ """Represents a reference on the stack."""
3327
+
3328
+ instr: int # Index of instruction that produced the reference or DUMMY_INSTR
3329
+ local: int # The local to which the reference refers or NOT_LOCAL
3330
+
3331
+
3332
+ class UninitializedVariableChecker:
3333
+ # Opcodes which may clear a variable
3334
+ clear_ops = (
3335
+ "DELETE_FAST",
3336
+ "LOAD_FAST_AND_CLEAR",
3337
+ "STORE_FAST_MAYBE_NULL",
3338
+ )
3339
+ # Opcodes which guarantee that a variable is stored
3340
+ stored_ops = ("STORE_FAST", "LOAD_FAST_CHECK")
3341
+
3342
+ def __init__(self, flow_graph: PyFlowGraph312) -> None:
3343
+ self.flow_graph = flow_graph
3344
+ self.stack: list[Block] = []
3345
+ self.unsafe_locals: dict[Block, int] = {}
3346
+ self.visited: set[Block] = set()
3347
+
3348
+ def check(self) -> None:
3349
+ nlocals = len(self.flow_graph.varnames)
3350
+ nparams = (
3351
+ len(self.flow_graph.args)
3352
+ + len(self.flow_graph.starargs)
3353
+ + len(self.flow_graph.kwonlyargs)
3354
+ )
3355
+
3356
+ if nlocals == 0:
3357
+ return
3358
+
3359
+ if nlocals > 64:
3360
+ # To avoid O(nlocals**2) compilation, locals beyond the first
3361
+ # 64 are only analyzed one basicblock at a time: initialization
3362
+ # info is not passed between basicblocks.
3363
+ self.fast_scan_many_locals(nlocals)
3364
+ nlocals = 64
3365
+
3366
+ # First origin of being uninitialized:
3367
+ # The non-parameter locals in the entry block.
3368
+ start_mask = 0
3369
+ for i in range(nparams, nlocals):
3370
+ start_mask |= 1 << i
3371
+
3372
+ self.maybe_push(self.flow_graph.entry, start_mask)
3373
+
3374
+ # Second origin of being uninitialized:
3375
+ # There could be DELETE_FAST somewhere, so
3376
+ # be sure to scan each basicblock at least once.
3377
+ for b in self.flow_graph.ordered_blocks:
3378
+ self.scan_block_for_locals(b)
3379
+
3380
+ # Now propagate the uncertainty from the origins we found: Use
3381
+ # LOAD_FAST_CHECK for any LOAD_FAST where the local could be undefined.
3382
+ while self.stack:
3383
+ block = self.stack.pop()
3384
+ self.visited.remove(block)
3385
+ self.scan_block_for_locals(block)
3386
+
3387
+ def scan_block_for_locals(self, block: Block) -> None:
3388
+ unsafe_mask = self.unsafe_locals.get(block, 0)
3389
+ for instr in block.insts:
3390
+ if instr.exc_handler is not None:
3391
+ self.maybe_push(instr.exc_handler, unsafe_mask)
3392
+
3393
+ if instr.ioparg >= 64:
3394
+ continue
3395
+
3396
+ bit = 1 << instr.ioparg
3397
+ if instr.opname in self.clear_ops:
3398
+ unsafe_mask |= bit
3399
+ elif instr.opname in self.stored_ops:
3400
+ unsafe_mask &= ~bit
3401
+ elif instr.opname == "LOAD_FAST":
3402
+ if unsafe_mask & bit:
3403
+ instr.opname = "LOAD_FAST_CHECK"
3404
+ unsafe_mask &= ~bit
3405
+
3406
+ if block.next and block.has_fallthrough:
3407
+ self.maybe_push(block.next, unsafe_mask)
3408
+
3409
+ if block.insts and block.insts[-1].is_jump(self.flow_graph.opcode):
3410
+ target = block.insts[-1].target
3411
+ assert target is not None
3412
+ self.maybe_push(target, unsafe_mask)
3413
+
3414
+ def fast_scan_many_locals(self, nlocals: int) -> None:
3415
+ states = [0] * (nlocals - 64)
3416
+ block_num = 0
3417
+ # state[i - 64] == blocknum if local i is guaranteed to
3418
+ # be initialized, i.e., if it has had a previous LOAD_FAST or
3419
+ # STORE_FAST within that basicblock (not followed by
3420
+ # DELETE_FAST/LOAD_FAST_AND_CLEAR/STORE_FAST_MAYBE_NULL).
3421
+ for block in self.flow_graph.ordered_blocks:
3422
+ block_num += 1
3423
+ for instr in block.insts:
3424
+ if instr.ioparg < 64:
3425
+ continue
3426
+
3427
+ arg = instr.ioparg - 64
3428
+ if instr.opname in self.clear_ops:
3429
+ states[arg] = block_num - 1
3430
+ elif instr.opname == "STORE_FAST":
3431
+ states[arg] = block_num
3432
+ elif instr.opname == "LOAD_FAST":
3433
+ if states[arg] != block_num:
3434
+ instr.opname = "LOAD_FAST_CHECK"
3435
+ states[arg] = block_num
3436
+
3437
+ def maybe_push(self, block: Block, unsafe_mask: int) -> None:
3438
+ block_unsafe = self.unsafe_locals.get(block, 0)
3439
+ both = block_unsafe | unsafe_mask
3440
+ if block_unsafe != both:
3441
+ self.unsafe_locals[block] = both
3442
+ if block not in self.visited:
3443
+ self.stack.append(block)
3444
+ self.visited.add(block)
3445
+
3446
+
3447
+ class LineAddrTable:
3448
+ """linetable / lnotab
3449
+
3450
+ This class builds the linetable, which is documented in
3451
+ Objects/lnotab_notes.txt. Here's a brief recap:
3452
+
3453
+ For each new lineno after the first one, two bytes are added to the
3454
+ linetable. (In some cases, multiple two-byte entries are added.) The first
3455
+ byte is the distance in bytes between the instruction for the current lineno
3456
+ and the next lineno. The second byte is offset in line numbers. If either
3457
+ offset is greater than 255, multiple two-byte entries are added -- see
3458
+ lnotab_notes.txt for the delicate details.
3459
+
3460
+ """
3461
+
3462
+ def __init__(self) -> None:
3463
+ self.current_line = 0
3464
+ self.prev_line = 0
3465
+ self.linetable: list[int] = []
3466
+
3467
+ def setFirstLine(self, lineno: int) -> None:
3468
+ self.current_line = lineno
3469
+ self.prev_line = lineno
3470
+
3471
+ def nextLine(self, lineno: int, start: int, end: int) -> None:
3472
+ assert lineno
3473
+ self.emitCurrentLine(start, end)
3474
+
3475
+ if self.current_line >= 0:
3476
+ self.prev_line = self.current_line
3477
+ self.current_line = lineno
3478
+
3479
+ def emitCurrentLine(self, start: int, end: int) -> None:
3480
+ # compute deltas
3481
+ addr_delta = end - start
3482
+ if not addr_delta:
3483
+ return
3484
+ if self.current_line < 0:
3485
+ line_delta = -128
3486
+ else:
3487
+ line_delta = self.current_line - self.prev_line
3488
+ while line_delta < -127 or 127 < line_delta:
3489
+ if line_delta < 0:
3490
+ k = -127
3491
+ else:
3492
+ k = 127
3493
+ self.push_entry(0, k)
3494
+ line_delta -= k
3495
+
3496
+ while addr_delta > 254:
3497
+ self.push_entry(254, line_delta)
3498
+ line_delta = -128 if self.current_line < 0 else 0
3499
+ addr_delta -= 254
3500
+
3501
+ assert -128 <= line_delta and line_delta <= 127
3502
+ self.push_entry(addr_delta, line_delta)
3503
+
3504
+ def getTable(self) -> bytes:
3505
+ return bytes(self.linetable)
3506
+
3507
+ def push_entry(self, addr_delta: int, line_delta: int) -> None:
3508
+ self.linetable.append(addr_delta)
3509
+ self.linetable.append(cast_signed_byte_to_unsigned(line_delta))
3510
+
3511
+
3512
+ class CodeLocationInfoKind(IntEnum):
3513
+ SHORT0 = 0
3514
+ ONE_LINE0 = 10
3515
+ ONE_LINE1 = 11
3516
+ ONE_LINE2 = 12
3517
+ NO_COLUMNS = 13
3518
+ LONG = 14
3519
+ NONE = 15
3520
+
3521
+
3522
+ class LinePositionTable:
3523
+ """Generates the Python 3.12 and later position table which tracks
3524
+ line numbers as well as column information."""
3525
+
3526
+ def __init__(self, firstline: int) -> None:
3527
+ self.linetable = bytearray()
3528
+ self.lineno = firstline
3529
+
3530
+ # https://github.com/python/cpython/blob/3.12/Python/assemble.c#L170
3531
+ # https://github.com/python/cpython/blob/3.12/Objects/locations.md
3532
+ def emit_location(self, loc: AST | SrcLocation, size: int) -> None:
3533
+ if size == 0:
3534
+ return
3535
+
3536
+ while size > 8:
3537
+ self.write_entry(loc, 8)
3538
+ size -= 8
3539
+
3540
+ self.write_entry(loc, size)
3541
+
3542
+ def write_entry(self, loc: AST | SrcLocation, size: int) -> None:
3543
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute `lineno`.
3544
+ if loc.lineno < 0:
3545
+ return self.write_entry_no_location(size)
3546
+
3547
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute `lineno`.
3548
+ line_delta = loc.lineno - self.lineno
3549
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3550
+ # `col_offset`.
3551
+ column = loc.col_offset
3552
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3553
+ # `end_col_offset`.
3554
+ end_column = loc.end_col_offset
3555
+ assert isinstance(end_column, int)
3556
+ assert column >= -1
3557
+ assert end_column >= -1
3558
+ if column < 0 or end_column < 0:
3559
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3560
+ # `end_lineno`.
3561
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3562
+ # `lineno`.
3563
+ if loc.end_lineno == loc.lineno or loc.end_lineno == -1:
3564
+ self.write_no_column(size, line_delta)
3565
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3566
+ # `lineno`.
3567
+ self.lineno = loc.lineno
3568
+ return
3569
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3570
+ # `end_lineno`.
3571
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute `lineno`.
3572
+ elif loc.end_lineno == loc.lineno:
3573
+ if (
3574
+ line_delta == 0
3575
+ and column < 80
3576
+ and end_column - column < 16
3577
+ and end_column >= column
3578
+ ):
3579
+ return self.write_short_form(size, column, end_column)
3580
+ if 0 <= line_delta < 3 and column < 128 and end_column < 128:
3581
+ self.write_one_line_form(size, line_delta, column, end_column)
3582
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3583
+ # `lineno`.
3584
+ self.lineno = loc.lineno
3585
+ return
3586
+
3587
+ self.write_long_form(loc, size)
3588
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute `lineno`.
3589
+ self.lineno = loc.lineno
3590
+
3591
+ def write_short_form(self, size: int, column: int, end_column: int) -> None:
3592
+ assert size > 0 and size <= 8
3593
+ column_low_bits = column & 7
3594
+ column_group = column >> 3
3595
+ assert column < 80
3596
+ assert end_column >= column
3597
+ assert end_column - column < 16
3598
+ self.write_first_byte(CodeLocationInfoKind.SHORT0 + column_group, size)
3599
+ # Start column / end column
3600
+ self.write_byte((column_low_bits << 4) | (end_column - column))
3601
+
3602
+ def write_one_line_form(
3603
+ self, size: int, line_delta: int, column: int, end_column: int
3604
+ ) -> None:
3605
+ assert size > 0 and size <= 8
3606
+ assert line_delta >= 0 and line_delta < 3
3607
+ assert column < 128
3608
+ assert end_column < 128
3609
+ # Start line delta
3610
+ self.write_first_byte(CodeLocationInfoKind.ONE_LINE0 + line_delta, size)
3611
+ self.write_byte(column) # Start column
3612
+ self.write_byte(end_column) # End column
3613
+
3614
+ def write_no_column(self, size: int, line_delta: int) -> None:
3615
+ self.write_first_byte(CodeLocationInfoKind.NO_COLUMNS, size)
3616
+ self.write_signed_varint(line_delta) # Start line delta
3617
+
3618
+ def write_long_form(self, loc: AST | SrcLocation, size: int) -> None:
3619
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3620
+ # `end_lineno`.
3621
+ end_lineno = loc.end_lineno
3622
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3623
+ # `end_col_offset`.
3624
+ end_col_offset = loc.end_col_offset
3625
+
3626
+ assert size > 0 and size <= 8
3627
+ assert end_lineno is not None and end_col_offset is not None
3628
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute `lineno`.
3629
+ assert end_lineno >= loc.lineno
3630
+
3631
+ self.write_first_byte(CodeLocationInfoKind.LONG, size)
3632
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute `lineno`.
3633
+ self.write_signed_varint(loc.lineno - self.lineno) # Start line delta
3634
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute `lineno`.
3635
+ self.write_varint(end_lineno - loc.lineno) # End line delta
3636
+ # pyre-fixme[16]: Item `AST` of `AST | SrcLocation` has no attribute
3637
+ # `col_offset`.
3638
+ self.write_varint(loc.col_offset + 1) # Start column
3639
+ self.write_varint(end_col_offset + 1) # End column
3640
+
3641
+ def write_entry_no_location(self, size: int) -> None:
3642
+ self.write_first_byte(CodeLocationInfoKind.NONE, size)
3643
+
3644
+ def write_varint(self, value: int) -> None:
3645
+ while value >= 64:
3646
+ self.linetable.append(0x40 | (value & 0x3F))
3647
+ value >>= 6
3648
+
3649
+ self.linetable.append(value)
3650
+
3651
+ def write_signed_varint(self, value: int) -> None:
3652
+ if value < 0:
3653
+ uval = ((-value) << 1) | 1
3654
+ else:
3655
+ uval = value << 1
3656
+
3657
+ self.write_varint(uval)
3658
+
3659
+ def write_first_byte(self, code: int, length: int) -> None:
3660
+ assert code & 0x0F == code
3661
+ self.linetable.append(0x80 | (code << 3) | (length - 1))
3662
+
3663
+ def write_byte(self, code: int) -> None:
3664
+ self.linetable.append(code & 0xFF)
3665
+
3666
+ def getTable(self) -> bytes:
3667
+ return bytes(self.linetable)
3668
+
3669
+
3670
+ class ExceptionTable:
3671
+ """Generates the Python 3.12+ exception table."""
3672
+
3673
+ # https://github.com/python/cpython/blob/3.12/Objects/exception_handling_notes.txt
3674
+
3675
+ def __init__(self) -> None:
3676
+ self.exception_table: bytearray = bytearray()
3677
+
3678
+ def getTable(self) -> bytes:
3679
+ return bytes(self.exception_table)
3680
+
3681
+ def write_byte(self, byte: int) -> None:
3682
+ self.exception_table.append(byte)
3683
+
3684
+ def emit_item(self, value: int, msb: int) -> None:
3685
+ assert (msb | 128) == 128
3686
+ assert value >= 0 and value < (1 << 30)
3687
+ CONTINUATION_BIT = 64
3688
+
3689
+ if value > (1 << 24):
3690
+ self.write_byte((value >> 24) | CONTINUATION_BIT | msb)
3691
+ msb = 0
3692
+ for i in (18, 12, 6):
3693
+ if value >= (1 << i):
3694
+ v = (value >> i) & 0x3F
3695
+ self.write_byte(v | CONTINUATION_BIT | msb)
3696
+ msb = 0
3697
+ self.write_byte((value & 0x3F) | msb)
3698
+
3699
+ def emit_entry(self, start: int, end: int, handler: Block) -> None:
3700
+ size = end - start
3701
+ assert end > start
3702
+ target = handler.offset
3703
+ depth = handler.startdepth - 1
3704
+ if handler.preserve_lasti:
3705
+ depth = depth - 1
3706
+ assert depth >= 0
3707
+ depth_lasti = (depth << 1) | int(handler.preserve_lasti)
3708
+ self.emit_item(start, (1 << 7))
3709
+ self.emit_item(size, 0)
3710
+ self.emit_item(target, 0)
3711
+ self.emit_item(depth_lasti, 0)