unidecompiler-plugin-python-pyc 0.1.1__tar.gz → 0.1.3__tar.gz

{unidecompiler_plugin_python_pyc-0.1.1 → unidecompiler_plugin_python_pyc-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unidecompiler-plugin-python-pyc
-Version: 0.1.1
+Version: 0.1.3
 Summary: Python bytecode frontend plugin for unidecompiler
 Author-email: Wker <1670133844@qq.com>
 License-Expression: AGPL-3.0-or-later
@@ -9,7 +9,7 @@ Project-URL: Repository, https://github.com/Wker666/unidecompiler
 Project-URL: Issues, https://github.com/Wker666/unidecompiler/issues
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: unidecompiler<0.2.0,>=0.1.1
+Requires-Dist: unidecompiler<0.2.0,>=0.1.6
 
 # unidecompiler-plugin-python-pyc
 

{unidecompiler_plugin_python_pyc-0.1.1 → unidecompiler_plugin_python_pyc-0.1.3}/pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unidecompiler-plugin-python-pyc"
-version = "0.1.1"
+version = "0.1.3"
 description = "Python bytecode frontend plugin for unidecompiler"
 readme = "README.md"
 license = "AGPL-3.0-or-later"
 authors = [{ name = "Wker", email = "1670133844@qq.com" }]
 requires-python = ">=3.11"
-dependencies = ["unidecompiler>=0.1.1,<0.2.0"]
+dependencies = ["unidecompiler>=0.1.6,<0.2.0"]
 
 [project.urls]
 Homepage = "https://github.com/Wker666/unidecompiler"

{unidecompiler_plugin_python_pyc-0.1.1 → unidecompiler_plugin_python_pyc-0.1.3}/src/unidecompiler_plugin_python_pyc/lifter.py

@@ -32,11 +32,13 @@ from unidecompiler.core.effects import (
     Iterate,
     MakeFunctionValue,
     MergeMap,
+    OfferImplicitCallArguments,
     Pop,
     Push,
     RaiseTop,
     RaiseWithCause,
     ReraiseTop,
+    ResumeValue,
     ReturnTop,
     ReturnVoid,
     StoreAttr,
@@ -55,7 +57,6 @@ from unidecompiler.core.effects import (
 )
 from unidecompiler.core.vm_module import assemble_vm_module
 from unidecompiler.core.ir import (
-    Assign,
     BinaryOp,
     CapturedVar,
     Const,
@@ -79,18 +80,17 @@ from unidecompiler.core.vm_function import (
 from unidecompiler.core.vm_bytecode import VMBytecodeStep
 from unidecompiler.core.vm_hints import VMHint
 from unidecompiler.core.vm_operands import VMDecodedInstruction, VMOperand
+from unidecompiler.provenance import ByteRange
 from unidecompiler.core.vm_region import (
     VMLinearState,
     VMRegionCallbacks,
+    VMRegionSlice,
     VMRegionOpcodeClasses,
     VMRegionProfile,
     VMStatefulCallbacks,
     build_hint_region_profile,
     lift_control_region as lift_vm_control_region,
 )
-from unidecompiler.core.vm_structures import (
-    vm_unsupported,
-)
 from unidecompiler_plugin_python_pyc.pyc import PycCodeObject, PycExceptionRegion, PycModule
 
 
@@ -121,7 +121,6 @@ BINARY_SYMBOLS = {
 COMPLEX_OPS = {
 }
 IGNORED_OPS = {
-    "RESUME",
     "CACHE",
     "NOP",
     "EXTENDED_ARG",
@@ -129,7 +128,7 @@ IGNORED_OPS = {
     "COPY_FREE_VARS",
 }
 PYTHON_REGION_OPCODE_CLASSES = VMRegionOpcodeClasses(
-    noise=frozenset({"END_FOR", "POP_TOP", "NOP", "RESUME"}),
+    noise=frozenset({"END_FOR", "POP_TOP", "NOP"}),
     control=frozenset(
         {
             "POP_JUMP_IF_FALSE",
@@ -282,7 +281,10 @@ def _build_set(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
 
 
 def _get_iter(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
-    return (Iterate(source=source),)
+    return (
+        Iterate(source=source),
+        OfferImplicitCallArguments(source=source, max_explicit_arg_count=0),
+    )
 
 
 def _unpack_sequence(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
@@ -359,7 +361,10 @@ def _binary_slice(_context, _instruction, source: SourceRef) -> tuple[Effect, ..
 
 
 def _store_slice(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
-    return (StoreItemEffect(source=source),)
+    return (
+        BuildCall(source=source, arg_count=2, callee=Global(name="slice", source=source)),
+        StoreItemEffect(source=source, order="value-obj-key"),
+    )
 
 
 def _list_extend(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
@@ -442,8 +447,8 @@ def _map_add(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
     return (StoreItemAtDepth(source=source, depth=_instruction_count(instruction)),)
 
 
-def _set_add(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
-    return (SetAdd(source=source),)
+def _set_add(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
+    return (SetAdd(source=source, depth=_instruction_count(instruction)),)
 
 
 def _store_local(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
@@ -483,6 +488,13 @@ def _yield_value(_context, instruction, source: SourceRef) -> tuple[Effect, ...]
     return (YieldTop(source=source, default=Const(value=None, source=source)),)
 
 
+def _resume(_context, instruction, source: SourceRef) -> tuple[Effect, ...]:
+    # CPython's resume mode 5 continues a ``yield`` expression, whose value
+    # is supplied by the caller.  Await/send modes manage their result in the
+    # surrounding protocol and must not manufacture an extra stack value.
+    return (ResumeValue(source=source),) if _instruction_count(instruction) == 5 else ()
+
+
 def _send(_context, _instruction, source: SourceRef) -> tuple[Effect, ...]:
     return (Pop(source=source, count=1, allow_missing=True),)
 
@@ -604,6 +616,7 @@ PYTHON_EFFECT_TABLE = VMEffectTable(
         "BUILD_TUPLE": _build_tuple,
         "BUILD_SET": _build_set,
         "SET_ADD": _set_add,
+        "RESUME": _resume,
         "GET_ITER": _get_iter,
         "GET_YIELD_FROM_ITER": _get_iter,
         "UNPACK_SEQUENCE": _unpack_sequence,
@@ -683,8 +696,8 @@ def lift_code_object(code: PycCodeObject) -> FunctionIR:
         _python_function_spec(code),
         steps,
         profile=_python_region_profile(steps, instructions),
-        callbacks=_python_region_callbacks(code, instructions),
-        stateful_callbacks=_python_stateful_callbacks(code, instructions),
+        callbacks_factory=lambda normalized_steps: _python_region_callbacks(code, normalized_steps),
+        stateful_callbacks_factory=lambda normalized_steps: _python_stateful_callbacks(code, normalized_steps),
         initial_locals=_python_initial_locals(code),
         raw_window=lambda index: _raw_instruction_window(instructions, index),
     )
@@ -811,6 +824,7 @@ def _python_decoded_instruction(instruction, source: SourceRef) -> VMDecodedInst
         source=source,
         operands=tuple(operands),
         raw=_raw_instruction_line(instruction),
+        artifact_range=(None if instruction.artifact_offset is None or instruction.size is None else ByteRange(instruction.artifact_offset, instruction.size)),
     )
 
 
@@ -882,21 +896,40 @@ def _python_initial_locals(code: PycCodeObject) -> dict[str, Expr]:
 
 def _python_region_callbacks(
     code: PycCodeObject,
-    instructions: tuple[object, ...],
+    prepared_steps: tuple[VMBytecodeStep, ...],
 ) -> VMRegionCallbacks[VMBytecodeStep]:
+    def lift_slice(slice_start, slice_end, stack):
+        slice_steps = prepared_steps[slice_start:slice_end]
+        result = lift_steps(
+            slice_steps,
+            initial_locals=_python_initial_locals(code),
+            initial_stack=stack,
+        )
+        if result.state.diagnostics:
+            return VMRegionSlice(stopped_at=slice_start)
+        output: list[object] = list(result.state.statements)
+        if result.stopped_at is not None and result.state.terminator is None:
+            return VMRegionSlice(
+                statements=tuple(output),
+                stopped_at=slice_start + slice_steps.index(result.stopped_at),
+            )
+        if result.state.terminator is not None:
+            output.append(result.state.terminator)
+        return VMRegionSlice(statements=tuple(output))
+
+    def lift_expr(slice_start, slice_end, stack):
+        result = lift_steps(
+            prepared_steps[slice_start:slice_end],
+            initial_locals=_python_initial_locals(code),
+            initial_stack=stack,
+        )
+        if result.state.diagnostics or not result.state.stack:
+            return None
+        return result.state.stack[-1]
+
     return VMRegionCallbacks(
-        lift_slice=lambda slice_start, slice_end, stack: _lift_instruction_slice(
-            code,
-            instructions[slice_start:slice_end],
-            [],
-            stack,
-        ),
-        lift_expr=lambda slice_start, slice_end, stack: _lift_stack_expr(
-            code,
-            instructions[slice_start:slice_end],
-            [],
-            stack,
-        ),
+        lift_slice=lift_slice,
+        lift_expr=lift_expr,
         lift_iter_loop=lambda _get_iter_index, _iterable: None,
         lift_async_iter_loop=lambda _prefix_start, _get_aiter_index, _region_end: None,
         lift_comprehension=lambda _prefix_start, _get_iter_index, _region_end, _iterable: None,
@@ -905,45 +938,38 @@ def _python_region_callbacks(
 
 def _python_stateful_callbacks(
     code: PycCodeObject,
-    instructions: tuple[object, ...],
+    prepared_steps: tuple[VMBytecodeStep, ...],
 ) -> VMStatefulCallbacks[VMBytecodeStep]:
+    def lift_linear(start, end, locals_, stack):
+        slice_steps = prepared_steps[start:end]
+        result = lift_steps(
+            slice_steps,
+            initial_locals=locals_,
+            initial_stack=stack,
+        )
+        stopped_at_terminal_end = (
+            result.stopped_at is slice_steps[-1]
+            if slice_steps and result.state.terminator is not None
+            else False
+        )
+        if result.state.diagnostics or (
+            result.stopped_at is not None and not stopped_at_terminal_end
+        ):
+            return None
+        return VMLinearState(
+            locals=result.state.locals,
+            stack=tuple(result.state.stack),
+            statements=tuple(result.state.statements),
+            terminator=result.state.terminator,
+        )
+
     return VMStatefulCallbacks(
         initial_locals=lambda: _python_initial_locals(code),
-        lift_linear=lambda start, end, locals, stack: _python_linear_state(
-            instructions[start:end],
-            locals,
-            stack,
-        ),
+        lift_linear=lift_linear,
         branch_condition=_python_branch_condition,
     )
 
 
-def _python_linear_state(
-    instructions: tuple[object, ...],
-    initial_locals: dict[str, Expr],
-    initial_stack: tuple[Expr, ...],
-) -> VMLinearState | None:
-    result = _run_python_stack_slice(
-        _PycLinearLocals(tuple(initial_locals)),
-        instructions,
-        dict(initial_locals),
-        initial_stack,
-    )
-    stopped_at_terminal_end = (
-        getattr(result.stopped_at, "source", None) == _python_source(instructions[-1])
-        if instructions and result.state.terminator is not None
-        else False
-    )
-    if result.state.diagnostics or (result.stopped_at is not None and not stopped_at_terminal_end):
-        return None
-    return VMLinearState(
-        locals=result.state.locals,
-        stack=tuple(result.state.stack),
-        statements=tuple(result.state.statements),
-        terminator=result.state.terminator,
-    )
-
-
 def _python_branch_condition(branch: VMBytecodeStep, stack: tuple[Expr, ...]) -> Expr | None:
     if not stack:
         return None
@@ -960,11 +986,6 @@ def _python_branch_condition(branch: VMBytecodeStep, stack: tuple[Expr, ...]) ->
     return condition
 
 
-class _PycLinearLocals:
-    def __init__(self, varnames: tuple[str, ...]) -> None:
-        self.varnames = varnames
-
-
 def _python_region_profile(
     steps: tuple[VMBytecodeStep, ...],
     instructions: tuple[object, ...],
@@ -997,70 +1018,6 @@ def _await_region_end(instructions: tuple[object, ...], start: int, end: int) ->
     return None
 
 
-def _lift_instruction_slice(
-    code: PycCodeObject,
-    instructions: tuple[object, ...],
-    assignments: list[Assign],
-    preloaded_stack: tuple[Expr, ...] = (),
-) -> tuple[object, ...]:
-    if not instructions:
-        return ()
-    initial_locals = {
-        name: Var(name=name, source=SourceRef(frontend="python-pyc", detail=f"local:{name}"))
-        for name in code.varnames
-    }
-    for assignment in assignments:
-        initial_locals[assignment.target.name] = assignment.target
-    result = _run_python_stack_slice(code, instructions, initial_locals, preloaded_stack)
-    if result.state.diagnostics:
-        return ()
-    output: list[object] = list(result.state.statements)
-    if result.stopped_at is not None and result.state.terminator is None:
-        try:
-            stopped_index = instructions.index(result.stopped_at)
-        except ValueError:
-            stopped_index = 0
-        output.append(
-            vm_unsupported(
-                source=result.stopped_at.source,
-                message="unsupported region",
-                detail=f"stopped at {result.stopped_at.opcode}",
-                raw=(result.stopped_at.raw,) if result.stopped_at.raw else _raw_instruction_window(instructions, stopped_index),
-            )
-        )
-    if result.state.terminator is not None:
-        output.append(result.state.terminator)
-    return tuple(output)
-
-
-def _lift_stack_expr(
-    code: PycCodeObject,
-    instructions: tuple[object, ...],
-    assignments: list[Assign],
-    preloaded_stack: tuple[Expr, ...] = (),
-) -> Expr | None:
-    initial_locals = {
-        name: Var(name=name, source=SourceRef(frontend="python-pyc", detail=f"local:{name}"))
-        for name in code.varnames
-    }
-    for assignment in assignments:
-        initial_locals[assignment.target.name] = assignment.target
-    result = _run_python_stack_slice(code, instructions, initial_locals, preloaded_stack)
-    if result.state.diagnostics or not result.state.stack:
-        return None
-    return result.state.stack[-1]
-
-
-def _run_python_stack_slice(
-    code: PycCodeObject,
-    instructions: tuple[object, ...],
-    initial_locals: dict[str, Expr],
-    initial_stack: tuple[Expr, ...] = (),
-):
-    steps = _python_bytecode_steps(instructions)
-    return lift_steps(steps, initial_locals=initial_locals, initial_stack=initial_stack)
-
-
 def _instruction_count(instruction) -> int:
     return instruction.arg if isinstance(instruction.arg, int) else 0
 

{unidecompiler_plugin_python_pyc-0.1.1 → unidecompiler_plugin_python_pyc-0.1.3}/src/unidecompiler_plugin_python_pyc/pyc.py

@@ -24,6 +24,8 @@ class PycInstruction:
     argval: object
     argrepr: str
     starts_line: int | None
+    artifact_offset: int | None = None
+    size: int | None = None
 
 
 @dataclass(frozen=True)
@@ -51,6 +53,7 @@ class PycCodeObject:
     instructions: tuple[PycInstruction, ...]
     exception_regions: tuple[PycExceptionRegion, ...]
     children: tuple["PycCodeObject", ...]
+    artifact_code_offset: int | None = None
 
 
 @dataclass(frozen=True)
@@ -85,17 +88,19 @@ def decode_pyc(data: bytes, filename: str | None = None) -> PycModule:
     return PycModule(
         magic=data[:4],
         flags=flags,
-        code=_decode_code_object(code),
+        code=_decode_code_object(code, _marshal_code_offsets(data, code)),
         filename=filename,
     )
 
 
-def _decode_code_object(code: types.CodeType) -> PycCodeObject:
+def _decode_code_object(code: types.CodeType, code_offsets: dict[int, int]) -> PycCodeObject:
     children = tuple(
-        _decode_code_object(const)
+        _decode_code_object(const, code_offsets)
         for const in code.co_consts
         if isinstance(const, types.CodeType)
     )
+    instructions = tuple(dis.get_instructions(code))
+    code_start = code_offsets.get(id(code))
     return PycCodeObject(
         name=code.co_name,
         argcount=code.co_argcount,
@@ -119,8 +124,10 @@ def _decode_code_object(code: types.CodeType) -> PycCodeObject:
                 argval=instruction.argval,
                 argrepr=instruction.argrepr,
                 starts_line=instruction.starts_line,
+                artifact_offset=None if code_start is None else code_start + instruction.offset,
+                size=None if code_start is None else _instruction_size(instructions, index, len(code.co_code)),
             )
-            for instruction in dis.get_instructions(code)
+            for index, instruction in enumerate(instructions)
         ),
         exception_regions=tuple(
             PycExceptionRegion(
@@ -133,4 +140,58 @@ def _decode_code_object(code: types.CodeType) -> PycCodeObject:
             for entry in dis.Bytecode(code).exception_entries
         ),
         children=children,
+        artifact_code_offset=code_start,
     )
+
+
+def _instruction_size(instructions: tuple[dis.Instruction, ...], index: int, code_size: int) -> int:
+    next_offset = instructions[index + 1].offset if index + 1 < len(instructions) else code_size
+    return next_offset - instructions[index].offset
+
+
+def _marshal_code_offsets(data: bytes, root: types.CodeType) -> dict[int, int]:
+    """Find only unambiguous marshal byte-string records for ``co_code`` values.
+
+    CPython serializes code bytes as a marshal ``TYPE_STRING`` record.  The
+    frontend verifies both that record's tag and its length, then accepts a
+    location only when that code byte sequence has exactly one candidate.  A
+    repeated or otherwise ambiguous value is deliberately left unmapped.
+    """
+    code_objects = tuple(_walk_code_objects(root))
+    candidates: dict[bytes, list[int]] = {}
+    for code in code_objects:
+        payload = code.co_code
+        if payload in candidates:
+            continue
+        candidates[payload] = _marshal_string_payload_offsets(data, payload)
+    assigned: set[int] = set()
+    offsets: dict[int, int] = {}
+    for code in code_objects:
+        matches = [offset for offset in candidates[code.co_code] if offset not in assigned]
+        if len(matches) == 1:
+            offsets[id(code)] = matches[0]
+            assigned.add(matches[0])
+    return offsets
+
+
+def _walk_code_objects(code: types.CodeType):
+    yield code
+    for value in code.co_consts:
+        if isinstance(value, types.CodeType):
+            yield from _walk_code_objects(value)
+
+
+def _marshal_string_payload_offsets(data: bytes, payload: bytes) -> list[int]:
+    if not payload:
+        return []
+    result: list[int] = []
+    size = len(payload)
+    for header in range(PYC_MIN_HEADER_SIZE, len(data) - size - 4):
+        if (data[header] & 0x7F) != ord("s"):
+            continue
+        if int.from_bytes(data[header + 1 : header + 5], "little", signed=True) != size:
+            continue
+        start = header + 5
+        if data[start : start + size] == payload:
+            result.append(start)
+    return result

{unidecompiler_plugin_python_pyc-0.1.1 → unidecompiler_plugin_python_pyc-0.1.3}/src/unidecompiler_plugin_python_pyc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unidecompiler-plugin-python-pyc
-Version: 0.1.1
+Version: 0.1.3
 Summary: Python bytecode frontend plugin for unidecompiler
 Author-email: Wker <1670133844@qq.com>
 License-Expression: AGPL-3.0-or-later
@@ -9,7 +9,7 @@ Project-URL: Repository, https://github.com/Wker666/unidecompiler
 Project-URL: Issues, https://github.com/Wker666/unidecompiler/issues
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: unidecompiler<0.2.0,>=0.1.1
+Requires-Dist: unidecompiler<0.2.0,>=0.1.6
 
 # unidecompiler-plugin-python-pyc
 

unidecompiler_plugin_python_pyc-0.1.3/src/unidecompiler_plugin_python_pyc.egg-info/requires.txt

@@ -0,0 +1 @@
+unidecompiler<0.2.0,>=0.1.6

unidecompiler_plugin_python_pyc-0.1.1/src/unidecompiler_plugin_python_pyc.egg-info/requires.txt

@@ -1 +0,0 @@
-unidecompiler<0.2.0,>=0.1.1