angr 9.2.139__py3-none-manylinux2014_x86_64.whl → 9.2.141__py3-none-manylinux2014_x86_64.whl

angr/analyses/decompiler/ail_simplifier.py

@@ -27,6 +27,7 @@ from ailment.expression import (
 from angr.analyses.s_propagator import SPropagatorAnalysis
 from angr.analyses.s_reaching_definitions import SRDAModel
 from angr.utils.ail import is_phi_assignment, HasExprWalker
+from angr.utils.ssa import has_call_in_between_stmts, has_store_stmt_in_between_stmts, has_load_expr_in_between_stmts
 from angr.code_location import CodeLocation, ExternalCodeLocation
 from angr.sim_variable import SimStackVariable, SimMemoryVariable, SimVariable
 from angr.knowledge_plugins.propagations.states import Equivalence
@@ -98,6 +99,7 @@ class AILSimplifier(Analysis):
         removed_vvar_ids: set[int] | None = None,
         arg_vvars: dict[int, tuple[VirtualVariable, SimVariable]] | None = None,
         avoid_vvar_ids: set[int] | None = None,
+        secondary_stackvars: set[int] | None = None,
     ):
         self.func = func
         self.func_graph = func_graph if func_graph is not None else func.graph
@@ -118,6 +120,7 @@ class AILSimplifier(Analysis):
         self._arg_vvars = arg_vvars
         self._avoid_vvar_ids = avoid_vvar_ids
         self._propagator_dead_vvar_ids: set[int] = set()
+        self._secondary_stackvars: set[int] = secondary_stackvars if secondary_stackvars is not None else set()
 
         self._calls_to_remove: set[CodeLocation] = set()
         self._assignments_to_remove: set[CodeLocation] = set()
@@ -292,7 +295,7 @@ class AILSimplifier(Analysis):
 
         narrowed = False
 
-        addr_and_idx_to_block: dict[tuple[int, int], Block] = {}
+        addr_and_idx_to_block: dict[tuple[int, int | None], Block] = {}
         for block in self.func_graph.nodes():
             addr_and_idx_to_block[(block.addr, block.idx)] = block
 
@@ -424,6 +427,7 @@ class AILSimplifier(Analysis):
                 return ExprNarrowingInfo(False)
 
             block = self.blocks.get(old_block, old_block)
+            assert loc.stmt_idx is not None
             if loc.stmt_idx >= len(block.statements):
                 # missing a statement for whatever reason
                 return ExprNarrowingInfo(False)
@@ -551,14 +555,23 @@ class AILSimplifier(Analysis):
                 return None, None
             return expr.size, ("expr", (expr,))
 
-        first_op = walker.operations[0]
+        ops = walker.operations
+        first_op = ops[0]
+        if isinstance(first_op, BinaryOp) and first_op.op in {"Add", "Sub"}:
+            # expr + x
+            ops = ops[1:]
+            if not ops:
+                if expr is None:
+                    return None, None
+                return expr.size, ("expr", (expr,))
+            first_op = ops[0]
         if isinstance(first_op, Convert) and first_op.to_bits >= self.project.arch.byte_width:
             # we need at least one byte!
             return first_op.to_bits // self.project.arch.byte_width, ("convert", (first_op,))
         if isinstance(first_op, BinaryOp):
             second_op = None
-            if len(walker.operations) >= 2:
-                second_op = walker.operations[1]
+            if len(ops) >= 2:
+                second_op = ops[1]
             if (
                 first_op.op == "And"
                 and isinstance(first_op.operands[1], Const)
@@ -623,9 +636,9 @@ class AILSimplifier(Analysis):
             block = blocks_by_addr_and_idx[(block_addr, block_idx)]
 
             # only replace loads if there are stack arguments in this block
-            replace_loads = insn_addrs_using_stack_args is not None and {
-                stmt.ins_addr for stmt in block.statements
-            }.intersection(insn_addrs_using_stack_args)
+            replace_loads: bool = insn_addrs_using_stack_args is not None and bool(
+                {stmt.ins_addr for stmt in block.statements}.intersection(insn_addrs_using_stack_args)
+            )
 
             # remove virtual variables in the avoid list
             if self._avoid_vvar_ids:
@@ -662,7 +675,7 @@ class AILSimplifier(Analysis):
         if not equivalence:
             return simplified
 
-        addr_and_idx_to_block: dict[tuple[int, int], Block] = {}
+        addr_and_idx_to_block: dict[tuple[int, int | None], Block] = {}
         for block in self.func_graph.nodes():
             addr_and_idx_to_block[(block.addr, block.idx)] = block
 
@@ -943,6 +956,8 @@ class AILSimplifier(Analysis):
                     for use_loc in all_use_locs:
                         if use_loc == eq.codeloc:
                             continue
+                        assert use_loc.block_addr is not None
+                        assert use_loc.stmt_idx is not None
                         block = addr_and_idx_to_block[(use_loc.block_addr, use_loc.block_idx)]
                         stmt = block.statements[use_loc.stmt_idx]
                         if isinstance(stmt, Assignment) or (isinstance(replace_with, Load) and isinstance(stmt, Store)):
@@ -954,11 +969,15 @@ class AILSimplifier(Analysis):
 
                 remove_initial_assignment = False  # expression folding will take care of it
 
+            assert replace_with is not None
+
             if any(not isinstance(use_and_expr[1], VirtualVariable) for _, use_and_expr in all_uses_with_def):
                 # if any of the uses are phi assignments, we skip
                 used_in_phi_assignment = False
                 for _, use_and_expr in all_uses_with_def:
                     u = use_and_expr[0]
+                    assert u.block_addr is not None
+                    assert u.stmt_idx is not None
                     block = addr_and_idx_to_block[(u.block_addr, u.block_idx)]
                     stmt = block.statements[u.stmt_idx]
                     if is_phi_assignment(stmt):
@@ -1120,8 +1139,6 @@ class AILSimplifier(Analysis):
         than once after simplification and graph structuring where conditions might be duplicated (e.g., in Dream).
         In such cases, the one-use expression folder in RegionSimplifier will perform this transformation.
         """
-        # Disabled until https://github.com/angr/angr/issues/5112 and related folding issues are fixed
-        return False
 
         # pylint:disable=unreachable
         simplified = False
@@ -1130,7 +1147,7 @@ class AILSimplifier(Analysis):
         if not equivalence:
             return simplified
 
-        addr_and_idx_to_block: dict[tuple[int, int], Block] = {}
+        addr_and_idx_to_block: dict[tuple[int, int | None], Block] = {}
         for block in self.func_graph.nodes():
             addr_and_idx_to_block[(block.addr, block.idx)] = block
 
@@ -1168,6 +1185,8 @@ class AILSimplifier(Analysis):
                     ),
                     eq.codeloc,
                 )
+                assert the_def.codeloc.block_addr is not None
+                assert the_def.codeloc.stmt_idx is not None
 
                 all_uses: set[tuple[CodeLocation, Any]] = set(rd.get_vvar_uses_with_expr(the_def.atom))
 
@@ -1176,6 +1195,8 @@ class AILSimplifier(Analysis):
                 u, used_expr = next(iter(all_uses))
                 if used_expr is None:
                     continue
+                assert u.block_addr is not None
+                assert u.stmt_idx is not None
 
                 if u in def_locations_to_remove:
                     # this use site has been altered by previous folding attempts. the corresponding statement will be
@@ -1196,41 +1217,28 @@ class AILSimplifier(Analysis):
                 if u.block_addr not in {b.addr for b in super_node_blocks}:
                     continue
 
-                # check if the register has been overwritten by statements in between the def site and the use site
-                # usesite_atom_defs = set(rd.get_defs(the_def.atom, u, OP_BEFORE))
-                # if len(usesite_atom_defs) != 1:
-                #     continue
-                # usesite_atom_def = next(iter(usesite_atom_defs))
-                # if usesite_atom_def != the_def:
-                #     continue
-
-                # check if any atoms that the call relies on has been overwritten by statements in between the def site
-                # and the use site.
-                # TODO: Prove non-interference
-                # defsite_all_expr_uses = set(rd.all_uses.get_uses_by_location(the_def.codeloc))
-                # defsite_used_atoms = set()
-                # for dd in defsite_all_expr_uses:
-                #     defsite_used_atoms.add(dd.atom)
-                # usesite_expr_def_outdated = False
-                # for defsite_expr_atom in defsite_used_atoms:
-                #     usesite_expr_uses = set(rd.get_defs(defsite_expr_atom, u, OP_BEFORE))
-                #     if not usesite_expr_uses:
-                #         # the atom is not defined at the use site - it's fine
-                #         continue
-                #     defsite_expr_uses = set(rd.get_defs(defsite_expr_atom, the_def.codeloc, OP_BEFORE))
-                #     if usesite_expr_uses != defsite_expr_uses:
-                #         # special case: ok if this atom is assigned to at the def site and has not been overwritten
-                #         if len(usesite_expr_uses) == 1:
-                #             usesite_expr_use = next(iter(usesite_expr_uses))
-                #             if usesite_expr_use.atom == defsite_expr_atom and (
-                #                 usesite_expr_use.codeloc == the_def.codeloc
-                #                 or usesite_expr_use.codeloc.block_addr == call_addr
-                #             ):
-                #                 continue
-                #         usesite_expr_def_outdated = True
-                #         break
-                # if usesite_expr_def_outdated:
-                #     continue
+                # ensure there are no other calls between the def site and the use site.
+                # this is because we do not want to alter the order of calls.
+                u_inclusive = CodeLocation(u.block_addr, u.stmt_idx + 1, block_idx=u.block_idx)
+                # note that the target statement being a store is fine
+                if (
+                    has_call_in_between_stmts(
+                        self.func_graph,
+                        addr_and_idx_to_block,
+                        the_def.codeloc,
+                        u_inclusive,
+                        skip_if_contains_vvar=the_def.atom.varid,
+                    )
+                    or has_store_stmt_in_between_stmts(self.func_graph, addr_and_idx_to_block, the_def.codeloc, u)
+                    or has_load_expr_in_between_stmts(
+                        self.func_graph,
+                        addr_and_idx_to_block,
+                        the_def.codeloc,
+                        u_inclusive,
+                        skip_if_contains_vvar=the_def.atom.varid,
+                    )
+                ):
+                    continue
 
                 # check if there are any calls in between the def site and the use site
                 if self._count_calls_in_supernodeblocks(super_node_blocks, the_def.codeloc, u) > 0:
@@ -1316,8 +1324,8 @@ class AILSimplifier(Analysis):
         # keeping tracking of statements to remove and statements (as well as dead vvars) to keep allows us to handle
         # cases where a statement defines more than one atoms, e.g., a call statement that defines both the return
         # value and the floating-point return value.
-        stmts_to_remove_per_block: dict[tuple[int, int], set[int]] = defaultdict(set)
-        stmts_to_keep_per_block: dict[tuple[int, int], set[int]] = defaultdict(set)
+        stmts_to_remove_per_block: dict[tuple[int, int | None], set[int]] = defaultdict(set)
+        stmts_to_keep_per_block: dict[tuple[int, int | None], set[int]] = defaultdict(set)
         dead_vvar_ids: set[int] = set()
 
         # Find all statements that should be removed
@@ -1342,8 +1350,12 @@ class AILSimplifier(Analysis):
                         if rd.is_phi_vvar_id(def_.atom.varid):
                             # we always remove unused phi variables
                             pass
+                        elif def_.atom.varid in self._secondary_stackvars:
+                            # secondary stack variables are potentially removable
+                            pass
                         elif stackarg_offsets is not None:
                             # we always remove definitions for stack arguments
+                            assert def_.atom.stack_offset is not None
                             if (def_.atom.stack_offset & mask) not in stackarg_offsets:
                                 continue
                         else:
@@ -1364,11 +1376,18 @@ class AILSimplifier(Analysis):
                     dead_vvar_ids.add(def_.atom.varid)
 
                 if not isinstance(def_.codeloc, ExternalCodeLocation):
+                    assert def_.codeloc.block_addr is not None
+                    assert def_.codeloc.stmt_idx is not None
                     stmts_to_remove_per_block[(def_.codeloc.block_addr, def_.codeloc.block_idx)].add(
                         def_.codeloc.stmt_idx
                     )
             else:
-                stmts_to_keep_per_block[(def_.codeloc.block_addr, def_.codeloc.block_idx)].add(def_.codeloc.stmt_idx)
+                if not isinstance(def_.codeloc, ExternalCodeLocation):
+                    assert def_.codeloc.block_addr is not None
+                    assert def_.codeloc.stmt_idx is not None
+                    stmts_to_keep_per_block[(def_.codeloc.block_addr, def_.codeloc.block_idx)].add(
+                        def_.codeloc.stmt_idx
+                    )
 
         # find all phi variables that rely on variables that no longer exist
         all_removed_var_ids = self._removed_vvar_ids.copy()
@@ -1380,6 +1399,7 @@ class AILSimplifier(Analysis):
                     vvarid in removed_vvar_ids for vvarid in phi_use_varids
                 ):
                     loc = rd.all_vvar_definitions[rd.varid_to_vvar[phi_varid]]
+                    assert loc.block_addr is not None and loc.stmt_idx is not None
                     stmts_to_remove_per_block[(loc.block_addr, loc.block_idx)].add(loc.stmt_idx)
                     new_removed_vvar_ids.add(phi_varid)
                     all_removed_var_ids.add(phi_varid)
@@ -1391,11 +1411,13 @@ class AILSimplifier(Analysis):
         redundant_phi_and_dirty_varids = self._find_cyclic_dependent_phis_and_dirty_vvars(rd)
         for varid in redundant_phi_and_dirty_varids:
             loc = rd.all_vvar_definitions[rd.varid_to_vvar[varid]]
+            assert loc.block_addr is not None and loc.stmt_idx is not None
             stmts_to_remove_per_block[(loc.block_addr, loc.block_idx)].add(loc.stmt_idx)
             stmts_to_keep_per_block[(loc.block_addr, loc.block_idx)].discard(loc.stmt_idx)
 
         for codeloc in self._calls_to_remove | self._assignments_to_remove:
             # this call can be removed. make sure it exists in stmts_to_remove_per_block
+            assert codeloc.block_addr is not None and codeloc.stmt_idx is not None
             stmts_to_remove_per_block[codeloc.block_addr, codeloc.block_idx].add(codeloc.stmt_idx)
 
         simplified = False
@@ -1488,8 +1510,36 @@ class AILSimplifier(Analysis):
 
         return simplified
 
+    @staticmethod
+    def _get_vvar_used_by(
+        vvar_id: int, rd: SRDAModel, blocks_dict: dict[tuple[int, int | None], Block]
+    ) -> set[int | None]:
+        """
+        Get all atoms that use a specified virtual variable. The atoms are in the form of virtual variable ID or None
+        (indicating the virtual variable is used by another statement like Store).
+
+        :param vvar_id:     ID of the virtual variable.
+        :param rd:          The SRDA model.
+        :return:            The set of vvar use atoms.
+        """
+
+        vvar = rd.varid_to_vvar[vvar_id]
+        used_by: set[int | None] = set()
+        for used_vvar, loc in rd.all_vvar_uses[vvar]:
+            if used_vvar is None:
+                # no explicit reference
+                used_by.add(None)
+            elif loc.block_addr is not None:
+                assert loc.stmt_idx is not None
+                stmt = blocks_dict[(loc.block_addr, loc.block_idx)].statements[loc.stmt_idx]
+                if isinstance(stmt, Assignment) and isinstance(stmt.dst, VirtualVariable):
+                    used_by.add(stmt.dst.varid)
+                else:
+                    used_by.add(None)
+        return used_by
+
     def _find_cyclic_dependent_phis_and_dirty_vvars(self, rd: SRDAModel) -> set[int]:
-        blocks_dict = {(bb.addr, bb.idx): bb for bb in self.func_graph}
+        blocks_dict: dict[tuple[int, int | None], Block] = {(bb.addr, bb.idx): bb for bb in self.func_graph}
 
         # find dirty vvars and vexccall vvars
         dirty_vvar_ids = set()
@@ -1505,25 +1555,14 @@ class AILSimplifier(Analysis):
 
         phi_and_dirty_vvar_ids = rd.phi_vvar_ids | dirty_vvar_ids
 
-        vvar_used_by: dict[int, set[int]] = defaultdict(set)
+        vvar_used_by: dict[int, set[int | None]] = defaultdict(set)
         for var_id in phi_and_dirty_vvar_ids:
             if var_id in rd.phivarid_to_varids:
                 for used_by_varid in rd.phivarid_to_varids[var_id]:
-                    vvar_used_by[used_by_varid].add(var_id)
-
-            vvar = rd.varid_to_vvar[var_id]
-            used_by = set()
-            for used_vvar, loc in rd.all_vvar_uses[vvar]:
-                if used_vvar is None:
-                    # no explicit reference
-                    used_by.add(None)
-                else:
-                    stmt = blocks_dict[loc.block_addr, loc.block_idx].statements[loc.stmt_idx]
-                    if isinstance(stmt, Assignment) and isinstance(stmt.dst, VirtualVariable):
-                        used_by.add(stmt.dst.varid)
-                    else:
-                        used_by.add(None)
-            vvar_used_by[var_id] |= used_by
+                    if used_by_varid not in vvar_used_by:
+                        vvar_used_by[used_by_varid] |= self._get_vvar_used_by(used_by_varid, rd, blocks_dict)
+                    vvar_used_by[used_by_varid].add(var_id)  # probably unnecessary
+            vvar_used_by[var_id] |= self._get_vvar_used_by(var_id, rd, blocks_dict)
 
         g = networkx.DiGraph()
         dummy_vvar_id = -1
@@ -1542,8 +1581,12 @@ class AILSimplifier(Analysis):
 
             bail = False
             for varid in scc:
-                # if this vvar is a phi var, ensure this vvar is not used by anything else outside the scc
-                if varid in rd.phi_vvar_ids:
+                # ensure this vvar is not used by anything else outside the scc (regardless of whether this vvar is a
+                # phi variable or not)
+                if varid in vvar_used_by and None in vvar_used_by[varid]:
+                    bail = True
+                    break
+                if bail is False:
                     succs = list(g.successors(varid))
                     if any(succ_varid not in scc for succ_varid in succs):
                         bail = True
@@ -1565,7 +1608,7 @@ class AILSimplifier(Analysis):
         if rewriter_cls is None:
             return False
 
-        walker = None
+        walker = AILBlockWalker()
 
         class _any_update:
             """
@@ -1574,7 +1617,9 @@ class AILSimplifier(Analysis):
 
             v = False
 
-        def _handle_expr(expr_idx: int, expr: Expression, stmt_idx: int, stmt: Statement, block) -> Expression | None:
+        def _handle_expr(
+            expr_idx: int, expr: Expression, stmt_idx: int, stmt: Statement | None, block: Block | None
+        ) -> Expression | None:
             if isinstance(expr, VEXCCallExpression):
                 rewriter = rewriter_cls(expr, self.project.arch)
                 if rewriter.result is not None:
@@ -1585,8 +1630,6 @@ class AILSimplifier(Analysis):
             return AILBlockWalker._handle_expr(walker, expr_idx, expr, stmt_idx, stmt, block)
 
         blocks_by_addr_and_idx = {(node.addr, node.idx): node for node in self.func_graph.nodes()}
-
-        walker = AILBlockWalker()
         walker._handle_expr = _handle_expr
 
         updated = False

angr/analyses/decompiler/callsite_maker.py

@@ -45,7 +45,7 @@ class CallSiteMaker(Analysis):
         self._ail_manager = ail_manager
 
         self.result_block = None
-        self.stack_arg_offsets: set[tuple[int, int]] | None = None  # ins_addr, stack_offset
+        self.stack_arg_offsets: set[tuple[int, int]] | None = None  # call ins addr, stack_offset
         self.removed_vvar_ids: set[int] = set()
 
         self._analyze()
@@ -372,7 +372,9 @@ class CallSiteMaker(Analysis):
 
         return None
 
-    def _resolve_stack_argument(self, call_stmt, arg_loc) -> tuple[Any, Any]:  # pylint:disable=unused-argument
+    def _resolve_stack_argument(
+        self, call_stmt: Stmt.Call, arg_loc
+    ) -> tuple[Any, Any]:  # pylint:disable=unused-argument
         assert self._stack_pointer_tracker is not None
 
         size = arg_loc.size
@@ -399,15 +401,26 @@ class CallSiteMaker(Analysis):
                     # FIXME: vvar may be larger than that we ask; we may need to chop the correct value of vvar
                     value = view.get_vvar_value(vvar)
                     if value is not None and not isinstance(value, Expr.Phi):
-                        return None, value
-                    return None, Expr.VirtualVariable(
-                        self._atom_idx(),
-                        vvar.varid,
-                        vvar.bits,
-                        vvar.category,
-                        oident=vvar.oident,
-                        ins_addr=call_stmt.ins_addr,
-                    )
+                        v: Expr.Expression = value
+                    else:
+                        v: Expr.Expression = Expr.VirtualVariable(
+                            self._atom_idx(),
+                            vvar.varid,
+                            vvar.bits,
+                            vvar.category,
+                            oident=vvar.oident,
+                            ins_addr=call_stmt.ins_addr,
+                        )
+                    if v.size > size:
+                        v = Expr.Convert(
+                            self._atom_idx(),
+                            v.bits,
+                            size * self.project.arch.byte_width,
+                            False,
+                            v,
+                            ins_addr=call_stmt.ins_addr,
+                        )
+                    return None, v
 
             return None, Expr.Load(
                 self._atom_idx(),