angr 9.2.117__py3-none-manylinux2014_x86_64.whl → 9.2.118__py3-none-manylinux2014_x86_64.whl

angr/analyses/decompiler/optimization_passes/duplication_reverter/errors.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+
+class SAILRSemanticError(Exception):
+    """
+    These types of errors may not kill the entire analysis, but they do kill the current working round.
+    They are caused when some non-expected state happens, like a ConditionalJump that is not the last statement in a
+    block.
+    """
+
+
+class UnsupportedAILNodeError(SAILRSemanticError):
+    """
+    These type of errors are raised when an AIL node is not supported by the current implementation.
+    Most common with Switch heads.
+    """

angr/analyses/decompiler/optimization_passes/duplication_reverter/similarity.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+import itertools
+import logging
+
+import networkx as nx
+
+from ailment.block import Block
+
+from .utils import bfs_list_blocks
+from ...block_similarity import longest_ail_subseq, is_similar
+
+_l = logging.getLogger(name=__name__)
+
+
+def longest_ail_graph_subseq(block_list, graph):
+    # generate a graph similarity for each pair in the provided blocks
+    all_sims = [ail_graph_similarity(pair[0], pair[1], graph) for pair in itertools.combinations(block_list, 2)]
+
+    lcs, _ = longest_ail_subseq(all_sims, graph=graph)
+    return lcs
+
+
+def ail_graph_similarity(block0: Block, block1: Block, graph: nx.DiGraph, only_blocks=False):
+    b0_blocks = bfs_list_blocks(block0, graph)
+    b1_blocks = bfs_list_blocks(block1, graph)
+    similarity = []
+
+    discontinuity_blocks = set()
+    for i, b0 in enumerate(b0_blocks):
+        # getting to a block with no matching index is grounds to stop cmp
+        try:
+            b1 = b1_blocks[i]
+        except IndexError:
+            break
+
+        # if a block in the chain before did not end in LCS, don't follow it.
+        if b0 in discontinuity_blocks or b1 in discontinuity_blocks:
+            continue
+
+        # SPECIAL CASE: 1
+        # ┌─┐         ┌─┐
+        # │A├─┐     ┌─┤A│
+        # └─┘ │     │ └─┘
+        #     ▼     ▼
+        #    ┌──┐ ┌──┐
+        #    │C'│ │C'│
+        #    └┬─┘ └┬─┘
+        #     │    │
+        #     ▼    ▼
+        #       ...
+        #
+        # Both similar blocks end in a block that is actually the same block. In this case, we don't
+        # want to count this since we will create a N blocks long similarity that continues from C' all
+        # the way to the end of the graph. This is only true if C' is not the end block of the graph.
+        if b0 is b1:
+            preds = list(graph.predecessors(b0))
+            all_match = True
+            for pred1 in preds:
+                for pred2 in preds:
+                    if set(graph.successors(pred1)) != set(graph.successors(pred2)):
+                        all_match = False
+                        break
+
+                if not all_match:
+                    break
+
+            # CASE 1 confirmed, all edges in look the same. Now we check a special subset to see if the
+            # matching node has any edges out. If this node has no edges out, we are no longer ture for 1A
+            if all_match and len(list(graph.successors(b0))) != 0:
+                continue
+
+        lcs, lcs_idxs = longest_ail_subseq([b0.statements, b1.statements], graph=graph)
+        if not lcs:
+            break
+
+        # verify all the blocks end in that statement or exclude its children
+        for idx, b in enumerate([b0, b1]):
+            if len(lcs) + lcs_idxs[idx] != len(b.statements):
+                discontinuity_blocks.update(set(graph.successors(b)))
+
+        # can output blocks only if needed
+        similarity += lcs if not only_blocks else [(b0, b1)]
+
+    return similarity
+
+
+def ail_similarity_to_orig_blocks(orig_block, graph_similarity, graph):
+    traversal_blocks = bfs_list_blocks(orig_block, graph)
+
+    graph_sim = graph_similarity.copy()
+    orig_blocks = []
+    split_blocks = {}  # [block] = (lcs, idx)
+    for block in traversal_blocks:
+        if not graph_sim:
+            break
+
+        lcs, lcs_idxs = longest_ail_subseq([block.statements, graph_sim[: len(block.statements)]], graph=graph)
+        if block is orig_block:
+            lcs_1, lcs_idxs_1 = longest_ail_subseq([graph_sim[: len(block.statements)], block.statements], graph=graph)
+            if lcs_idxs_1[1] > lcs_idxs[0]:
+                lcs, lcs_idxs = lcs_1, lcs_idxs_1[::-1]
+
+        if not lcs:
+            break
+
+        orig_blocks.append(block)
+
+        if len(lcs) != len(block.statements):
+            split_blocks[block] = (lcs, lcs_idxs[0])
+
+        graph_sim = graph_sim[len(lcs) :]
+
+    return orig_blocks, split_blocks
+
+
+def find_block_by_similarity(block, graph, node_list=None):
+    nodes = node_list if node_list else list(graph.nodes())
+    similar_blocks = []
+    for other_block in nodes:
+        if is_similar(block, other_block, graph=graph):
+            similar_blocks.append(other_block)
+
+    if len(similar_blocks) > 1:
+        _l.warning("found multiple similar blocks")
+
+    return similar_blocks[0]

angr/analyses/decompiler/optimization_passes/duplication_reverter/utils.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+import logging
+
+import networkx as nx
+
+import ailment
+from ailment import Const
+from ailment.block import Block
+from ailment.statement import Statement, ConditionalJump, Jump
+
+from .errors import UnsupportedAILNodeError
+from ...structuring.structurer_nodes import IncompleteSwitchCaseHeadStatement
+
+
+_l = logging.getLogger(name=__name__)
+
+
+#
+# Graph Utilities
+#
+
+
+def find_block_in_successors_by_addr(addr: int, block: ailment.Block, graph: nx.DiGraph) -> Block | None:
+    for succ in graph.successors(block):
+        if addr in (succ.addr, succ.statements[0].ins_addr):
+            return succ
+
+    return None
+
+
+def replace_node_in_graph(graph: nx.DiGraph, node, replace_with):
+    in_edges = list(graph.in_edges(node))
+    out_edges = list(graph.out_edges(node))
+
+    graph.remove_node(node)
+    graph.add_node(replace_with)
+
+    for src, _ in in_edges:
+        if src is node:
+            graph.add_edge(replace_with, replace_with)
+        else:
+            graph.add_edge(src, replace_with)
+
+    for _, dst in out_edges:
+        if dst is node:
+            graph.add_edge(replace_with, replace_with)
+        else:
+            graph.add_edge(replace_with, dst)
+
+    assert node not in graph
+
+
+def bfs_list_blocks(start_block: Block, graph: nx.DiGraph):
+    blocks = []
+    bfs = list(nx.bfs_successors(graph, start_block, depth_limit=10))
+    for blk_tree in bfs:
+        source, children = blk_tree
+        last_src_stmt = source.statements[-1] if source.statements else None
+        if (
+            last_src_stmt is None
+            or not isinstance(last_src_stmt, Statement)
+            or isinstance(last_src_stmt, IncompleteSwitchCaseHeadStatement)
+        ):
+            raise UnsupportedAILNodeError(f"Stmt {last_src_stmt} is unsupported")
+
+        if len(children) == 1:
+            blocks += children
+        elif len(children) == 2:
+            if_stmt: ConditionalJump = source.statements[-1]
+            if children[0].addr == if_stmt.true_target.value:
+                blocks += [children[0], children[1]]
+            else:
+                blocks += [children[1], children[0]]
+
+    return [start_block, *blocks]
+
+
+def copy_graph_and_nodes(graph: nx.DiGraph, new_idx=False):
+    new_graph = nx.DiGraph()
+    nodes_map = {}
+    for node in graph.nodes:
+        node_copy = node.copy()
+        node_copy.statements = list(node_copy.statements)
+        if new_idx:
+            node_copy.idx = node_copy.idx + 1 if isinstance(node_copy.idx, int) else 1
+        nodes_map[node] = node_copy
+
+    new_graph.add_nodes_from(nodes_map.values())
+    for src, dst in graph.edges:
+        new_graph.add_edge(nodes_map[src], nodes_map[dst])
+
+    return new_graph
+
+
+#
+# AIL Modification Utilities
+#
+
+
+def ail_block_from_stmts(stmts, idx=None, block_addr=None) -> Block | None:
+    if not stmts:
+        return None
+
+    first_stmt = stmts[0]
+
+    return Block(
+        block_addr if block_addr else first_stmt.ins_addr,
+        0,
+        statements=list(stmts),
+        idx=idx or 1,
+    )
+
+
+def deepcopy_ail_jump(stmt: Jump, idx=1):
+    target: Const = stmt.target
+    tags = stmt.tags.copy()
+
+    return Jump(idx, Const(1, target.variable, target.value, target.bits, **target.tags.copy()), **tags)
+
+
+def deepcopy_ail_condjump(stmt: ConditionalJump, idx=1):
+    true_target: Const = stmt.true_target
+    false_target: Const = stmt.false_target
+    tags = stmt.tags.copy()
+
+    return ConditionalJump(
+        idx,
+        stmt.condition.copy(),
+        Const(1, true_target.variable, true_target.value, true_target.bits, **true_target.tags.copy()),
+        Const(1, false_target.variable, false_target.value, false_target.bits, **false_target.tags.copy()),
+        **tags,
+    )
+
+
+def deepcopy_ail_anyjump(stmt: Jump | ConditionalJump, idx=1):
+    if isinstance(stmt, Jump):
+        return deepcopy_ail_jump(stmt, idx=idx)
+    if isinstance(stmt, ConditionalJump):
+        return deepcopy_ail_condjump(stmt, idx=idx)
+    raise ValueError(
+        "Attempting to deepcopy non-jump stmt, likely happen to a "
+        "block ending in no jump. Place a jump there to fix it."
+    )
+
+
+def correct_jump_targets(stmt, replacement_map: dict[int, int], new_stmt=True):
+    if not replacement_map or not isinstance(stmt, Statement):
+        return stmt
+
+    if isinstance(stmt, ConditionalJump):
+        cond_stmt = deepcopy_ail_condjump(stmt) if new_stmt else stmt
+        true_target, false_target = cond_stmt.true_target, cond_stmt.false_target
+
+        if isinstance(true_target, Const) and true_target.value in replacement_map:
+            true_target.value = replacement_map[true_target.value]
+
+        if isinstance(false_target, Const) and false_target.value in replacement_map:
+            false_target.value = replacement_map[false_target.value]
+
+        return cond_stmt
+    if isinstance(stmt, Jump) and isinstance(stmt.target, Const):
+        jump_stmt = deepcopy_ail_jump(stmt) if new_stmt else stmt
+        target = jump_stmt.target
+
+        if isinstance(target, Const) and target.value in replacement_map:
+            target.value = replacement_map[target.value]
+
+        return jump_stmt
+    return stmt

angr/analyses/decompiler/optimization_passes/engine_base.py

@@ -1,3 +1,5 @@
+# pylint:disable=too-many-boolean-expressions
+from __future__ import annotations
 import logging
 
 from ailment import Expr, Stmt
@@ -21,36 +23,24 @@ class SimplifierAILState:
         return "<SimplifierAILState>"
 
     def copy(self):
-        rd = SimplifierAILState(
+        return SimplifierAILState(
             self.arch,
             variables=self._variables.copy(),
         )
 
-        return rd
-
     def merge(self, *others):
-        raise NotImplementedError()
+        raise NotImplementedError
 
-    def store_variable(self, old, new):
+    def store_variable(self, old: Expr.VirtualVariable, new):
         if new is not None:
-            self._variables[old] = new
+            self._variables[old.varid] = new
 
-    def get_variable(self, old):
-        return self._variables.get(old, None)
+    def get_variable(self, old: Expr.VirtualVariable):
+        return self._variables.get(old.varid, None)
 
     def remove_variable(self, old):
         self._variables.pop(old, None)
 
-    def filter_variables(self, atom):
-        keys_to_remove = set()
-
-        for k, v in self._variables.items():
-            if isinstance(v, Expr.Expression) and (v == atom or v.has_atom(atom, identity=False)):
-                keys_to_remove.add(k)
-
-        for k in keys_to_remove:
-            self._variables.pop(k)
-
 
 class SimplifierAILEngine(
     SimEngineLightAILMixin,
@@ -85,19 +75,17 @@ class SimplifierAILEngine(
 
     # handle stmt
     def _ail_handle_Stmt(self, stmt):
-        handler = "_ail_handle_%s" % type(stmt).__name__
+        handler = f"_ail_handle_{type(stmt).__name__}"
         if hasattr(self, handler):
             return getattr(self, handler)(stmt)
-        else:
-            _l.debug("Unsupported statement type %s.", type(stmt).__name__)
-            return stmt
+        _l.debug("Unsupported statement type %s.", type(stmt).__name__)
+        return stmt
 
     def _ail_handle_Assignment(self, stmt):
         src = self._expr(stmt.src)
         dst = self._expr(stmt.dst)
 
-        if isinstance(dst, Expr.Register) and not src.has_atom(dst, identity=False):
-            self.state.filter_variables(dst)
+        if isinstance(dst, Expr.VirtualVariable) and not isinstance(src, Expr.Phi):
             self.state.store_variable(dst, src)
 
         if (src, dst) != (stmt.src, stmt.dst):
@@ -141,6 +129,8 @@ class SimplifierAILEngine(
             prototype=stmt.prototype,
             args=new_args,
             ret_expr=stmt.ret_expr,
+            fp_ret_expr=stmt.fp_ret_expr,
+            bits=stmt.bits,
             **stmt.tags,
         )
 
@@ -157,20 +147,10 @@ class SimplifierAILEngine(
                 return new_stmt
         return stmt
 
-    def _ail_handle_Load(self, expr):
-        # We don't want to load new values and construct new AIL expressions in caller methods without def-use
-        # information. Otherwise, we may end up creating incorrect expressions.
-        # Therefore, we do not perform memory load, which essentially turns SimplifierAILEngine into a peephole
-        # optimization engine.
-        addr = self._expr(expr.addr)
-        if addr != expr.addr:
-            return Expr.Load(expr.idx, addr, expr.size, expr.endness, **expr.tags)
-        return expr
-
     # handle expr
 
     def _expr(self, expr):
-        handler = "_ail_handle_%s" % type(expr).__name__
+        handler = f"_ail_handle_{type(expr).__name__}"
         if hasattr(self, handler):
             v = getattr(self, handler)(expr)
             if v is None:
@@ -182,6 +162,26 @@ class SimplifierAILEngine(
     def _ail_handle_StackBaseOffset(self, expr):  # pylint:disable=no-self-use
         return expr
 
+    def _ail_handle_VirtualVariable(self, expr: Expr.VirtualVariable):  # pylint:disable=no-self-use
+        # We don't want to return new values and construct new AIL expressions in caller methods without def-use
+        # information. Otherwise, we may end up creating incorrect expressions.
+        # Therefore, we do not perform vvar load, which essentially turns SimplifierAILEngine into a peephole
+        # optimization engine.
+        return expr
+
+    def _ail_handle_Phi(self, expr: Expr.Phi):  # pylint:disable=no-self-use
+        return expr
+
+    def _ail_handle_Load(self, expr):
+        # We don't want to load new values and construct new AIL expressions in caller methods without def-use
+        # information. Otherwise, we may end up creating incorrect expressions.
+        # Therefore, we do not perform memory load, which essentially turns SimplifierAILEngine into a peephole
+        # optimization engine.
+        addr = self._expr(expr.addr)
+        if addr != expr.addr:
+            return Expr.Load(expr.idx, addr, expr.size, expr.endness, **expr.tags)
+        return expr
+
     def _ail_handle_Register(self, expr):  # pylint:disable=no-self-use
         # We don't want to return new values and construct new AIL expressions in caller methods without def-use
         # information. Otherwise, we may end up creating incorrect expressions.
@@ -207,19 +207,18 @@ class SimplifierAILEngine(
             if expr.from_bits == operand_expr.to_bits and expr.to_bits == operand_expr.from_bits:
                 # eliminate the redundant Convert
                 return operand_expr.operand
-            else:
-                return Expr.Convert(
-                    expr.idx,
-                    operand_expr.from_bits,
-                    expr.to_bits,
-                    expr.is_signed,
-                    operand_expr.operand,
-                    from_type=operand_expr.from_type,
-                    to_type=expr.to_type,
-                    rounding_mode=expr.rounding_mode,
-                    **expr.tags,
-                )
-        elif (
+            return Expr.Convert(
+                expr.idx,
+                operand_expr.from_bits,
+                expr.to_bits,
+                expr.is_signed,
+                operand_expr.operand,
+                from_type=operand_expr.from_type,
+                to_type=expr.to_type,
+                rounding_mode=expr.rounding_mode,
+                **expr.tags,
+            )
+        if (
             type(operand_expr) is Expr.Const
             and expr.from_type == Expr.Convert.TYPE_INT
             and expr.to_type == Expr.Convert.TYPE_INT
@@ -229,7 +228,7 @@ class SimplifierAILEngine(
             mask = (2**expr.to_bits) - 1
             value &= mask
             return Expr.Const(expr.idx, operand_expr.variable, value, expr.to_bits, **expr.tags)
-        elif type(operand_expr) is Expr.BinaryOp and operand_expr.op in {
+        if type(operand_expr) is Expr.BinaryOp and operand_expr.op in {
             "Mul",
             "Shl",
             "Div",
@@ -275,21 +274,20 @@ class SimplifierAILEngine(
                 isinstance(operand_expr.operands[0], Expr.Convert)
                 and isinstance(operand_expr.operands[1], Expr.Convert)
                 and operand_expr.operands[0].from_bits == operand_expr.operands[1].from_bits
+            ) and (
+                operand_expr.operands[0].to_bits == operand_expr.operands[1].to_bits
+                and expr.from_bits == operand_expr.operands[0].to_bits
+                and expr.to_bits == operand_expr.operands[1].from_bits
             ):
-                if (
-                    operand_expr.operands[0].to_bits == operand_expr.operands[1].to_bits
-                    and expr.from_bits == operand_expr.operands[0].to_bits
-                    and expr.to_bits == operand_expr.operands[1].from_bits
-                ):
-                    return Expr.BinaryOp(
-                        operand_expr.idx,
-                        operand_expr.op,
-                        [operand_expr.operands[0].operand, operand_expr.operands[1].operand],
-                        expr.is_signed,
-                        **operand_expr.tags,
-                    )
+                return Expr.BinaryOp(
+                    operand_expr.idx,
+                    operand_expr.op,
+                    [operand_expr.operands[0].operand, operand_expr.operands[1].operand],
+                    expr.is_signed,
+                    **operand_expr.tags,
+                )
 
-        converted = Expr.Convert(
+        return Expr.Convert(
             expr.idx,
             expr.from_bits,
             expr.to_bits,
@@ -300,4 +298,3 @@ class SimplifierAILEngine(
             rounding_mode=expr.rounding_mode,
             **expr.tags,
         )
-        return converted

angr/analyses/decompiler/optimization_passes/expr_op_swapper.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import logging
 from typing import Any, TYPE_CHECKING
 from collections.abc import Callable
@@ -26,7 +27,7 @@ class OuterWalker(SequenceWalker):
         super().__init__()
         self.desc: dict[OpDescriptor, str] = desc
 
-    def _handle_Condition(self, node: "ConditionNode", **kwargs):
+    def _handle_Condition(self, node: ConditionNode, **kwargs):
         for desc, new_op in self.desc.items():
             if (
                 hasattr(node.condition, "ins_addr")
@@ -36,7 +37,7 @@ class OuterWalker(SequenceWalker):
                 node.condition = self._swap_expr_op(new_op, node.condition)
         return super()._handle_Condition(node, **kwargs)
 
-    def _handle_Loop(self, node: "LoopNode", **kwargs):
+    def _handle_Loop(self, node: LoopNode, **kwargs):
         for desc, new_op in self.desc.items():
             if (
                 hasattr(node.condition, "ins_addr")
@@ -46,7 +47,7 @@ class OuterWalker(SequenceWalker):
                 node.condition = self._swap_expr_op(new_op, node.condition)
         return super()._handle_Loop(node, **kwargs)
 
-    def _handle_ConditionalBreak(self, node: "ConditionalBreakNode", **kwargs):
+    def _handle_ConditionalBreak(self, node: ConditionalBreakNode, **kwargs):
         for desc, new_op in self.desc.items():
             if (
                 hasattr(node.condition, "ins_addr")
@@ -59,10 +60,9 @@ class OuterWalker(SequenceWalker):
     @staticmethod
     def _swap_expr_op(new_op: str, atom: Expression) -> Expression | None:
         # swap
-        new_expr = BinaryOp(
+        return BinaryOp(
             atom.idx, new_op, (atom.operands[1], atom.operands[0]), atom.signed, bits=atom.bits, **atom.tags
         )
-        return new_expr
 
 
 class ExpressionReplacer(AILBlockWalker):
@@ -80,8 +80,7 @@ class ExpressionReplacer(AILBlockWalker):
         self, expr_idx: int, expr: Expression, stmt_idx: int, stmt: Statement | None, block: AILBlock | None
     ) -> Any:
         if self._target_expr_predicate(expr):
-            new_expr = self._callback(self._block_addr, expr)
-            return new_expr
+            return self._callback(self._block_addr, expr)
         return super()._handle_expr(expr_idx, expr, stmt_idx, stmt, block)
 
 

angr/analyses/decompiler/optimization_passes/flip_boolean_cmp.py

@@ -1,4 +1,5 @@
 # pylint:disable=arguments-renamed,too-many-boolean-expressions
+from __future__ import annotations
 from typing import Any
 
 import ailment