angr 9.2.117__py3-none-win_amd64.whl → 9.2.119__py3-none-win_amd64.whl

angr/analyses/decompiler/{call_counter.py → counters/call_counter.py}

@@ -1,9 +1,10 @@
-from typing import Optional, TYPE_CHECKING
+from __future__ import annotations
+from typing import TYPE_CHECKING
 
 from ailment import Block
 from ailment.block_walker import AILBlockWalkerBase
 
-from .sequence_walker import SequenceWalker
+from angr.analyses.decompiler.sequence_walker import SequenceWalker
 
 if TYPE_CHECKING:
     from ailment.statement import Call
@@ -16,11 +17,11 @@ class AILBlockCallCounter(AILBlockWalkerBase):
 
     calls = 0
 
-    def _handle_CallExpr(self, expr_idx: int, expr: "Call", stmt_idx: int, stmt, block: Optional["Block"]):
+    def _handle_CallExpr(self, expr_idx: int, expr: Call, stmt_idx: int, stmt, block: Block | None):
         self.calls += 1
         super()._handle_CallExpr(expr_idx, expr, stmt_idx, stmt, block)
 
-    def _handle_Call(self, stmt_idx: int, stmt: "Call", block: Optional["Block"]):
+    def _handle_Call(self, stmt_idx: int, stmt: Call, block: Block | None):
         self.calls += 1
         super()._handle_Call(stmt_idx, stmt, block)
 

angr/analyses/decompiler/{expression_counters.py → counters/expression_counters.py}

@@ -1,4 +1,5 @@
-from typing import Any, DefaultDict, TYPE_CHECKING
+from __future__ import annotations
+from typing import Any, TYPE_CHECKING
 from collections.abc import Iterable
 from collections import defaultdict
 
@@ -37,7 +38,7 @@ class RegisterExpressionCounter(AILBlockWalkerBase):
 
     def __init__(self, expr_or_stmt: Expression | Statement):
         super().__init__()
-        self.counts: DefaultDict[tuple[int, int], int] = defaultdict(int)
+        self.counts: defaultdict[tuple[int, int], int] = defaultdict(int)
         if isinstance(expr_or_stmt, Expression):
             self.walk_expression(expr_or_stmt)
         elif isinstance(expr_or_stmt, Statement):
@@ -65,12 +66,12 @@ class OperatorCounter(AILBlockWalkerBase):
         else:
             raise TypeError(f"Unsupported argument type {type(expr_or_stmt)}")
 
-    def _handle_BinaryOp(self, expr_idx: int, expr: "BinaryOp", stmt_idx: int, stmt: Statement, block: Block | None):
+    def _handle_BinaryOp(self, expr_idx: int, expr: BinaryOp, stmt_idx: int, stmt: Statement, block: Block | None):
         if expr.op in self.operators:
             self.count += 1
         return super()._handle_BinaryOp(expr_idx, expr, stmt_idx, stmt, block)
 
-    def _handle_UnaryOp(self, expr_idx: int, expr: "UnaryOp", stmt_idx: int, stmt: Statement, block: Block | None):
+    def _handle_UnaryOp(self, expr_idx: int, expr: UnaryOp, stmt_idx: int, stmt: Statement, block: Block | None):
         if expr.op in self.operators:
             self.count += 1
         return super()._handle_UnaryOp(expr_idx, expr, stmt_idx, stmt, block)

angr/analyses/decompiler/counters/seq_cf_structure_counter.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+from collections import defaultdict
+
+import ailment
+
+from angr.analyses.decompiler.sequence_walker import SequenceWalker
+from angr.analyses.decompiler.structuring.structurer_nodes import LoopNode
+
+
+class ControlFlowStructureCounter(SequenceWalker):
+    """
+    Counts the number of different types of control flow structures found in a sequence of nodes.
+    This should be used after the sequence has been simplified.
+    """
+
+    def __init__(self, node):
+        handlers = {
+            LoopNode: self._handle_Loop,
+            ailment.Block: self._handle_Block,
+        }
+        super().__init__(handlers)
+
+        self.while_loops = 0
+        self.do_while_loops = 0
+        self.for_loops = 0
+        self.ordered_labels = []
+        self.goto_targets = defaultdict(int)
+
+        self.walk(node)
+
+        # eliminate gotos without labels
+        self.goto_targets = {k: v for k, v in self.goto_targets.items() if k in self.ordered_labels}
+        # correct labels that are not used
+        self.ordered_labels = [lbl for lbl in self.ordered_labels if lbl in self.goto_targets]
+
+    # pylint: disable=unused-argument
+    def _handle_Block(self, node: ailment.Block, **kwargs):
+        if not node.statements:
+            return
+
+        for stmt in node.statements:
+            # labels found in the block at this point will be labels in the output
+            if isinstance(stmt, ailment.statement.Label):
+                label_addr = stmt.ins_addr
+                if label_addr is not None:
+                    self.ordered_labels.append(label_addr)
+
+            # goto targets found in the block at this point will be goto targets in the output
+            if isinstance(stmt, ailment.statement.Jump):
+                target = stmt.target
+                target_value = target.value if target is not None and isinstance(target, ailment.Expr.Const) else None
+                if target_value is not None:
+                    self.goto_targets[target_value] += 1
+
+    def _handle_Loop(self, node: LoopNode, **kwargs):
+        if node.sort == "while":
+            self.while_loops += 1
+        elif node.sort == "do-while":
+            self.do_while_loops += 1
+        elif node.sort == "for":
+            self.for_loops += 1
+
+        return super()._handle_Loop(node, **kwargs)

angr/analyses/decompiler/decompilation_cache.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 from typing import Any, TYPE_CHECKING
 
 from .clinic import Clinic
@@ -31,7 +32,7 @@ class DecompilationCache:
         self.codegen: BaseStructuredCodeGenerator | None = None
         self.clinic: Clinic | None = None
         self.ite_exprs: set[tuple[int, Any]] | None = None
-        self.binop_operators: dict["OpDescriptor", str] | None = None
+        self.binop_operators: dict[OpDescriptor, str] | None = None
 
     @property
     def local_types(self):

angr/analyses/decompiler/decompilation_options.py

@@ -1,4 +1,5 @@
 # decompilation options
+from __future__ import annotations
 from collections.abc import Callable
 from collections import defaultdict
 

angr/analyses/decompiler/decompiler.py

@@ -1,4 +1,5 @@
 # pylint:disable=unused-import
+from __future__ import annotations
 import logging
 from collections import defaultdict
 from typing import Optional, Union, Any, TYPE_CHECKING
@@ -24,11 +25,11 @@ from .decompilation_options import DecompilationOption
 from .decompilation_cache import DecompilationCache
 from .utils import remove_labels
 from .sequence_walker import SequenceWalker
+from .structuring.structurer_nodes import SequenceNode
 
 if TYPE_CHECKING:
     from angr.knowledge_plugins.cfg.cfg_model import CFGModel
     from .peephole_optimizations import PeepholeOptimizationExprBase, PeepholeOptimizationStmtBase
-    from .structuring.structurer_nodes import SequenceNode
     from .structured_codegen.c import CStructuredCodeGenerator
 
 l = logging.getLogger(name=__name__)
@@ -49,7 +50,7 @@ class Decompiler(Analysis):
     def __init__(
         self,
         func: Function | str | int,
-        cfg: Union["CFGFast", "CFGModel"] | None = None,
+        cfg: CFGFast | CFGModel | None = None,
         options=None,
         optimization_passes=None,
         sp_tracker_track_memory=True,
@@ -92,16 +93,18 @@ class Decompiler(Analysis):
         self._inline_functions = inline_functions
 
         self.clinic = None  # mostly for debugging purposes
-        self.codegen: Optional["CStructuredCodeGenerator"] = None
+        self.codegen: CStructuredCodeGenerator | None = None
         self.cache: DecompilationCache | None = None
         self.options_by_class = None
-        self.seq_node: Optional["SequenceNode"] = None
+        self.seq_node: SequenceNode | None = None
         self.unoptimized_ail_graph: networkx.DiGraph | None = None
         self.ail_graph: networkx.DiGraph | None = None
+        self.vvar_id_start = None
 
         if decompile:
             self._decompile()
 
+    @timethis
     def _decompile(self):
         if self.func.is_simprocedure:
             return
@@ -130,10 +133,9 @@ class Decompiler(Analysis):
         self._update_progress(5.0, text="Converting to AIL")
 
         variable_kb = self._variable_kb
-        if variable_kb is None:
-            # fall back to old codegen
-            if old_codegen is not None:
-                variable_kb = old_codegen._variable_kb
+        # fall back to old codegen
+        if variable_kb is None and old_codegen is not None:
+            variable_kb = old_codegen._variable_kb
 
         if variable_kb is None:
             reset_variable_names = True
@@ -188,6 +190,7 @@ class Decompiler(Analysis):
         self.cache = cache
         self._variable_kb = clinic.variable_kb
         self._update_progress(70.0, text="Identifying regions")
+        self.vvar_id_start = clinic.vvar_id_start
 
         if clinic.graph is None:
             # the function is empty
@@ -220,6 +223,10 @@ class Decompiler(Analysis):
             ite_exprs=ite_exprs,
         )
 
+        # Rewrite the graph to remove phi expressions
+        # this is probably optional if we do not pretty-print clinic.graph
+        clinic.graph = self._transform_graph_from_ssa(clinic.graph)
+
         # save the graph before structuring happens (for AIL view)
         clinic.cc_graph = remove_labels(clinic.copy_graph())
 
@@ -251,6 +258,10 @@ class Decompiler(Analysis):
             seq_node = self._run_post_structuring_simplification_passes(
                 seq_node, binop_operators=cache.binop_operators, goto_manager=s.goto_manager, graph=clinic.graph
             )
+
+            # rewrite the sequence node to remove phi expressions
+            seq_node = self._transform_seqnode_from_ssa(seq_node)
+
             # update memory data
             if self._cfg is not None and self._update_memory_data:
                 self.find_data_references_and_update_memory_data(seq_node)
@@ -297,7 +308,7 @@ class Decompiler(Analysis):
         Runs optimizations that should be executed before region identification.
 
         :param ail_graph:   DiGraph with AIL Statements
-        :param reaching_defenitions: ReachingDefenitionAnalysis
+        :param reaching_definitions: ReachingDefinitionAnalysis
         :return:            The possibly new AIL DiGraph and RegionIdentifier
         """
         addr_and_idx_to_blocks: dict[tuple[int, int | None], ailment.Block] = {}
@@ -315,15 +326,15 @@ class Decompiler(Analysis):
             # only for post region id opts
             if pass_.STAGE != OptimizationPassStage.BEFORE_REGION_IDENTIFICATION:
                 continue
-            if pass_.STRUCTURING:
-                if self._recursive_structurer_params["structurer_cls"].NAME not in pass_.STRUCTURING:
-                    l.warning(
-                        "Skipping %s because it does not support structuring algorithm: %s",
-                        pass_,
-                        self._recursive_structurer_params["structurer_cls"].NAME,
-                    )
-                    continue
+            if pass_.STRUCTURING and self._recursive_structurer_params["structurer_cls"].NAME not in pass_.STRUCTURING:
+                l.warning(
+                    "Skipping %s because it does not support structuring algorithm: %s",
+                    pass_,
+                    self._recursive_structurer_params["structurer_cls"].NAME,
+                )
+                continue
 
+            pass_ = timethis(pass_)
             a = pass_(
                 self.func,
                 blocks_by_addr=addr_to_blocks,
@@ -353,7 +364,7 @@ class Decompiler(Analysis):
 
         :param ail_graph:   DiGraph with AIL Statements
         :param ri:          RegionIdentifier
-        :param reaching_defenitions: ReachingDefenitionAnalysis
+        :param reaching_definitions: ReachingDefinitionAnalysis
         :return:            The possibly new AIL DiGraph and RegionIdentifier
         """
         addr_and_idx_to_blocks: dict[tuple[int, int | None], ailment.Block] = {}
@@ -371,15 +382,15 @@ class Decompiler(Analysis):
             # only for post region id opts
             if pass_.STAGE != OptimizationPassStage.DURING_REGION_IDENTIFICATION:
                 continue
-            if pass_.STRUCTURING:
-                if self._recursive_structurer_params["structurer_cls"].NAME not in pass_.STRUCTURING:
-                    l.warning(
-                        "Skipping %s because it does not support structuring algorithm: %s",
-                        pass_,
-                        self._recursive_structurer_params["structurer_cls"].NAME,
-                    )
-                    continue
+            if pass_.STRUCTURING and self._recursive_structurer_params["structurer_cls"].NAME not in pass_.STRUCTURING:
+                l.warning(
+                    "Skipping %s because it does not support structuring algorithm: %s",
+                    pass_,
+                    self._recursive_structurer_params["structurer_cls"].NAME,
+                )
+                continue
 
+            pass_ = timethis(pass_)
             a = pass_(
                 self.func,
                 blocks_by_addr=addr_to_blocks,
@@ -388,6 +399,7 @@ class Decompiler(Analysis):
                 variable_kb=self._variable_kb,
                 region_identifier=ri,
                 reaching_definitions=reaching_definitions,
+                vvar_id_start=self.vvar_id_start,
                 **kwargs,
             )
 
@@ -405,6 +417,8 @@ class Decompiler(Analysis):
                 # always update RI on graph change
                 ri = self._recover_regions(ail_graph, cond_proc, update_graph=False)
 
+                self.vvar_id_start = a.vvar_id_start
+
         return ail_graph, self._recover_regions(ail_graph, ConditionProcessor(self.project.arch), update_graph=True)
 
     @timethis
@@ -413,6 +427,7 @@ class Decompiler(Analysis):
             if pass_.STAGE != OptimizationPassStage.AFTER_STRUCTURING:
                 continue
 
+            pass_ = timethis(pass_)
             a = pass_(self.func, seq=seq_node, **kwargs)
             if a.out_seq:
                 seq_node = a.out_seq
@@ -499,7 +514,7 @@ class Decompiler(Analysis):
 
         return codegen
 
-    def find_data_references_and_update_memory_data(self, seq_node: "SequenceNode"):
+    def find_data_references_and_update_memory_data(self, seq_node: SequenceNode):
         const_values: set[int] = set()
 
         def _handle_Const(expr_idx: int, expr: ailment.Expr.Const, *args, **kwargs):  # pylint:disable=unused-argument
@@ -534,6 +549,14 @@ class Decompiler(Analysis):
             memory_data_addrs=added_memory_data_addrs,
         )
 
+    def _transform_graph_from_ssa(self, ail_graph: networkx.DiGraph) -> networkx.DiGraph:
+        dephication = self.project.analyses.GraphDephication(self.func, ail_graph, rewrite=True)
+        return dephication.output
+
+    def _transform_seqnode_from_ssa(self, seq_node: SequenceNode) -> SequenceNode:
+        dephication = self.project.analyses.SeqNodeDephication(self.func, seq_node, rewrite=True)
+        return dephication.output
+
     @staticmethod
     def options_to_params(options: list[tuple[DecompilationOption, Any]]) -> dict[str, Any]:
         """

angr/analyses/decompiler/dephication/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+from .graph_vvar_mapping import GraphDephicationVVarMapping
+from .graph_dephication import GraphDephication
+from .seqnode_dephication import SeqNodeDephication
+
+__all__ = ["GraphDephicationVVarMapping", "GraphDephication", "SeqNodeDephication"]

angr/analyses/decompiler/dephication/dephication_base.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+from typing import Any
+import logging
+from collections import defaultdict
+
+from angr.analyses import Analysis
+
+l = logging.getLogger(name=__name__)
+
+
+class DephicationBase(Analysis):
+    """
+    This is the base class for analyses that removes phi expressions from AIL graphs, SequenceNodes, and other types of
+    AIL statement containers.
+    """
+
+    def __init__(self, func, vvar_to_vvar_mapping: dict[int, int] | None = None, rewrite: bool = False):
+        if isinstance(func, str):
+            self._function = self.kb.functions[func]
+        else:
+            self._function = func
+
+        self.vvar_to_vvar_mapping = vvar_to_vvar_mapping if vvar_to_vvar_mapping is not None else None
+        self.rewrite = rewrite
+        self.output = None
+
+    def _analyze(self):
+        if self.vvar_to_vvar_mapping is None:
+            self.vvar_to_vvar_mapping = self._collect_and_remap()
+
+        if self.rewrite:
+            self.output = self._rewrite_container()
+
+    def _collect_and_remap(self) -> dict[int, int]:
+        # collect phi assignments
+        phi_to_srcvarid = self._collect_phi_assignments()
+
+        vvar_to_vvar = {}  # a mapping between vvar IDs to phi vvar IDs when applicable
+        vvar_to_phivarids = defaultdict(set)
+        for phi_varid, varids in phi_to_srcvarid.items():
+            for varid in varids:
+                vvar_to_phivarids[varid].add(phi_varid)
+
+        # iterate until fixed point
+        while True:
+            changed = False
+            for varid in list(vvar_to_phivarids):
+                phivarids = vvar_to_phivarids[varid]
+                new = phivarids.copy()
+                for vv in phivarids:
+                    if vv in vvar_to_phivarids:
+                        new |= vvar_to_phivarids[vv]
+                if new != phivarids:
+                    changed = True
+                    vvar_to_phivarids[varid] = new
+
+            if not changed:
+                break
+
+        # unify those that are mapped to multiple phi variables
+        phivarid_to_phivarid = {}
+        for phivarids in vvar_to_phivarids.values():
+            if len(phivarids) > 1:
+                min_phivarid = min(phivarids)
+                # remap the existing ones first
+                for phivarid, mapped_to in list(phivarid_to_phivarid.items()):
+                    if phivarid in phivarids:
+                        continue
+                    if mapped_to in phivarids:
+                        phivarid_to_phivarid[phivarid] = min_phivarid
+                # map the new ones
+                for phivarid in phivarids:
+                    phivarid_to_phivarid[phivarid] = min_phivarid
+
+        # fill in vvar_to_vvar mapping
+        for phi_varid, varids in phi_to_srcvarid.items():
+            mapped_phivarid = phivarid_to_phivarid.get(phi_varid, phi_varid)
+            for varid in varids:
+                vvar_to_vvar[varid] = mapped_phivarid
+
+        return vvar_to_vvar
+
+    def _collect_phi_assignments(self) -> dict[int, set[int]]:
+        raise NotImplementedError
+
+    def _rewrite_container(self) -> Any:
+        raise NotImplementedError

angr/analyses/decompiler/dephication/graph_dephication.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+import logging
+from collections import defaultdict
+
+import networkx
+
+from ailment.expression import Phi, VirtualVariable
+from ailment.statement import Assignment
+
+from angr.knowledge_plugins.functions import Function
+from angr.analyses import register_analysis
+from .graph_rewriting import GraphRewritingAnalysis
+from .dephication_base import DephicationBase
+
+l = logging.getLogger(name=__name__)
+
+
+class GraphDephication(DephicationBase):  # pylint:disable=abstract-method
+    """
+    GraphDephication removes phi expressions from an AIL graph, essentially transforms a partial-SSA form of AIL graph
+    to a normal AIL graph.
+    """
+
+    def __init__(
+        self,
+        func: Function | str,
+        ail_graph,
+        vvar_to_vvar_mapping: dict[int, int] | None = None,
+        rewrite: bool = False,
+    ):
+        """
+        :param func:                            The subject of the analysis: a function, or a single basic block
+        :param ail_graph:                       The AIL graph to transform.
+        """
+
+        self._graph = ail_graph
+
+        super().__init__(func, vvar_to_vvar_mapping=vvar_to_vvar_mapping, rewrite=rewrite)
+
+        self._analyze()
+
+    def _collect_phi_assignments(self) -> dict[int, set[int]]:
+        g = self._graph
+        phi_to_src = defaultdict(set)
+
+        for block in g:
+            for stmt in block.statements:
+                if isinstance(stmt, Assignment) and isinstance(stmt.dst, VirtualVariable) and isinstance(stmt.src, Phi):
+                    for _, vvar in stmt.src.src_and_vvars:
+                        if vvar is None:
+                            l.debug("Invalid vvar None found in %r.src.src_and_vvars.", stmt)
+                        else:
+                            phi_to_src[stmt.dst.varid].add(vvar.varid)
+
+        return phi_to_src
+
+    def _rewrite_container(self) -> networkx.DiGraph:
+        # replace all vvars with phi variables in the graph
+        rewriter = GraphRewritingAnalysis(self.project, self._function, self._graph, self.vvar_to_vvar_mapping)
+        return rewriter.out_graph
+
+
+register_analysis(GraphDephication, "GraphDephication")

angr/analyses/decompiler/dephication/graph_rewriting.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+from typing import Any
+import logging
+
+import networkx
+
+import ailment
+
+from angr.utils.ail import is_phi_assignment
+from angr.analyses import ForwardAnalysis
+from angr.analyses.forward_analysis.visitors.graph import NodeType
+from angr.analyses.forward_analysis import FunctionGraphVisitor
+from .rewriting_engine import SimEngineDephiRewriting
+
+
+l = logging.getLogger(__name__)
+
+
+class GraphRewritingAnalysis(ForwardAnalysis[None, NodeType, object, object]):
+    """
+    This analysis traverses the AIL graph and rewrites virtual variables accordingly.
+    """
+
+    def __init__(
+        self,
+        project,
+        func,
+        ail_graph,
+        vvar_to_vvar: dict[int, int],
+    ):
+        self.project = project
+        self._function = func
+        self._graph_visitor = FunctionGraphVisitor(self._function, ail_graph)
+
+        ForwardAnalysis.__init__(
+            self, order_jobs=False, allow_merging=False, allow_widening=False, graph_visitor=self._graph_visitor
+        )
+        self._graph = ail_graph
+        self._vvar_to_vvar = vvar_to_vvar
+        self._engine_ail = SimEngineDephiRewriting(self.project.arch, self._vvar_to_vvar)
+
+        self._visited_blocks: set[Any] = set()
+        self.out_blocks = {}
+
+        self._analyze()
+
+        # remove phi statements
+        # modifying blocks inline should be fine here - no one has ever used these blocks
+        for block in self.out_blocks.values():
+            block.statements = [stmt for stmt in block.statements if not is_phi_assignment(stmt)]
+
+        self.out_graph = self._make_new_graph(ail_graph)
+
+    def _make_new_graph(self, old_graph: networkx.DiGraph) -> networkx.DiGraph:
+        new_graph = networkx.DiGraph()
+        for node in old_graph:
+            new_graph.add_node(self.out_blocks.get((node.addr, node.idx), node))
+
+        for src, dst in old_graph.edges:
+            new_graph.add_edge(
+                self.out_blocks.get((src.addr, src.idx), src), self.out_blocks.get((dst.addr, dst.idx), dst)
+            )
+
+        return new_graph
+
+    #
+    # Main analysis routines
+    #
+
+    def _pre_analysis(self):
+        pass
+
+    def _initial_abstract_state(self, node) -> None:
+        return None
+
+    def _run_on_node(self, node, state: None):
+        """
+
+        :param node:    The current node.
+        :param state:   The analysis state.
+        :return:        A tuple: (any changes occur, successor state)
+        """
+
+        if isinstance(node, ailment.Block):
+            block = node
+            block_key = (node.addr, node.idx)
+            engine = self._engine_ail
+        else:
+            l.warning("Unsupported node type %s.", node.__class__)
+            return False, None
+
+        if block_key in self._visited_blocks:
+            return False, None
+
+        engine: SimEngineDephiRewriting
+        engine.out_block = None
+        engine.process(None, block=block)
+
+        self._visited_blocks.add(block_key)
+
+        if engine.out_block is not None:
+            assert engine.out_block.addr == block.addr
+
+            if self.out_blocks.get(block_key, None) == engine.out_block:
+                return True, None
+            self.out_blocks[block_key] = engine.out_block
+            engine.out_block = None
+            return True, None
+
+        return True, None
+
+    def _intra_analysis(self):
+        pass
+
+    def _post_analysis(self):
+        pass