angr 9.2.87__py3-none-manylinux2014_x86_64.whl → 9.2.89__py3-none-manylinux2014_x86_64.whl

angr/__init__.py

@@ -1,7 +1,7 @@
 # pylint: disable=wildcard-import
 # pylint: disable=wrong-import-position
 
-__version__ = "9.2.87"
+__version__ = "9.2.89"
 
 if bytes is str:
     raise Exception(
@@ -88,6 +88,7 @@ from .state_plugins.heap import SimHeapBrk, SimHeapPTMalloc, PTChunk
 from . import concretization_strategies
 from .distributed import Server
 from .knowledge_base import KnowledgeBase
+from .procedures.definitions import load_external_definitions
 
 # for compatibility reasons
 from . import sim_manager as manager
@@ -95,6 +96,8 @@ from . import sim_manager as manager
 # now that we have everything loaded, re-grab the list of loggers
 loggers.load_all_loggers()
 
+load_external_definitions()
+
 __all__ = (
     "SimProcedure",
     "SIM_PROCEDURES",

angr/analyses/decompiler/clinic.py

@@ -638,6 +638,19 @@ class Clinic(Analysis):
             return block_node
 
         block = self.project.factory.block(block_node.addr, block_node.size, cross_insn_opt=False)
+        if block.vex.jumpkind not in {"Ijk_Call", "Ijk_Boring", "Ijk_Ret"} and not block.vex.jumpkind.startswith(
+            "Ijk_Sys"
+        ):
+            # we don't support lifting this block. use a dummy block instead
+            statements = [
+                ailment.Stmt.DirtyStatement(
+                    self._ail_manager.next_atom(),
+                    f"Unsupported jumpkind {block.vex.jumpkind} at address {block_node.addr}",
+                    ins_addr=block_node.addr,
+                )
+            ]
+            ail_block = ailment.Block(block_node.addr, block_node.size, statements=statements)
+            return ail_block
 
         ail_block = ailment.IRSBConverter.convert(block.vex, self._ail_manager)
         return ail_block
@@ -1140,6 +1153,7 @@ class Clinic(Analysis):
         try:
             tp = self.project.analyses.Typehoon(
                 vr.type_constraints,
+                vr.func_typevar,
                 kb=tmp_kb,
                 var_mapping=vr.var_to_typevars,
                 must_struct=must_struct,
@@ -1246,6 +1260,8 @@ class Clinic(Analysis):
                 self._link_variables_on_expr(variable_manager, global_variables, block, stmt_idx, stmt, ret_expr)
 
     def _link_variables_on_call(self, variable_manager, global_variables, block, stmt_idx, stmt, is_expr=False):
+        if not isinstance(stmt.target, ailment.Expr.Const):
+            self._link_variables_on_expr(variable_manager, global_variables, block, stmt_idx, stmt, stmt.target)
         if stmt.args:
             for arg in stmt.args:
                 self._link_variables_on_expr(variable_manager, global_variables, block, stmt_idx, stmt, arg)

angr/analyses/decompiler/decompiler.py

@@ -349,6 +349,9 @@ class Decompiler(Analysis):
             # only for post region id opts
             if pass_.STAGE != OptimizationPassStage.DURING_REGION_IDENTIFICATION:
                 continue
+            if pass_.STRUCTURING:
+                if self._recursive_structurer_params["structurer_cls"].NAME not in pass_.STRUCTURING:
+                    continue
 
             a = pass_(
                 self.func,

angr/analyses/decompiler/optimization_passes/__init__.py

@@ -21,6 +21,7 @@ from .x86_gcc_getpc_simplifier import X86GccGetPcSimplifier
 from .flip_boolean_cmp import FlipBooleanCmp
 from .ret_deduplicator import ReturnDeduplicator
 from .win_stack_canary_simplifier import WinStackCanarySimplifier
+from .cross_jump_reverter import CrossJumpReverter
 
 # order matters!
 _all_optimization_passes = [
@@ -40,9 +41,13 @@ _all_optimization_passes = [
     (ReturnDuplicator, True),
     (LoweredSwitchSimplifier, False),
     (ReturnDeduplicator, True),
+    (CrossJumpReverter, True),
     (FlipBooleanCmp, True),
 ]
 
+# these passes may duplicate code to remove gotos or improve the structure of the graph
+DUPLICATING_OPTS = [ReturnDuplicator, CrossJumpReverter]
+
 
 def get_optimization_passes(arch, platform):
     if isinstance(arch, Arch):

angr/analyses/decompiler/optimization_passes/cross_jump_reverter.py

@@ -0,0 +1,108 @@
+from collections import defaultdict
+from itertools import count
+import copy
+import logging
+import inspect
+
+from .optimization_pass import OptimizationPassStage, StructuringOptimizationPass
+from ..call_counter import AILBlockCallCounter
+
+l = logging.getLogger(__name__)
+
+
+class CrossJumpReverter(StructuringOptimizationPass):
+    """
+    This is an implementation to revert the compiler optimization Cross Jumping, and ISC optimization discussed
+    in the USENIX 2024 paper SAILR. This optimization is somewhat aggressive and as such should be run last in your
+    decompiler deoptimization chain. This deoptimization will take any goto it finds and attempt to duplicate its
+    target block if its target only has one outgoing edge.
+
+    There are some heuristics in place to prevent duplication everywhere. First, this deoptimization will only run
+    a max of max_opt_iters times. Second, it will not duplicate a block with too many calls.
+    """
+
+    ARCHES = None
+    PLATFORMS = None
+    STAGE = OptimizationPassStage.DURING_REGION_IDENTIFICATION
+    NAME = "Duplicate linear blocks with gotos"
+    STRUCTURING = ["phoenix"]
+    DESCRIPTION = inspect.cleandoc(__doc__).strip()
+
+    def __init__(
+        self,
+        func,
+        # internal parameters that should be used by Clinic
+        node_idx_start: int = 0,
+        # settings
+        max_opt_iters: int = 3,
+        max_call_duplications: int = 2,
+        **kwargs,
+    ):
+        super().__init__(func, max_opt_iters=max_opt_iters, strictly_less_gotos=True, **kwargs)
+
+        self.node_idx = count(start=node_idx_start)
+        self._max_call_dup = max_call_duplications
+        self.analyze()
+
+    def _check(self):
+        return True, None
+
+    def _analyze(self, cache=None):
+        to_update = defaultdict(list)
+        for node in self.out_graph.nodes:
+            gotos = self._goto_manager.gotos_in_block(node)
+            # TODO: support if-stmts
+            if not gotos or len(gotos) >= 2:
+                continue
+
+            # only blocks that have a single outgoing goto are candidates
+            # for duplicates
+            goto = list(gotos)[0]
+            for goto_target in self.out_graph.successors(node):
+                if goto_target.addr == goto.dst_addr:
+                    break
+            else:
+                goto_target = None
+
+            # the target goto block should only have a single outgoing edge
+            # this prevents duplication of conditions
+            if goto_target is None or self.out_graph.out_degree(goto_target) != 1:
+                continue
+
+            # minimize the number of calls in the target block that can be duplicated
+            # to prevent duplication of big blocks
+            counter = AILBlockCallCounter()
+            counter.walk(goto_target)
+            if counter.calls > self._max_call_dup:
+                continue
+
+            # [goto_target] = (pred1, pred2, ...)
+            to_update[goto_target].append(node)
+
+        if not to_update:
+            return False
+
+        updates = False
+        sorted_targets = sorted(to_update.items(), key=lambda x: x[0].addr)
+        for goto_target, pred_to_update in sorted_targets:
+            pred_to_update = sorted(pred_to_update, key=lambda x: x.addr)
+            # do some sanity checks
+            update_edges = [(pred, goto_target) for pred in pred_to_update]
+            if not all(self.out_graph.has_edge(*edge) for edge in update_edges):
+                continue
+
+            current_preds = list(self.out_graph.predecessors(goto_target))
+            delete_original = len(current_preds) == len(pred_to_update)
+
+            # update the edges
+            for src, goto_blk in update_edges:
+                cp = copy.deepcopy(goto_blk)
+                cp.idx = next(self.node_idx)
+                self.out_graph.remove_edge(src, goto_blk)
+                self.out_graph.add_edge(src, cp)
+
+            updates = True
+            if delete_original:
+                self.out_graph.remove_node(goto_target)
+
+        return updates

angr/analyses/decompiler/optimization_passes/optimization_pass.py

@@ -249,10 +249,18 @@ class StructuringOptimizationPass(OptimizationPass):
     STAGE = OptimizationPassStage.DURING_REGION_IDENTIFICATION
 
     def __init__(
-        self, func, prevent_new_gotos=True, recover_structure_fails=True, max_opt_iters=1, simplify_ail=True, **kwargs
+        self,
+        func,
+        prevent_new_gotos=True,
+        strictly_less_gotos=False,
+        recover_structure_fails=True,
+        max_opt_iters=1,
+        simplify_ail=True,
+        **kwargs,
     ):
         super().__init__(func, **kwargs)
         self._prevent_new_gotos = prevent_new_gotos
+        self._strictly_less_gotos = strictly_less_gotos
         self._recover_structure_fails = recover_structure_fails
         self._max_opt_iters = max_opt_iters
         self._simplify_ail = simplify_ail
@@ -298,9 +306,14 @@ class StructuringOptimizationPass(OptimizationPass):
             # this should not (TM) change the structure of the graph but is needed for later optimizations
             self.out_graph = self._simplify_ail_graph(self.out_graph)
 
-        if self._prevent_new_gotos and (len(self._goto_manager.gotos) > len(initial_gotos)):
-            self.out_graph = None
-            return
+        if self._prevent_new_gotos:
+            prev_gotos = len(initial_gotos)
+            new_gotos = len(self._goto_manager.gotos)
+            if (self._strictly_less_gotos and (new_gotos >= prev_gotos)) or (
+                not self._strictly_less_gotos and (new_gotos > prev_gotos)
+            ):
+                self.out_graph = None
+                return
 
     def _fixed_point_analyze(self, cache=None):
         for _ in range(self._max_opt_iters):

angr/analyses/decompiler/optimization_passes/return_duplicator.py

@@ -1,4 +1,4 @@
-from typing import Any, Tuple, Dict, List, TYPE_CHECKING, Optional
+from typing import Any, Tuple, Dict, List
 from itertools import count
 import copy
 import logging
@@ -8,38 +8,18 @@ import ailment.expression
 import networkx
 
 from ailment import Block
-from ailment.statement import Jump, ConditionalJump, Assignment, Statement, Return, Label
+from ailment.statement import Jump, ConditionalJump, Assignment, Return, Label
 from ailment.expression import Const
-from ailment.block_walker import AILBlockWalkerBase
 
 from .optimization_pass import StructuringOptimizationPass
 from ..condition_processor import ConditionProcessor, EmptyBlockNotice
 from ..graph_region import GraphRegion
-from ..utils import remove_labels, to_ail_supergraph
+from ..utils import remove_labels, to_ail_supergraph, calls_in_graph
 from ..structuring.structurer_nodes import MultiNode
 
-if TYPE_CHECKING:
-    from ailment.statement import Call
-
 _l = logging.getLogger(name=__name__)
 
 
-class AILCallCounter(AILBlockWalkerBase):
-    """
-    Helper class to count AIL Calls in a block.
-    """
-
-    calls = 0
-
-    def _handle_Call(self, stmt_idx: int, stmt: "Call", block: Optional["Block"]):
-        self.calls += 1
-        super()._handle_Call(stmt_idx, stmt, block)
-
-    def _handle_CallExpr(self, expr_idx: int, expr: "Call", stmt_idx: int, stmt: Statement, block: Optional[Block]):
-        self.calls += 1
-        super()._handle_CallExpr(expr_idx, expr, stmt_idx, stmt, block)
-
-
 class ReturnDuplicator(StructuringOptimizationPass):
     """
     An optimization pass that reverts a subset of Irreducible Statement Condensing (ISC) optimizations, as described
@@ -207,7 +187,7 @@ class ReturnDuplicator(StructuringOptimizationPass):
 
             # to assure we are not copying like crazy, set a max amount of code (which is estimated in calls)
             # that can be copied in a region
-            if self._number_of_calls_in(region) > self._max_calls_in_region:
+            if calls_in_graph(region) > self._max_calls_in_region:
                 continue
 
             end_node_regions[region_head] = in_edges, region
@@ -383,14 +363,6 @@ class ReturnDuplicator(StructuringOptimizationPass):
 
         return valid_assignment
 
-    @staticmethod
-    def _number_of_calls_in(graph: networkx.DiGraph) -> int:
-        counter = AILCallCounter()
-        for node in graph.nodes:
-            counter.walk(node)
-
-        return counter.calls
-
     @staticmethod
     def _single_entry_region(graph, end_node) -> Tuple[networkx.DiGraph, Any]:
         """

angr/analyses/decompiler/structured_codegen/c.py

@@ -472,7 +472,7 @@ class CFunction(CConstruct):  # pylint:disable=abstract-method
             yield indent_str, None
 
             # pick the first cvariable
-            # this is enough since highlighting works on the unified variable
+            # picking any cvariable is enough since highlighting works on the unified variable
             try:
                 cvariable = next(iter(cvar_and_vartypes))[0]
             except StopIteration:
@@ -486,9 +486,19 @@ class CFunction(CConstruct):  # pylint:disable=abstract-method
             else:
                 name = str(variable)
 
-            # sort by number of occurrences
+            # sort by number of occurrences, with a preference of non-basic types
+            # TODO: The type selection should actually happen during variable unification
             vartypes = [x[1] for x in cvar_and_vartypes]
+            nonprimitive_vartypes = [
+                vt for vt in vartypes if not isinstance(vt, (SimTypeChar, SimTypeInt, SimTypeFloat))
+            ]
             vartypes = list(dict.fromkeys(sorted(vartypes, key=vartypes.count, reverse=True)))
+            if nonprimitive_vartypes:
+                nonprimitive_vartypes = list(
+                    dict.fromkeys(sorted(nonprimitive_vartypes, key=nonprimitive_vartypes.count, reverse=True))
+                )
+                vartypes.remove(nonprimitive_vartypes[0])
+                vartypes.insert(0, nonprimitive_vartypes[0])
 
             for i, var_type in enumerate(vartypes):
                 if i == 0:

angr/analyses/decompiler/utils.py

@@ -7,7 +7,9 @@ import logging
 import networkx
 
 import ailment
+
 import angr
+from .call_counter import AILBlockCallCounter
 
 _l = logging.getLogger(__name__)
 
@@ -712,6 +714,17 @@ def decompile_functions(path, functions=None, structurer=None, catch_errors=Fals
     return decompilation
 
 
+def calls_in_graph(graph: networkx.DiGraph) -> int:
+    """
+    Counts the number of calls in an graph full of AIL Blocks
+    """
+    counter = AILBlockCallCounter()
+    for node in graph.nodes:
+        counter.walk(node)
+
+    return counter.calls
+
+
 # delayed import
 from .structuring.structurer_nodes import (
     MultiNode,

angr/analyses/typehoon/dfa.py

@@ -0,0 +1,108 @@
+from typing import Set, List, Tuple, Optional, TYPE_CHECKING
+
+import networkx
+
+# FIXME: Remove the dependency on pyformlang
+from pyformlang.finite_automaton import Epsilon, EpsilonNFA, State, Symbol
+
+from angr.errors import AngrError
+from .typevars import BaseLabel, Subtype
+from .variance import Variance
+
+if TYPE_CHECKING:
+    from pyformlang.finite_automaton import DeterministicFiniteAutomaton
+
+
+START_STATE = State("START")
+END_STATE = State("END")
+
+
+class EmptyEpsilonNFAError(AngrError):
+    """
+    A notification exception generated when the epsilon NFA is empty.
+    """
+
+
+class DFAConstraintSolver:
+    """
+    Implements a DFA-based graph solver.
+    """
+
+    @staticmethod
+    def graph_to_epsilon_nfa(graph: networkx.DiGraph, starts: Set, ends: Set) -> EpsilonNFA:
+        enfa = EpsilonNFA()
+
+        # print("Converting graph to eNFA")
+
+        for src, dst, data in graph.edges(data=True):
+            if not data:
+                symbol = Epsilon()
+            else:
+                assert "label" in data
+                label, kind = data["label"]
+                symbol = Symbol((label, kind))
+
+            # print(src, "-----", symbol, "----->", dst)
+            enfa.add_transition(State(src), symbol, State(dst))
+
+        enfa.add_start_state(START_STATE)
+
+        for start in starts:
+            # print("Start transition", START_STATE, "----->", start)
+            enfa.add_transition(START_STATE, Symbol(start), State(start))
+
+        enfa.add_final_state(END_STATE)
+        for end in ends:
+            # print("End transition", end, "----->", END_STATE)
+            enfa.add_transition(State(end), Symbol(end), END_STATE)
+
+        # networkx.drawing.nx_pydot.write_dot(graph, "d:/enfa.dot")
+
+        if enfa.is_empty():
+            raise EmptyEpsilonNFAError()
+        return enfa
+
+    def generate_constraints_between(self, graph: networkx.DiGraph, starts: Set, ends: Set) -> Set:
+        epsilon_nfa = self.graph_to_epsilon_nfa(graph, starts, ends)
+        min_dfa: "DeterministicFiniteAutomaton" = epsilon_nfa.minimize()
+        dfa_graph: networkx.MultiDiGraph = min_dfa.to_networkx()
+
+        constraints = set()
+
+        for final_state in min_dfa.final_states:
+            for path in networkx.all_simple_edge_paths(dfa_graph, min_dfa.start_state, final_state):
+                path_labels = []
+                for src, dst, index in path:
+                    d = dfa_graph.get_edge_data(src, dst)[index]
+                    path_labels.append(d["label"])
+
+                start_node = path_labels[0]
+                end_node = path_labels[-1]
+
+                constraint = self._check_constraint(start_node, end_node, path_labels[1:-1])
+                if constraint is not None:
+                    constraints.add(constraint)
+
+        return constraints
+
+    @staticmethod
+    def _check_constraint(src, dst, string: List[Tuple[BaseLabel, str]]) -> Optional[Subtype]:
+        forgets = []
+        recalls = []
+        for label, kind in string:
+            if kind == "forget":
+                forgets.append(label)
+            elif kind == "recall":
+                recalls.append(label)
+
+        lhs = src
+        rhs = dst
+        for recall in recalls:
+            lhs = lhs.recall(recall)
+        for forget in reversed(forgets):
+            rhs = rhs.recall(forget)
+
+        if lhs.variance == Variance.COVARIANT and rhs.variance == Variance.COVARIANT:
+            if lhs.typevar != rhs.typevar:
+                return Subtype(lhs.typevar, rhs.typevar)
+        return None

angr/analyses/typehoon/lifter.py

@@ -1,5 +1,20 @@
-from ...sim_type import SimType, SimTypeChar, SimTypeShort, SimTypeInt, SimTypeLong, SimTypeLongLong, SimTypePointer
-from .typeconsts import BottomType, Int8, Int16, Int32, Int64, Pointer32, Pointer64
+from typing import Union, TYPE_CHECKING
+
+from ...sim_type import (
+    SimType,
+    SimTypeChar,
+    SimTypeShort,
+    SimTypeInt,
+    SimTypeLong,
+    SimTypeLongLong,
+    SimTypePointer,
+    SimStruct,
+    SimTypeArray,
+)
+from .typeconsts import BottomType, Int8, Int16, Int32, Int64, Pointer32, Pointer64, Struct, Array
+
+if TYPE_CHECKING:
+    from .typeconsts import TypeConstant
 
 
 class TypeLifter:
@@ -40,6 +55,21 @@ class TypeLifter:
             return Pointer64(self.lift(ty.pts_to))
         raise ValueError("Unsupported bits %s." % self.bits)
 
+    def _lift_SimStruct(self, ty: SimStruct) -> Union["TypeConstant", BottomType]:
+        converted_fields = {}
+        field_names = {}
+        ty_offsets = ty.offsets
+        for field_name, simtype in ty.fields.items():
+            if field_name not in ty_offsets:
+                return BottomType()
+            converted_fields[ty_offsets[field_name]] = self.lift(simtype)
+            field_names[ty_offsets[field_name]] = field_name
+        return Struct(fields=converted_fields, name=ty.name, field_names=field_names)
+
+    def _lift_SimTypeArray(self, ty: SimTypeArray) -> Array:
+        elem_type = self.lift(ty.elem_type)
+        return Array(elem_type, count=ty.length)
+
 
 _mapping = {
     SimTypeChar: TypeLifter._lift_SimTypeChar,
@@ -48,4 +78,6 @@ _mapping = {
     SimTypeLong: TypeLifter._lift_SimTypeInt,
     SimTypeLongLong: TypeLifter._lift_SimTypeLongLong,
     SimTypePointer: TypeLifter._lift_SimTypePointer,
+    SimStruct: TypeLifter._lift_SimStruct,
+    SimTypeArray: TypeLifter._lift_SimTypeArray,
 }