angr 9.2.158__cp310-abi3-manylinux2014_x86_64.whl → 9.2.160__cp310-abi3-manylinux2014_x86_64.whl

angr/ailment/tagged_object.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+
+class TaggedObject:
+    """
+    A class that takes arbitrary tags.
+    """
+
+    __slots__ = (
+        "_hash",
+        "_tags",
+        "idx",
+    )
+
+    def __init__(self, idx: int | None, **kwargs):
+        self._tags = None
+        self.idx = idx
+        self._hash = None
+        if kwargs:
+            self.initialize_tags(kwargs)
+
+    def initialize_tags(self, tags):
+        self._tags = {}
+        for k, v in tags.items():
+            self._tags[k] = v
+
+    def __getattr__(self, item):
+        try:
+            return self.tags[item]
+        except KeyError:
+            return super().__getattribute__(item)
+
+    def __new__(cls, *args, **kwargs):  # pylint:disable=unused-argument
+        """Create a new instance and set `_tags` attribute.
+
+        Since TaggedObject override `__getattr__` method and try to access the
+        `_tags` attribute, infinite recursion could occur if `_tags` not ready
+        to exists.
+
+        This behavior causes an infinite recursion error when copying
+        `TaggedObject` with `copy.deepcopy`.
+
+        Hence, we set `_tags` attribute here to prevent this problem.
+        """
+        self = super().__new__(cls)
+        self._tags = None
+        return self
+
+    def __hash__(self) -> int:
+        if self._hash is None:
+            self._hash = self._hash_core()
+        return self._hash
+
+    def _hash_core(self):
+        raise NotImplementedError
+
+    @property
+    def tags(self) -> dict:
+        if not self._tags:
+            self._tags = {}
+        return self._tags

angr/ailment/utils.py

@@ -0,0 +1,114 @@
+# pylint:disable=ungrouped-imports,wrong-import-position
+from __future__ import annotations
+from typing import TypeAlias
+import struct
+
+try:
+    from claripy.ast import Bits
+except ImportError:
+    from typing_extensions import Never as Bits
+
+try:
+    import _md5 as md5lib
+except ImportError:
+    import hashlib as md5lib
+
+GetBitsTypeParams: TypeAlias = "Expression"
+
+
+def get_bits(expr: GetBitsTypeParams) -> int:
+
+    if isinstance(expr, Expression):
+        return expr.bits
+    if isinstance(expr, Bits):
+        return expr.size()
+    raise TypeError(type(expr))
+
+
+md5_unpacker = struct.Struct("4I")
+
+
+def stable_hash(t: tuple) -> int:
+    cnt = _dump_tuple(t)
+    hd = md5lib.md5(cnt).digest()
+    return md5_unpacker.unpack(hd)[0]  # 32 bits
+
+
+def _dump_tuple(t: tuple) -> bytes:
+    cnt = b""
+    for item in t:
+        if item is not None:
+            type_ = type(item)
+            if type_ in _DUMP_BY_TYPE:
+                cnt += _DUMP_BY_TYPE[type_](item)
+            else:
+                # for TaggedObjects, hash(item) is stable
+                # other types of items may show up, such as pyvex.expr.CCall and Dirty. they will be removed some day.
+                cnt += struct.pack("<Q", hash(item) & 0xFFFF_FFFF_FFFF_FFFF)
+        cnt += b"\xf0"
+    return cnt
+
+
+def _dump_str(t: str) -> bytes:
+    return t.encode("ascii")
+
+
+def _dump_int(t: int) -> bytes:
+    prefix = b"" if t >= 0 else b"-"
+    t = abs(t)
+    if t <= 0xFFFF:
+        return prefix + struct.pack("<H", t)
+    if t <= 0xFFFF_FFFF:
+        return prefix + struct.pack("<I", t)
+    if t <= 0xFFFF_FFFF_FFFF_FFFF:
+        return prefix + struct.pack("<Q", t)
+    cnt = b""
+    while t > 0:
+        cnt += _dump_int(t & 0xFFFF_FFFF_FFFF_FFFF)
+        t >>= 64
+    return prefix + cnt
+
+
+def _dump_type(t: type) -> bytes:
+    return t.__name__.encode("ascii")
+
+
+_DUMP_BY_TYPE = {
+    tuple: _dump_tuple,
+    str: _dump_str,
+    int: _dump_int,
+    type: _dump_type,
+}
+
+
+def is_none_or_likeable(arg1, arg2, is_list=False):
+    """
+    Returns whether two things are both None or can like each other
+    """
+    if arg1 is None or arg2 is None:
+        return arg1 == arg2
+
+    if is_list:
+        return len(arg1) == len(arg2) and all(is_none_or_likeable(a1, a2) for a1, a2 in zip(arg1, arg2))
+
+    if isinstance(arg1, Expression):
+        return arg1.likes(arg2)
+    return arg1 == arg2
+
+
+def is_none_or_matchable(arg1, arg2, is_list=False):
+    """
+    Returns whether two things are both None or can match each other
+    """
+    if arg1 is None or arg2 is None:
+        return arg1 == arg2
+
+    if is_list:
+        return len(arg1) == len(arg2) and all(is_none_or_matchable(a1, a2) for a1, a2 in zip(arg1, arg2))
+
+    if isinstance(arg1, Expression):
+        return arg1.matches(arg2)
+    return arg1 == arg2
+
+
+from .expression import Expression

angr/analyses/calling_convention/calling_convention.py

@@ -9,7 +9,7 @@ import capstone
 
 from pyvex.stmt import Put
 from pyvex.expr import RdTmp
-import ailment
+import angr.ailment as ailment
 
 from angr.code_location import ExternalCodeLocation
 
@@ -988,7 +988,11 @@ class CallingConventionAnalysis(Analysis):
             for ret_block in self._function.ret_sites:
                 fpretval_updated, retval_updated = False, False
                 fp_reg_size = 0
-                irsb = self.project.factory.block(ret_block.addr, size=ret_block.size).vex
+                try:
+                    irsb = self.project.factory.block(ret_block.addr, size=ret_block.size).vex
+                except SimTranslationError:
+                    # failed to lift the block
+                    continue
                 for stmt in irsb.statements:
                     if isinstance(stmt, Put) and isinstance(stmt.data, RdTmp):
                         reg_size = irsb.tyenv.sizeof(stmt.data.tmp) // self.project.arch.byte_width  # type: ignore

angr/analyses/decompiler/ail_simplifier.py

@@ -8,10 +8,10 @@ import logging
 
 import networkx
 
-from ailment import AILBlockWalker
-from ailment.block import Block
-from ailment.statement import Statement, Assignment, Store, Call, ConditionalJump, DirtyStatement, WeakAssignment
-from ailment.expression import (
+from angr.ailment import AILBlockWalker
+from angr.ailment.block import Block
+from angr.ailment.statement import Statement, Assignment, Store, Call, ConditionalJump, DirtyStatement, WeakAssignment
+from angr.ailment.expression import (
     Register,
     Convert,
     Load,
@@ -51,7 +51,7 @@ from .ccall_rewriters import CCALL_REWRITERS
 from .counters.expression_counters import SingleExpressionCounter
 
 if TYPE_CHECKING:
-    from ailment.manager import Manager
+    from angr.ailment.manager import Manager
 
 
 _l = logging.getLogger(__name__)

angr/analyses/decompiler/block_io_finder.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 from collections import defaultdict
 from typing import Any
 
-from ailment import Block
-from ailment.statement import Call, Statement, ConditionalJump, Assignment, Store, Return, Jump
-from ailment.expression import (
+from angr.ailment import Block
+from angr.ailment.statement import Call, Statement, ConditionalJump, Assignment, Store, Return, Jump
+from angr.ailment.expression import (
     Load,
     VirtualVariable,
     Expression,
@@ -16,7 +16,7 @@ from ailment.expression import (
     Const,
     StackBaseOffset,
 )
-from ailment.block_walker import AILBlockWalkerBase
+from angr.ailment.block_walker import AILBlockWalkerBase
 
 
 from angr.knowledge_plugins.key_definitions.atoms import MemoryLocation, Register, SpOffset, ConstantSrc

angr/analyses/decompiler/block_similarity.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 import logging
 import networkx as nx
-from ailment.block import Block
-from ailment.statement import Statement, ConditionalJump
+from angr.ailment.block import Block
+from angr.ailment.statement import Statement, ConditionalJump
 
 from .utils import find_block_by_addr
 

angr/analyses/decompiler/block_simplifier.py

@@ -4,9 +4,9 @@ import logging
 from typing import TYPE_CHECKING
 from collections.abc import Iterable, Mapping
 
-from ailment.statement import Statement, Assignment, Call, Store, Jump
-from ailment.expression import Tmp, Load, Const, Register, Convert, Expression
-from ailment import AILBlockWalkerBase
+from angr.ailment.statement import Statement, Assignment, Call, Store, Jump
+from angr.ailment.expression import Tmp, Load, Const, Register, Convert, Expression
+from angr.ailment import AILBlockWalkerBase
 
 from angr.code_location import ExternalCodeLocation, CodeLocation
 
@@ -26,7 +26,7 @@ from .utils import peephole_optimize_exprs, peephole_optimize_stmts, peephole_op
 
 if TYPE_CHECKING:
     from angr.knowledge_plugins.key_definitions.live_definitions import Definition
-    from ailment.block import Block
+    from angr.ailment.block import Block
 
 
 _l = logging.getLogger(name=__name__)

angr/analyses/decompiler/callsite_maker.py

@@ -4,8 +4,8 @@ import copy
 import logging
 
 import archinfo
-from ailment import Stmt, Expr, Const
-from ailment.manager import Manager
+from angr.ailment import Stmt, Expr, Const
+from angr.ailment.manager import Manager
 
 from angr.procedures.stubs.format_parser import FormatParser, FormatSpecifier
 from angr.sim_type import (

angr/analyses/decompiler/ccall_rewriters/amd64_ccalls.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from ailment import Expr, Stmt
+from angr.ailment import Expr, Stmt
 
 from angr.calling_conventions import SimCCUsercall
 from angr.engines.vex.claripy.ccall import data

angr/analyses/decompiler/ccall_rewriters/rewriter_base.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-import ailment
+import angr.ailment as ailment
 
 
 class CCallRewriterBase:

angr/analyses/decompiler/ccall_rewriters/x86_ccalls.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from ailment import Expr
+from angr.ailment import Expr
 
 from angr.engines.vex.claripy.ccall import data
 from .rewriter_base import CCallRewriterBase

angr/analyses/decompiler/clinic.py

@@ -11,7 +11,7 @@ from dataclasses import dataclass
 import networkx
 import capstone
 
-import ailment
+import angr.ailment as ailment
 
 from angr.errors import AngrDecompilationError
 from angr.knowledge_base import KnowledgeBase
@@ -143,7 +143,8 @@ class Clinic(Analysis):
         desired_variables: set[str] | None = None,
         force_loop_single_exit: bool = True,
         complete_successors: bool = False,
-        max_type_constraints: int = 4000,
+        max_type_constraints: int = 100_000,
+        type_constraint_set_degradation_threshold: int = 150,
         ail_graph: networkx.DiGraph | None = None,
         arg_vvars: dict[int, tuple[ailment.Expr.VirtualVariable, SimVariable]] | None = None,
         start_stage: ClinicStage | None = ClinicStage.INITIALIZATION,
@@ -185,6 +186,7 @@ class Clinic(Analysis):
         self._cache = cache
         self._mode = mode
         self._max_type_constraints = max_type_constraints
+        self._type_constraint_set_degradation_threshold = type_constraint_set_degradation_threshold
         self.vvar_id_start = vvar_id_start
         self.vvar_to_vvar: dict[int, int] | None = None
         # during SSA conversion, we create secondary stack variables because they overlap and are larger than the
@@ -1871,6 +1873,7 @@ class Clinic(Analysis):
                     must_struct=must_struct,
                     ground_truth=groundtruth,
                     stackvar_max_sizes=tv_max_sizes,
+                    constraint_set_degradation_threshold=self._type_constraint_set_degradation_threshold,
                 )
                 # tp.pp_constraints()
                 # tp.pp_solution()

angr/analyses/decompiler/condition_processor.py

@@ -6,7 +6,7 @@ from collections.abc import Generator
 import operator
 import logging
 
-import ailment
+import angr.ailment as ailment
 import claripy
 import networkx
 from unique_log_filter import UniqueLogFilter

angr/analyses/decompiler/counters/boolean_counter.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 import typing
 
-from ailment import AILBlockWalkerBase
+from angr.ailment import AILBlockWalkerBase
 
 if typing.TYPE_CHECKING:
-    from ailment.expression import BinaryOp
-    from ailment.statement import Statement
-    from ailment.block import Block
+    from angr.ailment.expression import BinaryOp
+    from angr.ailment.statement import Statement
+    from angr.ailment.block import Block
 
 
 class BooleanCounter(AILBlockWalkerBase):

angr/analyses/decompiler/counters/call_counter.py

@@ -1,14 +1,14 @@
 from __future__ import annotations
 from typing import TYPE_CHECKING
 
-from ailment import Block
-from ailment.statement import Label
-from ailment.block_walker import AILBlockWalkerBase
+from angr.ailment import Block
+from angr.ailment.statement import Label
+from angr.ailment.block_walker import AILBlockWalkerBase
 
 from angr.analyses.decompiler.sequence_walker import SequenceWalker
 
 if TYPE_CHECKING:
-    from ailment.statement import Call
+    from angr.ailment.statement import Call
 
 
 class AILBlockCallCounter(AILBlockWalkerBase):

angr/analyses/decompiler/counters/expression_counters.py

@@ -3,13 +3,13 @@ from typing import Any, TYPE_CHECKING
 from collections.abc import Iterable
 from collections import defaultdict
 
-from ailment.expression import Expression, Register
-from ailment.statement import Statement
-from ailment.block_walker import AILBlockWalkerBase
-from ailment import Block
+from angr.ailment.expression import Expression, Register
+from angr.ailment.statement import Statement
+from angr.ailment.block_walker import AILBlockWalkerBase
+from angr.ailment import Block
 
 if TYPE_CHECKING:
-    from ailment.expression import BinaryOp, UnaryOp
+    from angr.ailment.expression import BinaryOp, UnaryOp
 
 
 class SingleExpressionCounter(AILBlockWalkerBase):

angr/analyses/decompiler/counters/seq_cf_structure_counter.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from collections import defaultdict
 
-import ailment
+import angr.ailment as ailment
 
 from angr.analyses.decompiler.sequence_walker import SequenceWalker
 from angr.analyses.decompiler.structuring.structurer_nodes import LoopNode

angr/analyses/decompiler/decompiler.py

@@ -7,7 +7,7 @@ from typing import Any, TYPE_CHECKING
 
 import networkx
 from cle import SymbolType
-import ailment
+import angr.ailment as ailment
 
 from angr.analyses.cfg import CFGFast
 from angr.knowledge_plugins.functions.function import Function
@@ -23,7 +23,7 @@ from .ailgraph_walker import AILGraphWalker
 from .condition_processor import ConditionProcessor
 from .decompilation_options import DecompilationOption
 from .decompilation_cache import DecompilationCache
-from .utils import remove_labels, remove_edges_in_ailgraph
+from .utils import remove_edges_in_ailgraph
 from .sequence_walker import SequenceWalker
 from .structuring.structurer_nodes import SequenceNode
 from .presets import DECOMPILATION_PRESETS, DecompilationPreset
@@ -319,12 +319,8 @@ class Decompiler(Analysis):
         # removed!
         remove_edges_in_ailgraph(clinic.graph, clinic.edges_to_remove)
 
-        # Rewrite the graph to remove phi expressions
-        # this is probably optional if we do not pretty-print clinic.graph
-        clinic.graph = self._transform_graph_from_ssa(clinic.graph)
-
         # save the graph before structuring happens (for AIL view)
-        clinic.cc_graph = remove_labels(clinic.copy_graph())
+        clinic.cc_graph = clinic.copy_graph()
 
         codegen = None
         seq_node = None
@@ -357,7 +353,7 @@ class Decompiler(Analysis):
             )
 
             # rewrite the sequence node to remove phi expressions
-            seq_node = self._transform_seqnode_from_ssa(seq_node)
+            seq_node = self.transform_seqnode_from_ssa(seq_node)
 
             # update memory data
             if self._cfg is not None and self._update_memory_data:
@@ -670,15 +666,24 @@ class Decompiler(Analysis):
             memory_data_addrs=added_memory_data_addrs,
         )
 
-    def _transform_graph_from_ssa(self, ail_graph: networkx.DiGraph) -> networkx.DiGraph:
+    def transform_graph_from_ssa(self, ail_graph: networkx.DiGraph) -> networkx.DiGraph:
+        """
+        Translate an SSA AIL graph out of SSA form. This is useful for producing a non-SSA AIL graph for displaying in
+        angr management.
+
+        :param ail_graph:   The AIL graph to transform out of SSA form.
+        :return:            The translated AIL graph.
+        """
+        variable_kb = self._variable_kb
         dephication = self.project.analyses.GraphDephication(
-            self.func, ail_graph, rewrite=True, kb=self.kb, fail_fast=self._fail_fast
+            self.func, ail_graph, rewrite=True, variable_kb=variable_kb, kb=self.kb, fail_fast=self._fail_fast
         )
         return dephication.output
 
-    def _transform_seqnode_from_ssa(self, seq_node: SequenceNode) -> SequenceNode:
+    def transform_seqnode_from_ssa(self, seq_node: SequenceNode) -> SequenceNode:
+        variable_kb = self._variable_kb
         dephication = self.project.analyses.SeqNodeDephication(
-            self.func, seq_node, rewrite=True, kb=self.kb, fail_fast=self._fail_fast
+            self.func, seq_node, rewrite=True, variable_kb=variable_kb, kb=self.kb, fail_fast=self._fail_fast
         )
         return dephication.output
 

angr/analyses/decompiler/dephication/dephication_base.py

@@ -1,10 +1,14 @@
 from __future__ import annotations
+from typing import TYPE_CHECKING
 from typing import Any
 import logging
 from collections import defaultdict
 
 from angr.analyses import Analysis
 
+if TYPE_CHECKING:
+    from angr import KnowledgeBase
+
 l = logging.getLogger(name=__name__)
 
 
@@ -14,12 +18,19 @@ class DephicationBase(Analysis):
     AIL statement containers.
     """
 
-    def __init__(self, func, vvar_to_vvar_mapping: dict[int, int] | None = None, rewrite: bool = False):
+    def __init__(
+        self,
+        func,
+        vvar_to_vvar_mapping: dict[int, int] | None = None,
+        rewrite: bool = False,
+        variable_kb: KnowledgeBase | None = None,
+    ):
         if isinstance(func, str):
             self._function = self.kb.functions[func]
         else:
             self._function = func
 
+        self.variable_kb = variable_kb
         self.vvar_to_vvar_mapping = vvar_to_vvar_mapping if vvar_to_vvar_mapping is not None else None
         self.rewrite = rewrite
         self.output = None

angr/analyses/decompiler/dephication/graph_dephication.py

@@ -1,17 +1,21 @@
 from __future__ import annotations
+from typing import TYPE_CHECKING
 import logging
 from collections import defaultdict
 
 import networkx
 
-from ailment.expression import Phi, VirtualVariable
-from ailment.statement import Assignment
-
+from angr.ailment.expression import Phi, VirtualVariable
+from angr.ailment.statement import Assignment
 from angr.knowledge_plugins.functions import Function
 from angr.analyses import register_analysis
 from .graph_rewriting import GraphRewritingAnalysis
 from .dephication_base import DephicationBase
 
+if TYPE_CHECKING:
+    from angr import KnowledgeBase
+
+
 l = logging.getLogger(name=__name__)
 
 
@@ -27,6 +31,7 @@ class GraphDephication(DephicationBase):  # pylint:disable=abstract-method
         ail_graph,
         vvar_to_vvar_mapping: dict[int, int] | None = None,
         rewrite: bool = False,
+        variable_kb: KnowledgeBase | None = None,
     ):
         """
         :param func:                            The subject of the analysis: a function, or a single basic block
@@ -35,7 +40,7 @@ class GraphDephication(DephicationBase):  # pylint:disable=abstract-method
 
         self._graph = ail_graph
 
-        super().__init__(func, vvar_to_vvar_mapping=vvar_to_vvar_mapping, rewrite=rewrite)
+        super().__init__(func, vvar_to_vvar_mapping=vvar_to_vvar_mapping, rewrite=rewrite, variable_kb=variable_kb)
 
         self._analyze()
 
@@ -56,7 +61,9 @@ class GraphDephication(DephicationBase):  # pylint:disable=abstract-method
 
     def _rewrite_container(self) -> networkx.DiGraph:
         # replace all vvars with phi variables in the graph
-        rewriter = GraphRewritingAnalysis(self.project, self._function, self._graph, self.vvar_to_vvar_mapping)
+        rewriter = GraphRewritingAnalysis(
+            self.project, self._function, self._graph, self.vvar_to_vvar_mapping, variable_kb=self.variable_kb
+        )
         return rewriter.out_graph
 
 

angr/analyses/decompiler/dephication/graph_rewriting.py

@@ -4,8 +4,7 @@ import logging
 
 import networkx
 
-import ailment
-
+from angr import ailment
 from angr.utils.ail import is_phi_assignment
 from angr.analyses import ForwardAnalysis
 from angr.analyses.forward_analysis.visitors.graph import NodeType
@@ -21,23 +20,20 @@ class GraphRewritingAnalysis(ForwardAnalysis[None, NodeType, object, object]):
     This analysis traverses the AIL graph and rewrites virtual variables accordingly.
     """
 
-    def __init__(
-        self,
-        project,
-        func,
-        ail_graph,
-        vvar_to_vvar: dict[int, int],
-    ):
+    def __init__(self, project, func, ail_graph, vvar_to_vvar: dict[int, int], variable_kb=None):
         self.project = project
         self._function = func
         self._graph_visitor = FunctionGraphVisitor(self._function, ail_graph)
+        self.variable_kb = variable_kb
 
         ForwardAnalysis.__init__(
             self, order_jobs=False, allow_merging=False, allow_widening=False, graph_visitor=self._graph_visitor
         )
         self._graph = ail_graph
         self._vvar_to_vvar = vvar_to_vvar
-        self._engine_ail = SimEngineDephiRewriting(self.project, self._vvar_to_vvar)
+        self._engine_ail = SimEngineDephiRewriting(
+            self.project, self._vvar_to_vvar, func_addr=self._function.addr, variable_kb=self.variable_kb
+        )
 
         self._visited_blocks: set[Any] = set()
         self.out_blocks = {}