angr 9.2.165__cp310-abi3-macosx_10_12_x86_64.whl

angr/exploration_techniques/oppologist.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+import functools
+import logging
+
+import claripy
+
+from angr.errors import AngrError, SimError, SimUnsupportedError, SimCCallError
+from angr import sim_options
+from angr.engines.successors import SimSuccessors
+from .base import ExplorationTechnique
+
+l = logging.getLogger(name=__name__)
+
+exc_list = (AngrError, SimError, claripy.ClaripyError, TypeError, ValueError, ArithmeticError, MemoryError)
+
+
+class Oppologist(ExplorationTechnique):
+    """
+    The Oppologist is an exploration technique that forces uncooperative code through qemu.
+    """
+
+    def __init__(self):
+        ExplorationTechnique.__init__(self)
+
+    @staticmethod
+    def _restore_state(old, new):
+        new.release_plugin("unicorn")
+        new.register_plugin("unicorn", old.unicorn.copy())
+        new.options = old.options.copy()
+
+    def _oppologize(self, simgr, state, pn, **kwargs):
+        l.debug("... pn: %s", pn)
+
+        pn.options.add(sim_options.UNICORN)
+        pn.options.add(sim_options.UNICORN_AGGRESSIVE_CONCRETIZATION)
+        pn.unicorn.max_steps = 1
+        pn.unicorn.countdown_symbolic_stop = 0
+        pn.unicorn.countdown_unsupported_stop = 0
+        pn.unicorn.countdown_nonunicorn_blocks = 0
+        pn.unicorn.countdown_stop_point = 0
+        ss = simgr.successors(pn, throw=True, **kwargs)
+
+        fixup = functools.partial(self._restore_state, state)
+
+        l.debug("... successors: %s", ss)
+        for s in ss.flat_successors + ss.unconstrained_successors + ss.unsat_successors + ss.successors:
+            fixup(s)
+
+        return ss
+
+    @staticmethod
+    def _combine_results(*results):
+        final = SimSuccessors(results[0].addr, results[0].initial_state)
+        final.description = "Oppology"
+        final.sort = "Oppologist"
+
+        for med in results:
+            final.processed = True
+            final.successors.extend(med.successors)
+            final.all_successors.extend(med.all_successors)
+            final.flat_successors.extend(med.flat_successors)
+            final.unsat_successors.extend(med.unsat_successors)
+            final.unconstrained_successors.extend(med.unsat_successors)
+
+        return final
+
+    def _delayed_oppology(self, simgr, state, e, **kwargs):
+        ss = simgr.successors(state, num_inst=e.executed_instruction_count, **kwargs)
+        need_oppologizing = [s for s in ss.flat_successors if s.addr == e.ins_addr]
+        ss.flat_successors = [s for s in ss.flat_successors if s.addr != e.ins_addr]
+        results = [ss]
+
+        results.extend(map(functools.partial(self._oppologize, simgr, state, **kwargs), need_oppologizing))
+        return self._combine_results(*results)
+
+    def successors(self, simgr, state, **kwargs):
+        try:
+            kwargs.pop("throw", None)
+            return simgr.successors(state, **kwargs)
+
+        except (SimUnsupportedError, SimCCallError) as e:
+            l.debug("Errored on path %s after %d instructions", state, e.executed_instruction_count)
+            try:
+                if e.executed_instruction_count:
+                    return self._delayed_oppology(simgr, state, e, **kwargs)
+                return self._oppologize(simgr, state, state.copy(), **kwargs)
+            except exc_list as err:
+                l.error("Oppologizer hit an error while trying to perform repairs", exc_info=True)
+                raise e from err
+        except Exception:  # pylint:disable=broad-except
+            l.error("Original block hit an unsupported error", exc_info=True)
+            raise

angr/exploration_techniques/slicecutor.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import logging
+
+from angr.errors import AngrExitError
+from .base import ExplorationTechnique
+
+
+l = logging.getLogger(name=__name__)
+
+
+class Slicecutor(ExplorationTechnique):
+    """
+    The Slicecutor is an exploration that executes provided code slices.
+    """
+
+    def __init__(self, annotated_cfg, force_taking_exit=False, force_sat: bool = False):
+        """
+        All parameters except `annotated_cfg` are optional.
+
+        :param annotated_cfg:       The AnnotatedCFG that provides the code slice.
+        :param force_taking_exit:   Set to True if you want to create a successor based on our slice in case of
+                                    unconstrained successors.
+        :param force_sat:           If a branch specified by the slice is unsatisfiable, set this option to True if you
+                                    want to force it to be satisfiable and be taken anyway.
+        """
+        super().__init__()
+
+        self._annotated_cfg = annotated_cfg
+        self._force_taking_exit = force_taking_exit
+        self._force_sat = force_sat
+
+    def setup(self, simgr):
+        for stash in ("cut", "mysteries"):
+            simgr.populate(stash, [])
+
+    def filter(self, simgr, state, **kwargs):
+        l.debug("Checking state %s for filtering...", state)
+        return simgr.filter(state, **kwargs)
+
+    def step_state(self, simgr, state, **kwargs):
+        l.debug("%s ticking state %s at address %#x.", self, state, state.addr)
+        stashes = simgr.step_state(state, **kwargs)
+
+        new_active = []
+        new_cut = []
+        new_mystery = []
+
+        # SimulationManager returns new active states in the None stash by default.
+        flat_successors = stashes.get(None, None)
+        if flat_successors is None:
+            # Did the user explicitly put them into the 'active' stash instead?
+            flat_successors = stashes.pop("active", [])
+
+        for successor in flat_successors:
+            l.debug("... checking exit to %#x from %#x.", successor.addr, state.addr)
+
+            try:
+                taken = self._annotated_cfg.should_take_exit(state.addr, successor.addr)
+            except AngrExitError:  # TODO: which exception?
+                l.debug("... annotated CFG did not know about it!")
+                new_mystery.append(successor)
+            else:
+                if taken:
+                    l.debug("... taking the exit.")
+                    new_active.append(successor)
+                    # the else case isn't here, because the state should set errored in this
+                    # case and we'll catch it below
+                else:
+                    l.debug("... not taking the exit.")
+                    new_cut.append(successor)
+
+        unconstrained_successors = stashes.get("unconstrained", [])
+        if not new_active and unconstrained_successors and self._force_taking_exit:
+            stashes["unconstrained"] = []
+            # somehow there is no feasible state. We are forced to create a successor based on our slice
+            if len(unconstrained_successors) != 1:
+                raise Exception("This should absolutely never happen, what?")
+            for target in self._annotated_cfg.get_targets(state.addr):
+                successor = unconstrained_successors[0].copy()
+                successor.regs._ip = target
+                new_active.append(successor)
+            l.debug("Got unconstrained: %d new states are created based on AnnotatedCFG.", len(new_active))
+
+        unsat_successors = stashes.get("unsat", None)
+        if not new_active and unsat_successors and self._force_sat:
+            stashes["unsat"] = []
+            # find the state
+            targets = self._annotated_cfg.get_targets(state.addr)
+            if targets is None:
+                targets = []
+            for target in targets:
+                try:
+                    suc = next(iter(u for u in unsat_successors if u.addr == target))
+                except StopIteration:
+                    continue
+
+                # drop all constraints
+                if suc.mode == "fastpath":
+                    # dropping constraints and making the state satisfiable again under fastpath mode is easy
+                    suc.solver._solver.constraints = []
+                    suc._satisfiable = True
+                    new_active.append(suc)
+                    l.debug("Forced unsat at %#x to be sat again.", suc.addr)
+                else:
+                    # with multiple possible solver frontends, dropping states in other state modes is not
+                    # straightforward. I'll leave it to the next person who uses this feature
+                    l.warning("force_sat is not implemented for solver mode %s.", suc.moe)
+
+        stashes[None] = new_active
+        stashes["mystery"] = new_mystery
+        stashes["cut"] = new_cut
+        return stashes
+
+    def successors(self, simgr, state, **kwargs):
+        kwargs["whitelist"] = self._annotated_cfg.get_whitelisted_statements(state.addr)
+        kwargs["last_stmt"] = self._annotated_cfg.get_last_statement_index(state.addr)
+        return simgr.successors(state, **kwargs)

angr/exploration_techniques/spiller.py

@@ -0,0 +1,280 @@
+# pylint:disable=no-member,import-outside-toplevel
+from __future__ import annotations
+
+import contextlib
+import logging
+
+from angr import vaults
+from .base import ExplorationTechnique
+
+
+l = logging.getLogger(name=__name__)
+
+
+class PickledStatesBase:
+    """
+    The base class of pickled states
+    """
+
+    def sort(self):
+        """
+        Sort pickled states.
+        """
+
+        raise NotImplementedError
+
+    def add(self, prio, sid):
+        """
+        Add a newly pickled state.
+
+        :param int prio:    Priority of the state.
+        :param str sid:     Persistent ID of the state.
+        :return:            None
+        """
+        raise NotImplementedError
+
+    def pop_n(self, n):
+        """
+        Pop the top N states.
+
+        :param int n:   Number of states to take.
+        :return:        A list of states.
+        """
+        raise NotImplementedError
+
+
+class PickledStatesList(PickledStatesBase):
+    """
+    List-backed pickled state storage.
+    """
+
+    def __init__(self):
+        self._picked_states = []
+
+    def sort(self):
+        self._picked_states.sort()
+
+    def add(self, prio, sid):
+        self._picked_states.append((prio, sid))
+
+    def pop_n(self, n):
+        ss = self._picked_states[:n]
+        self._picked_states[:n] = []
+        return ss
+
+
+class PickledStatesDb(PickledStatesBase):
+    """
+    Database-backed pickled state storage.
+    """
+
+    def __init__(self, db_str="sqlite:///:memory:"):
+        from .spiller_db import sqlalchemy, create_engine, Base, OperationalError, sessionmaker
+
+        if sqlalchemy is None:
+            raise ImportError(
+                f"Cannot import SQLAlchemy. Please install SQLAlchemy before using " f"{self.__class__.__name__}."
+            )
+
+        # ORM declarations
+        engine = create_engine(db_str)
+
+        # create table (unless it already exists)
+        with contextlib.suppress(OperationalError):
+            Base.metadata.create_all(engine, checkfirst=True)
+
+        self.Session = sessionmaker(bind=engine)
+
+    def sort(self):
+        pass
+
+    def add(self, prio, sid, taken=False, stash="spilled"):  # pylint:disable=arguments-differ
+        from .spiller_db import PickledState
+
+        record = PickledState(id=sid, priority=prio, taken=taken, stash=stash)
+        session = self.Session()
+        session.add(record)
+        session.commit()
+        session.close()
+
+    def pop_n(self, n, stash="spilled"):  # pylint:disable=arguments-differ
+        from .spiller_db import PickledState
+
+        session = self.Session()
+        q = (
+            session.query(PickledState)
+            .filter_by(taken=False)
+            .filter_by(stash=stash)
+            .order_by(PickledState.priority)
+            .limit(n)
+            .all()
+        )
+
+        ss = []
+        for r in q:
+            r.taken = True
+            ss.append((r.priority, r.id))
+        session.commit()
+        session.close()
+        return ss
+
+    def get_recent_n(self, n, stash="spilled"):
+        from .spiller_db import PickledState
+
+        session = self.Session()
+        q = session.query(PickledState).filter_by(stash=stash).order_by(PickledState.timestamp.desc()).limit(n).all()
+
+        ss = []
+        for r in q:
+            ss.append((r.timestamp, r.id))
+        session.close()
+        return ss
+
+    def count(self):
+        from .spiller_db import PickledState
+
+        session = self.Session()
+        q = session.query(PickledState).count()
+        session.close()
+        return q
+
+
+class Spiller(ExplorationTechnique):
+    """
+    Automatically spill states out. It can spill out states to a different stash, spill
+    them out to ANA, or first do the former and then (after enough states) the latter.
+    """
+
+    def __init__(
+        self,
+        src_stash="active",
+        min=5,
+        max=10,  # pylint:disable=redefined-builtin
+        staging_stash="spill_stage",
+        staging_min=10,
+        staging_max=20,
+        pickle_callback=None,
+        unpickle_callback=None,
+        post_pickle_callback=None,
+        priority_key=None,
+        vault=None,
+        states_collection=None,
+    ):
+        """
+        Initializes the spiller.
+
+        :param max:          the number of states that are *not* spilled
+        :param src_stash:    the stash from which to spill states (default: active)
+        :param staging_stash: the stash *to* which to spill states (default: "spill_stage")
+        :param staging_max:  the number of states that can be in the staging stash before things get spilled to ANA
+                             (default: None. If staging_stash is set, then this means unlimited, and ANA will not be
+                             used).
+        :param priority_key: a function that takes a state and returns its numerical priority (MAX_INT is lowest
+                             priority). By default, self.state_priority will be used, which prioritizes by object ID.
+        :param vault:        an angr.Vault object to handle storing and loading of states. If not provided, an
+                             angr.vaults.VaultShelf will be created with a temporary file.
+        """
+        super().__init__()
+        self.max = max
+        self.min = min
+        self.src_stash = src_stash
+        self.staging_stash = staging_stash
+        self.staging_max = staging_max
+        self.staging_min = staging_min
+
+        # various callbacks
+        self.priority_key = priority_key
+        self.unpickle_callback = unpickle_callback
+        self.pickle_callback = pickle_callback
+        self.post_pickle_callback = post_pickle_callback
+
+        # tracking of pickled stuff
+        self._pickled_states = PickledStatesList() if states_collection is None else states_collection
+        self._ever_pickled = 0
+        self._ever_unpickled = 0
+        self._vault = vaults.VaultShelf() if vault is None else vault
+
+    def _unpickle(self, n):
+        self._pickled_states.sort()
+        unpickled = [(sid, self._load_state(sid)) for _, sid in self._pickled_states.pop_n(n)]
+        self._ever_unpickled += len(unpickled)
+        if self.unpickle_callback:
+            for sid, u in unpickled:
+                self.unpickle_callback(sid, u)
+        return [u for _, u in unpickled]
+
+    def _get_priority(self, state):
+        return (self.priority_key or self.state_priority)(state)
+
+    def _pickle(self, states):
+        if self.pickle_callback:
+            for s in states:
+                self.pickle_callback(s)
+        self._ever_pickled += len(states)
+        for state in states:
+            try:
+                state_oid = self._store_state(state)
+            except RecursionError:
+                l.warning(
+                    "Couldn't store the state because of a recursion error. This is most likely to be pickle's "
+                    "fault. You may try to increase the recursion limit using sys.setrecursionlimit()."
+                )
+                continue
+            prio = self._get_priority(state)
+            if self.post_pickle_callback:
+                self.post_pickle_callback(state, prio, state_oid)
+            self._pickled_states.add(prio, state_oid)
+
+    def _store_state(self, state):
+        return self._vault.store(state)
+
+    def _load_state(self, sid):
+        return self._vault.load(sid)
+
+    def step(self, simgr, stash="active", **kwargs):
+        simgr = simgr.step(stash=stash, **kwargs)
+
+        l.debug(
+            "STASH STATUS: active: %d, staging: %d",
+            len(simgr.stashes[self.src_stash]),
+            len(simgr.stashes[self.staging_stash]),
+        )
+
+        states = simgr.stashes[self.src_stash]
+        staged_states = simgr.stashes.setdefault(self.staging_stash, []) if self.staging_stash else []
+
+        if len(states) < self.min:
+            missing = (self.max + self.min) // 2 - len(states)
+            l.debug("Too few states (%d/%d) in stash %s.", len(states), self.min, self.src_stash)
+            if self.staging_stash:
+                l.debug("... retrieving states from staging stash (%s)", self.staging_stash)
+                staged_states.sort(key=self.priority_key or self.state_priority)
+                states += staged_states[:missing]
+                staged_states[:missing] = []
+            else:
+                l.debug("... staging stash disabled; unpickling states")
+                states += self._unpickle(missing)
+
+        if len(states) > self.max:
+            l.debug("Too many states (%d/%d) in stash %s", len(states), self.max, self.src_stash)
+            states.sort(key=self.priority_key or self.state_priority)
+            staged_states += states[self.max :]
+            states[self.max :] = []
+
+        # if we have too few staged states, unpickle up to halfway between max and min
+        if len(staged_states) < self.staging_min:
+            l.debug("Too few states in staging stash (%s)", self.staging_stash)
+            staged_states += self._unpickle((self.staging_min + self.staging_max) // 2 - len(staged_states))
+
+        if len(staged_states) > self.staging_max:
+            l.debug("Too many states in staging stash (%s)", self.staging_stash)
+            self._pickle(staged_states[self.staging_max :])
+            staged_states[self.staging_max :] = []
+
+        simgr.stashes[self.src_stash] = states
+        simgr.stashes[self.staging_stash] = staged_states
+        return simgr
+
+    @staticmethod
+    def state_priority(state):
+        return id(state)

angr/exploration_techniques/spiller_db.py

@@ -0,0 +1,27 @@
+# pylint:disable=missing-class-docstring,unused-import
+from __future__ import annotations
+import datetime
+
+try:
+    import sqlalchemy
+    from sqlalchemy import Column, Integer, String, Boolean, DateTime, create_engine
+    from sqlalchemy.orm import declarative_base, sessionmaker
+    from sqlalchemy.exc import OperationalError
+
+    Base = declarative_base()
+
+    class PickledState(Base):
+        __tablename__ = "pickled_states"
+
+        id = Column(String, primary_key=True)
+        priority = Column(Integer)
+        taken = Column(Boolean, default=False)
+        stash = Column(String, default="")
+        timestamp = Column(DateTime, default=datetime.datetime.utcnow)
+
+except ImportError:
+    sqlalchemy = None
+    create_engine = None
+    Base = None
+    OperationalError = None
+    sessionmaker = None

angr/exploration_techniques/stochastic.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import random
+from collections import defaultdict
+
+from .base import ExplorationTechnique
+
+
+class StochasticSearch(ExplorationTechnique):
+    """
+    Stochastic Search.
+
+    Will only keep one path active at a time, any others will be discarded.
+    Before each pass through, weights are randomly assigned to each basic block.
+    These weights form a probability distribution for determining which state remains after splits.
+    When we run out of active paths to step, we start again from the start state.
+    """
+
+    def __init__(self, start_state, restart_prob=0.0001):
+        """
+        :param start_state:  The initial state from which exploration stems.
+        :param restart_prob: The probability of randomly restarting the search (default 0.0001).
+        """
+        super().__init__()
+        self.start_state = start_state
+        self.restart_prob = restart_prob
+        self._random = random.Random()
+        self._random.seed(42)
+        self.affinity = defaultdict(self._random.random)
+
+    def step(self, simgr, stash="active", **kwargs):
+        simgr = simgr.step(stash=stash, **kwargs)
+
+        if not simgr.stashes[stash] or self._random.random() < self.restart_prob:
+            simgr.stashes[stash] = [self.start_state]
+            self.affinity.clear()
+
+        if len(simgr.stashes[stash]) >= 2:
+
+            def weighted_pick(states):
+                """
+                param states: Diverging states.
+                """
+                assert len(states) >= 2
+                total_weight = sum(self.affinity[s.addr] for s in states)
+                selected = self._random.uniform(0, total_weight)
+                for i, state in enumerate(states):
+                    weight = self.affinity[state.addr]
+                    if selected < weight:
+                        return states[i]
+                    selected -= weight
+                return states[len(states) - 1]
+
+            simgr.stashes[stash] = [weighted_pick(simgr.stashes[stash])]
+
+        return simgr

angr/exploration_techniques/stub_stasher.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from .base import ExplorationTechnique
+
+
+class StubStasher(ExplorationTechnique):
+    """
+    Stash states that reach a stub SimProcedure.
+    """
+
+    @staticmethod
+    def post_filter(state):
+        hook = state.project.hooked_by(state.addr)
+        return hook and hook.is_stub
+
+    def step(self, simgr, stash="active", **kwargs):
+        simgr.step(stash=stash, **kwargs)
+        simgr.move(stash, "stub", filter_func=self.post_filter)
+        return simgr