crosshair-tool 0.0.56__cp39-cp39-macosx_11_0_arm64.whl → 0.0.100__cp39-cp39-macosx_11_0_arm64.whl

crosshair/statespace.py

@@ -10,6 +10,7 @@ from collections import Counter, defaultdict
 from dataclasses import dataclass
 from sys import _getframe
 from time import monotonic
+from traceback import extract_stack, format_tb
 from types import FrameType
 from typing import (
     Any,
@@ -17,6 +18,7 @@ from typing import (
     Dict,
     List,
     NewType,
+    NoReturn,
     Optional,
     Sequence,
     Set,
@@ -29,18 +31,22 @@ import z3  # type: ignore
 
 from crosshair import dynamic_typing
 from crosshair.condition_parser import ConditionExpr
+from crosshair.smtlib import parse_smtlib_literal
 from crosshair.tracers import NoTracing, ResumedTracing, is_tracing
 from crosshair.util import (
-    CrosshairInternal,
+    CROSSHAIR_EXTRA_ASSERTS,
+    CrossHairInternal,
     IgnoreAttempt,
+    NotDeterministic,
     PathTimeout,
     UnknownSatisfiability,
+    assert_tracing,
+    ch_stack,
     debug,
     in_debug,
     name_of_type,
-    test_stack,
 )
-from crosshair.z3util import z3Aassert, z3Not, z3PopNot
+from crosshair.z3util import z3Aassert, z3Not, z3Or, z3PopNot
 
 
 @functools.total_ordering
@@ -159,14 +165,19 @@ def model_value_to_python(value: z3.ExprRef) -> object:
         if value.num_args() == 1:
             ret.append(model_value_to_python(value.arg(0)))
         return ret
-    elif z3.is_int(value):
+    elif z3.is_fp(value):
+        return parse_smtlib_literal(value.sexpr())
+    elif hasattr(value, "py_value"):
+        # TODO: how many other cases could be handled with py_value nowadays?
+        return value.py_value()
+    elif z3.is_int(value):  # catch for older z3 versions that don't have py_value
         return value.as_long()
     else:
         return ast.literal_eval(repr(value))
 
 
+@assert_tracing(False)
 def prefer_true(v: Any) -> bool:
-    assert not is_tracing()
     if not (hasattr(v, "var") and z3.is_bool(v.var)):
         with ResumedTracing():
             v = v.__bool__()
@@ -176,6 +187,22 @@ def prefer_true(v: Any) -> bool:
     return space.choose_possible(v.var, probability_true=1.0)
 
 
+def force_true(v: Any) -> None:
+    with NoTracing():
+        if not (hasattr(v, "var") and z3.is_bool(v.var)):
+            with ResumedTracing():
+                v = v.__bool__()
+            if not (hasattr(v, "var")):
+                raise CrossHairInternal(
+                    "Attempted to call assert_true on a non-symbolic"
+                )
+        space = context_statespace()
+        # TODO: we can improve this by making a new kind of (unary) assertion node
+        # that would not create these useless forks when the space is near exhaustion.
+        if not space.choose_possible(v.var, probability_true=1.0):
+            raise IgnoreAttempt
+
+
 class StateSpaceCounter(Counter):
     @property
     def iterations(self) -> int:
@@ -194,12 +221,8 @@ class StateSpaceCounter(Counter):
         return "{" + ", ".join(parts) + "}"
 
 
-class NotDeterministic(Exception):
-    pass
-
-
 class AbstractPathingOracle:
-    def pre_path_hook(self, root: "RootNode") -> None:
+    def pre_path_hook(self, space: "StateSpace") -> None:
         pass
 
     def post_path_hook(self, path: Sequence["SearchTreeNode"]) -> None:
@@ -225,14 +248,16 @@ class StateSpaceContext:
 
     def __enter__(self):
         prev = real_getattr(_THREAD_LOCALS, "space", None)
-        assert prev is None, "Already in a state space context"
+        if prev is not None:
+            raise CrossHairInternal("Already in a state space context")
         space = self.space
         _THREAD_LOCALS.space = space
         space.mark_all_parent_frames()
 
     def __exit__(self, exc_type, exc_value, tb):
         prev = real_getattr(_THREAD_LOCALS, "space", None)
-        assert prev is self.space, "State space was altered in context"
+        if prev is not self.space:
+            raise CrossHairInternal("State space was altered in context")
         _THREAD_LOCALS.space = None
         return False
 
@@ -244,7 +269,7 @@ def optional_context_statespace() -> Optional["StateSpace"]:
 def context_statespace() -> "StateSpace":
     space = _THREAD_LOCALS.space
     if space is None:
-        raise CrosshairInternal
+        raise CrossHairInternal("Not in a statespace context")
     return space
 
 
@@ -268,59 +293,27 @@ class NodeLike:
         """
         raise NotImplementedError
 
-    def is_stem(self) -> bool:
-        return False
-
-    def grow_into(self, node: _N) -> _N:
-        raise NotImplementedError
-
-    def simplify(self) -> "NodeLike":
-        return self
-
     def stats(self) -> StateSpaceCounter:
         raise NotImplementedError
 
+    children_fields: Tuple[str, ...] = ()
 
-class NodeStem(NodeLike):
-    evolution: Optional["SearchTreeNode"] = None
-
-    def is_exhausted(self) -> bool:
-        return False if self.evolution is None else self.evolution.is_exhausted()
-
-    def get_result(self) -> CallAnalysis:
-        return (
-            CallAnalysis(VerificationStatus.UNKNOWN)
-            if self.evolution is None
-            else self.evolution.get_result()
-        )
-
-    def is_stem(self) -> bool:
-        return self.evolution is None
-
-    def grow_into(self, node: _N) -> _N:
-        self.evolution = node
-        return node
-
-    def simplify(self):
-        return self if self.evolution is None else self.evolution
-
-    def stats(self) -> StateSpaceCounter:
-        return StateSpaceCounter() if self.evolution is None else self.evolution.stats()
-
-    def __repr__(self) -> str:
-        return "NodeStem()"
+    def replace_child(self, current_child, replacement_child: "NodeLike") -> None:
+        for child_field in self.children_fields:
+            child = getattr(self, child_field)
+            if child is current_child:
+                setattr(self, child_field, replacement_child)
+                return
+        raise CrossHairInternal(f"Child {current_child} not found in {self}")
 
 
 class SearchTreeNode(NodeLike):
-    """
-    Represent a single decision point.
-
-    Abstract helper class for StateSpace.
-    """
+    """A node in the execution path tree."""
 
     stacktail: Tuple[str, ...] = ()
     result: CallAnalysis = CallAnalysis()
     exhausted: bool = False
+    iteration: Optional[int] = None
 
     def choose(
         self, space: "StateSpace", probability_true: Optional[float] = None
@@ -345,13 +338,34 @@ class SearchTreeNode(NodeLike):
         raise NotImplementedError
 
 
-def solver_is_sat(solver, *a) -> bool:
-    ret = solver.check(*a)
+class NodeStem(NodeLike):
+    def __init__(self, parent: SearchTreeNode, parent_attr_name: str):
+        self.parent = parent
+        self.parent_attr_name = parent_attr_name
+
+    def is_exhausted(self) -> bool:
+        return False
+
+    def get_result(self) -> CallAnalysis:
+        return CallAnalysis(VerificationStatus.UNKNOWN)
+
+    def stats(self) -> StateSpaceCounter:
+        return StateSpaceCounter()
+
+    def __repr__(self) -> str:
+        return "NodeStem()"
+
+
+def solver_is_sat(solver, *exprs) -> bool:
+    ret = solver.check(*exprs)
     if ret == z3.unknown:
-        debug("Z3 unknown sat reason:", solver.reason_unknown())
+        debug("Z3 Unknown satisfiability. Reason:", solver.reason_unknown())
+        debug("Call stack at time of unknown sat:", ch_stack())
         if solver.reason_unknown() == "interrupted from keyboard":
             raise KeyboardInterrupt
-        debug("Unknown satisfiability. Solver state follows:\n", solver.sexpr())
+        if exprs:
+            debug("While attempting to assert\n", *(e.sexpr() for e in exprs))
+        debug("Solver state follows:\n", solver.sexpr())
         raise UnknownSatisfiability
     return ret == z3.sat
 
@@ -390,7 +404,7 @@ class SinglePathNode(SearchTreeNode):
 
     def __init__(self, decision: bool):
         self.decision = decision
-        self.child = NodeStem()
+        self.child = NodeStem(self, "child")
         self._random = newrandom()
 
     def choose(
@@ -399,12 +413,14 @@ class SinglePathNode(SearchTreeNode):
         return (self.decision, 1.0, self.child)
 
     def compute_result(self, leaf_analysis: CallAnalysis) -> Tuple[CallAnalysis, bool]:
-        self.child = self.child.simplify()
+        assert isinstance(self.child, SearchTreeNode)
         return (self.child.get_result(), self.child.is_exhausted())
 
     def stats(self) -> StateSpaceCounter:
         return self.child.stats()
 
+    children_fields = ("child",)
+
 
 class BranchCounter:
     __slots__ = ["pos_ct", "neg_ct"]
@@ -424,10 +440,11 @@ class RootNode(SinglePathNode):
         )
         from crosshair.pathing_oracle import CoveragePathingOracle  # circular import
 
-        self.pathing_oracle = CoveragePathingOracle()
+        self.pathing_oracle: AbstractPathingOracle = CoveragePathingOracle()
+        self.iteration = 0
 
 
-class DeatchedPathNode(SinglePathNode):
+class DetachedPathNode(SinglePathNode):
     def __init__(self):
         super().__init__(True)
         # Seems like `exhausted` should be True, but we set to False until we can
@@ -437,7 +454,6 @@ class DeatchedPathNode(SinglePathNode):
         self._stats = None
 
     def compute_result(self, leaf_analysis: CallAnalysis) -> Tuple[CallAnalysis, bool]:
-        self.child = self.child.simplify()
         return (leaf_analysis, True)
 
     def stats(self) -> StateSpaceCounter:
@@ -467,13 +483,15 @@ class BinaryPathNode(SearchTreeNode):
     def stats(self) -> StateSpaceCounter:
         return self._stats
 
+    children_fields = ("negative", "positive")
+
 
 class RandomizedBinaryPathNode(BinaryPathNode):
     def __init__(self, rand: random.Random):
         super().__init__()
         self._random = rand
-        self.positive = NodeStem()
-        self.negative = NodeStem()
+        self.positive = NodeStem(self, "positive")
+        self.negative = NodeStem(self, "negative")
 
     def probability_true(
         self, space: "StateSpace", requested_probability: Optional[float] = None
@@ -498,10 +516,6 @@ class RandomizedBinaryPathNode(BinaryPathNode):
         else:
             return (positive_ok, 1.0, self.positive if positive_ok else self.negative)
 
-    def _simplify(self) -> None:
-        self.positive = self.positive.simplify()
-        self.negative = self.negative.simplify()
-
 
 class ParallelNode(RandomizedBinaryPathNode):
     """Choose either path; the first complete result will be used."""
@@ -515,7 +529,6 @@ class ParallelNode(RandomizedBinaryPathNode):
         return f"ParallelNode(false_pct={self._false_probability}, {self._desc})"
 
     def compute_result(self, leaf_analysis: CallAnalysis) -> Tuple[CallAnalysis, bool]:
-        self._simplify()
         positive, negative = self.positive, self.negative
         pos_exhausted = positive.is_exhausted()
         neg_exhausted = negative.is_exhausted()
@@ -592,13 +605,11 @@ class WorstResultNode(RandomizedBinaryPathNode):
             if not solver_is_sat(solver, expr):
                 self.forced_path = False
         else:
-            # TODO: we still run into soundness issues on occasion, so I'd like to
-            # leave _PERFORM_EXTRA_SAT_CHECKS enabled a little longer:
-            _PERFORM_EXTRA_SAT_CHECKS = True
-            if _PERFORM_EXTRA_SAT_CHECKS and not solver_is_sat(solver, expr):
+            # We run into soundness issues on occasion:
+            if CROSSHAIR_EXTRA_ASSERTS and not solver_is_sat(solver, expr):
                 debug(" *** Reached impossible code path *** ")
                 debug("Current solver state:\n", str(solver))
-                raise CrosshairInternal("Reached impossible code path")
+                raise CrossHairInternal("Reached impossible code path")
             self.forced_path = True
         self.expr = expr
 
@@ -635,7 +646,6 @@ class WorstResultNode(RandomizedBinaryPathNode):
         )
 
     def compute_result(self, leaf_analysis: CallAnalysis) -> Tuple[CallAnalysis, bool]:
-        self._simplify()
         positive, negative = self.positive, self.negative
         exhausted = self._is_exhausted()
         if node_status(positive) == VerificationStatus.REFUTED or (
@@ -660,7 +670,7 @@ class ModelValueNode(WorstResultNode):
     def __init__(self, rand: random.Random, expr: z3.ExprRef, solver: z3.Solver):
         if not solver_is_sat(solver):
             debug("Solver unexpectedly unsat; solver state:", solver.sexpr())
-            raise CrosshairInternal("Unexpected unsat from solver")
+            raise CrossHairInternal("Unexpected unsat from solver")
 
         self.condition_value = solver.model().evaluate(expr, model_completion=True)
         self._stats_key = f"realize_{expr}" if z3.is_const(expr) else None
@@ -677,7 +687,6 @@ class ModelValueNode(WorstResultNode):
 
 def debug_path_tree(node, highlights, prefix="") -> List[str]:
     highlighted = node in highlights
-    node = node.simplify()
     highlighted |= node in highlights
     if isinstance(node, BinaryPathNode):
         if isinstance(node, WorstResultNode) and node.forced_path is not None:
@@ -710,6 +719,17 @@ def debug_path_tree(node, highlights, prefix="") -> List[str]:
             return [f"{prefix} -> {str(node)} {node.stats()}"]
 
 
+def make_default_solver() -> z3.Solver:
+    """Create a new solver with default settings."""
+    smt_tactic = z3.Tactic("smt")
+    solver = smt_tactic.solver()
+    solver.set("mbqi", True)
+    # turn off every randomization thing we can think of:
+    solver.set("random-seed", 42)
+    solver.set("smt.random-seed", 42)
+    return solver
+
+
 class StateSpace:
     """Holds various information about the SMT solver's current state."""
 
@@ -723,16 +743,12 @@ class StateSpace:
         model_check_timeout: float,
         search_root: RootNode,
     ):
-        smt_timeout = model_check_timeout * 1000 + 1
-        smt_tactic = z3.Tactic("smt")
-        if smt_timeout < 1 << 63:
-            smt_tactic = z3.TryFor(smt_tactic, int(smt_timeout))
-        self.solver = smt_tactic.solver()
-        self.solver.set(mbqi=True)
-        # turn off every randomization thing we can think of:
-        self.solver.set("random-seed", 42)
-        self.solver.set("smt.random-seed", 42)
-        # self.solver.set('randomize', False)
+        self.solver = make_default_solver()
+        if model_check_timeout < 1 << 63:
+            self.smt_timeout: Optional[int] = int(model_check_timeout * 1000 + 1)
+            self.solver.set(timeout=self.smt_timeout)
+        else:
+            self.smt_timeout = None
         self.choices_made: List[SearchTreeNode] = []
         self.status_cap: Optional[VerificationStatus] = None
         self.heaps: List[List[Tuple[z3.ExprRef, Type, object]]] = [[]]
@@ -747,17 +763,30 @@ class StateSpace:
         self._random = search_root._random
         _, _, self._search_position = search_root.choose(self)
         self._deferred_assumptions = []
-        search_root.pathing_oracle.pre_path_hook(search_root)
+        assert search_root.iteration is not None
+        search_root.iteration += 1
+        search_root.pathing_oracle.pre_path_hook(self)
 
-    def add(self, expr: z3.ExprRef) -> None:
+    def add(self, expr) -> None:
         with NoTracing():
+            if hasattr(expr, "var"):
+                expr = expr.var
+            elif not isinstance(expr, z3.ExprRef):
+                if type(expr) is bool:
+                    raise CrossHairInternal(
+                        "Attempted to assert a concrete boolean (look for unexpected realization)"
+                    )
+                raise CrossHairInternal(
+                    "Expected symbolic boolean, but supplied expression of type",
+                    name_of_type(type(expr)),
+                )
             # debug('Committed to ', expr)
             already_known = self._exprs_known.get(expr)
             if already_known is None:
                 self.solver.add(expr)
                 self._exprs_known[expr] = True
             elif already_known is not True:
-                raise CrosshairInternal
+                raise CrossHairInternal
 
     def rand(self) -> random.Random:
         return self._random
@@ -771,31 +800,42 @@ class StateSpace:
         return value  # type: ignore
 
     def stats_lookahead(self) -> Tuple[StateSpaceCounter, StateSpaceCounter]:
-        node = self._search_position.simplify()
-        if node.is_stem():
+        node = self._search_position
+        if isinstance(node, NodeStem):
             return (StateSpaceCounter(), StateSpaceCounter())
-        assert isinstance(
-            node, BinaryPathNode
-        ), f"node {node} {node.is_stem()} is not a binarypathnode"
+        assert isinstance(node, BinaryPathNode), f"node {node} is not a binarypathnode"
         return node.stats_lookahead()
 
+    def grow_into(self, node: _N) -> _N:
+        assert isinstance(self._search_position, NodeStem)
+        self._search_position.parent.replace_child(self._search_position, node)
+        node.iteration = self._root.iteration
+        self._search_position = node
+        return node
+
     def fork_parallel(self, false_probability: float, desc: str = "") -> bool:
-        if self._search_position.is_stem():
-            node: NodeLike = self._search_position.grow_into(
-                ParallelNode(self._random, false_probability, desc)
-            )
+        node = self._search_position
+        if isinstance(node, NodeStem):
+            node = self.grow_into(ParallelNode(self._random, false_probability, desc))
+            node.stacktail = self.gen_stack_descriptions()
             assert isinstance(node, ParallelNode)
             self._search_position = node
         else:
-            node = self._search_position.simplify()
-            assert isinstance(node, ParallelNode)
+            if not isinstance(node, ParallelNode):
+                self.raise_not_deterministic(
+                    node, "Wrong node type (expected ParallelNode)"
+                )
             node._false_probability = false_probability
         self.choices_made.append(node)
         ret, _, next_node = node.choose(self)
         self._search_position = next_node
         return ret
 
-    def is_possible(self, expr: z3.ExprRef) -> bool:
+    def is_possible(self, expr) -> bool:
+        with NoTracing():
+            if hasattr(expr, "var"):
+                expr = expr.var
+            debug("is possible?", expr)
         return solver_is_sat(self.solver, expr)
 
     def mark_all_parent_frames(self):
@@ -811,15 +851,7 @@ class StateSpace:
 
     def gen_stack_descriptions(self) -> Tuple[str, ...]:
         f: Any = _getframe().f_back.f_back  # type: ignore
-        # TODO: if we deprecate 3.7, we could try this instead of the above:
-        # frames = [f := f.f_back or f for _ in range(8)]
-        frames = []
-        external_frame_ids = self.external_frame_ids
-        for _ in range(8):
-            if id(f) in external_frame_ids:
-                break
-            frames.append(f)
-            f = f.f_back
+        frames = [f := f.f_back or f for _ in range(8)]
         # TODO: To help oracles, I'd like to add sub-line resolution via f.f_lasti;
         # however, in Python >= 3.11, the instruction pointer can shift between
         # PRECALL and CALL opcodes, triggering our nondeterminism check.
@@ -834,66 +866,45 @@ class StateSpace:
             )
             raise PathTimeout
 
+    @assert_tracing(False)
     def choose_possible(
         self, expr: z3.ExprRef, probability_true: Optional[float] = None
     ) -> bool:
         known_result = self._exprs_known.get(expr)
         if isinstance(known_result, bool):
             return known_result
-        if is_tracing():
-            raise CrosshairInternal
-        if self._search_position.is_stem():
-            # We only allow time outs at stems - that's because we don't want
-            # to think about how mutating an existing path branch would work:
-            self.check_timeout()
-            node = self._search_position.grow_into(
-                WorstResultNode(self._random, expr, self.solver)
-            )
-        else:
-            node = self._search_position.simplify()  # type: ignore
-            if not isinstance(node, WorstResultNode):
-                debug(" *** Begin Not Deterministic Debug *** ")
-                debug("  Traceback: ", test_stack())
-                debug("Decision expression:")
-                debug(f"  {expr}")
-                debug("Now found at incompatible node of type:")
-                debug(f"  {type(node)}")
-                debug(" *** End Not Deterministic Debug *** ")
-                raise NotDeterministic
-            if not z3.eq(node.expr, expr):
-                debug(" *** Begin Not Deterministic Debug *** ")
-                debug("  Traceback: ", test_stack())
-                debug("Decision expression changed from:")
-                debug(f"  {node.expr}")
-                debug("To:")
-                debug(f"  {expr}")
-                debug(" *** End Not Deterministic Debug *** ")
-                raise NotDeterministic
-
-        self._search_position = node
         # NOTE: format_stack() is more human readable, but it pulls source file contents,
         # so it is (1) slow, and (2) unstable when source code changes while we are checking.
         stacktail = self.gen_stack_descriptions()
-        assert isinstance(node, SearchTreeNode)
-        if not node.stacktail:
-            node.stacktail = stacktail
+        if isinstance(self._search_position, SearchTreeNode):
+            node = self._search_position
+            not_deterministic_reason = (
+                (
+                    (not isinstance(node, WorstResultNode))
+                    and f"Wrong node type (is {name_of_type(type(node))}, expected WorstResultNode)"
+                )
+                # TODO: Not clear whether we want this stack trace check.
+                # A stack change usually indicates a serious problem, but not 100% of the time.
+                # Keeping it would mean that we fail earlier.
+                # But also see https://github.com/HypothesisWorks/hypothesis/pull/4034#issuecomment-2606415404
+                # or (node.stacktail != stacktail and "Stack trace changed")
+                or (
+                    (hasattr(node, "expr") and (not z3.eq(node.expr, expr)))
+                    and "SMT expression changed"
+                )
+            )
+            if not_deterministic_reason:
+                self.raise_not_deterministic(
+                    node, not_deterministic_reason, expr=expr, stacktail=stacktail
+                )
         else:
-            if node.stacktail != stacktail:
-                debug(" *** Begin Not Deterministic Debug *** ")
-                debug("     First state: ")
-                debug(node.stacktail)
-                debug("     Current state: ")
-                debug(stacktail)
-                debug("     Decision points prior to this:")
-                for choice in self.choices_made:
-                    debug("      ", choice)
-                debug("     Stack Diff: ")
-                import difflib
-
-                debug("\n".join(difflib.context_diff(node.stacktail, stacktail)))
-                debug(" *** End Not Deterministic Debug *** ")
-                raise NotDeterministic
+            # We only allow time outs at stems - that's because we don't want
+            # to think about how mutating an existing path branch would work:
+            self.check_timeout()
+            node = self.grow_into(WorstResultNode(self._random, expr, self.solver))
+            node.stacktail = stacktail
 
+        self._search_position = node
         choose_true, chosen_probability, stem = node.choose(
             self, probability_true=probability_true
         )
@@ -909,32 +920,64 @@ class StateSpace:
         chosen_expr = expr if choose_true else z3Not(expr)
         if in_debug():
             debug(
-                "SMT chose:",
-                chosen_expr,
-                "(chance:",
-                chosen_probability,
-                ")",
+                f"SMT chose: {chosen_expr} (chance: {chosen_probability}) at",
+                ch_stack(),
             )
         z3Aassert(self.solver, chosen_expr)
         self._exprs_known[expr] = choose_true
         return choose_true
 
+    def raise_not_deterministic(
+        self,
+        node: NodeLike,
+        reason: str,
+        expr: Optional[z3.ExprRef] = None,
+        stacktail: Optional[Tuple[str, ...]] = None,
+        currently_handling: Optional[BaseException] = None,
+    ) -> NoReturn:
+        lines = ["*** Begin Not Deterministic Debug ***"]
+        if getattr(node, "iteration", None) is not None:
+            lines.append(f"Previous iteration: {node.iteration}")  # type: ignore
+        if hasattr(node, "expr"):
+            lines.append(f"Previous SMT expression: {node.expr}")  # type: ignore
+        if expr is not None:
+            lines.append(f"Current SMT expression: {expr}")
+        if not stacktail:
+            if currently_handling is not None:
+                stacktail = tuple(format_tb(currently_handling.__traceback__))
+            else:
+                stacktail = self.gen_stack_descriptions()
+        lines.append("Current stack tail:")
+        lines.extend(f"  {x}" for x in stacktail)
+        if hasattr(node, "stacktail"):
+            lines.append("Previous stack tail:")
+            lines.extend(f"  {x}" for x in node.stacktail)
+        lines.append(f"Reason: {reason}")
+        lines.append("*** End Not Deterministic Debug ***")
+        for line in lines:
+            print(line)
+        exc = NotDeterministic()
+        if currently_handling:
+            raise exc from currently_handling
+        else:
+            raise exc
+
     def find_model_value(self, expr: z3.ExprRef) -> Any:
         with NoTracing():
             while True:
-                if self._search_position.is_stem():
-                    self._search_position = self._search_position.grow_into(
+                if isinstance(self._search_position, NodeStem):
+                    self._search_position = self.grow_into(
                         ModelValueNode(self._random, expr, self.solver)
                     )
-                node = self._search_position.simplify()
+                node = self._search_position
                 if isinstance(node, SearchLeaf):
-                    raise CrosshairInternal(
+                    raise CrossHairInternal(
                         f"Cannot use symbolics; path is already terminated"
                     )
                 if not isinstance(node, ModelValueNode):
                     debug(" *** Begin Not Deterministic Debug *** ")
                     debug(f"Model value node expected; found {type(node)} instead.")
-                    debug("  Traceback: ", test_stack())
+                    debug("  Traceback: ", ch_stack())
                     debug(" *** End Not Deterministic Debug *** ")
                     raise NotDeterministic
                 (chosen, _, next_node) = node.choose(self, probability_true=1.0)
@@ -950,14 +993,14 @@ class StateSpace:
                     ):
                         self._already_logged.add(expr)
                         debug("SMT realized symbolic:", expr, "==", repr(ret))
-                        debug("Realized at", test_stack())
+                        debug("Realized at", ch_stack())
                     return ret
                 else:
                     self.solver.add(expr != node.condition_value)
 
     def find_model_value_for_function(self, expr: z3.ExprRef) -> object:
         if not solver_is_sat(self.solver):
-            raise CrosshairInternal("model unexpectedly became unsatisfiable")
+            raise CrossHairInternal("model unexpectedly became unsatisfiable")
         # TODO: this need to go into a tree node that returns UNKNOWN or worse
         # (because it just returns one example function; it's not covering the space)
 
@@ -989,7 +1032,7 @@ class StateSpace:
     ) -> object:
         with NoTracing():
             # TODO: needs more testing
-            for (curref, curtyp, curval) in self.heaps[snapshot]:
+            for curref, curtyp, curval in self.heaps[snapshot]:
 
                 # TODO: using unify() is almost certainly wrong; just because the types
                 # have some instances in common does not mean that `curval` actually
@@ -999,7 +1042,7 @@ class StateSpace:
                     continue
                 if self.smt_fork(curref == ref, probability_true=0.1):
                     debug(
-                        "HEAP key lookup ",
+                        "Heap key lookup ",
                         ref,
                         ": Found existing. ",
                         "type:",
@@ -1010,7 +1053,7 @@ class StateSpace:
                     return curval
             ret = proxy_generator(typ)
             debug(
-                "HEAP key lookup ",
+                "Heap key lookup ",
                 ref,
                 ": Created new. ",
                 "type:",
@@ -1026,6 +1069,51 @@ class StateSpace:
         self.next_uniq += 1
         return "_{:x}".format(self.next_uniq)
 
+    @assert_tracing(False)
+    def smt_fanout(
+        self,
+        exprs_and_results: Sequence[Tuple[z3.ExprRef, object]],
+        desc: str,
+        weights: Optional[Sequence[float]] = None,
+        none_of_the_above_weight: float = 0.0,
+    ):
+        """Performs a weighted binary search over the given SMT expressions."""
+        exprs = [e for (e, _) in exprs_and_results]
+        final_weights = [1.0] * len(exprs) if weights is None else weights
+        if CROSSHAIR_EXTRA_ASSERTS:
+            if len(final_weights) != len(exprs):
+                raise CrossHairInternal("inconsistent smt_fanout exprs and weights")
+            if not all(0 < w for w in final_weights):
+                raise CrossHairInternal("smt_fanout weights must be greater than zero")
+            if not self.is_possible(z3Or(*exprs)):
+                raise CrossHairInternal(
+                    "no smt_fanout option is possible: " + repr(exprs)
+                )
+            if self.is_possible(z3Not(z3Or(*exprs))):
+                raise CrossHairInternal(
+                    "smt_fanout options are not exhaustive: " + repr(exprs)
+                )
+
+        def attempt(start: int, end: int):
+            size = end - start
+            if size == 1:
+                return exprs_and_results[start][1]
+            mid = (start + end) // 2
+            left_exprs = exprs[start:mid]
+            left_weight = sum(final_weights[start:mid])
+            right_weight = sum(final_weights[mid:end])
+            if self.smt_fork(
+                z3Or(*left_exprs),
+                probability_true=left_weight / (left_weight + right_weight),
+                desc=f"{desc}_fan_size_{size}",
+            ):
+                return attempt(start, mid)
+            else:
+                return attempt(mid, end)
+
+        return attempt(0, len(exprs))
+
+    @assert_tracing(False)
     def smt_fork(
         self,
         expr: Optional[z3.ExprRef] = None,
@@ -1039,7 +1127,15 @@ class StateSpace:
     def defer_assumption(self, description: str, checker: Callable[[], bool]) -> None:
         self._deferred_assumptions.append((description, checker))
 
-    def detach_path(self) -> None:
+    def extend_timeouts(
+        self, constant_factor: float = 0.0, smt_multiple: Optional[float] = None
+    ) -> None:
+        self.execution_deadline += constant_factor
+        if self.smt_timeout is not None and smt_multiple is not None:
+            self.smt_timeout = int(self.smt_timeout * smt_multiple)
+            self.solver.set(timeout=self.smt_timeout)
+
+    def detach_path(self, currently_handling: Optional[BaseException] = None) -> None:
         """
         Mark the current path exhausted.
 
@@ -1052,36 +1148,62 @@ class StateSpace:
             if self.is_detached:
                 debug("Path is already detached")
                 return
-            else:
-                # Give ourselves a time extension for deferred assumptions and
-                # (likely) counterexample generation to follow.
-                self.execution_deadline += 2.0
+            # Give ourselves a time extension for deferred assumptions and
+            # (likely) counterexample generation to follow.
+            self.extend_timeouts(constant_factor=4.0, smt_multiple=2.0)
+            self.is_detached = True
+            if isinstance(
+                currently_handling,
+                (NotDeterministic, UnknownSatisfiability, PathTimeout),
+            ):
+                # These exceptions can happen at any time; we may not be at a stem node.
+                node = DetachedPathNode()
+                self.choices_made.append(node)
+                self._search_position = node.child
+                return
             for description, checker in self._deferred_assumptions:
                 with ResumedTracing():
                     check_ret = checker()
                 if not prefer_true(check_ret):
+                    node = self.grow_into(DetachedPathNode())
+                    self.choices_made.append(node)
+                    self._search_position = node.child
                     raise IgnoreAttempt("deferred assumption failed: " + description)
-            self.is_detached = True
-            assert self._search_position.is_stem()
-            node = self._search_position.grow_into(DeatchedPathNode())
-            assert node.child.is_stem()
+            if not isinstance(self._search_position, NodeStem):
+                # Nondeterminism detected
+                # We'll just overwrite the prior path with this one.
+                # (note that this might leave some stats in the parents
+                # that is no longer justified by the leaves?)
+                if isinstance(self._search_position, DetachedPathNode):
+                    return
+                previous_node = self._search_position
+                node = DetachedPathNode()
+                if self.choices_made:
+                    self.choices_made[-1].replace_child(previous_node, node)
+                self.choices_made.append(node)
+                self._search_position = node.child
+                self.raise_not_deterministic(
+                    previous_node,
+                    f"Expect to detach path at a stem node, not at this node: {previous_node}",
+                    currently_handling=currently_handling,
+                )
+            node = self.grow_into(DetachedPathNode())
             self.choices_made.append(node)
             self._search_position = node.child
             debug("Detached from search tree")
 
     def cap_result_at_unknown(self):
+        # TODO: this doesn't seem to work as intended.
+        # If any execution path is confirmed, the end result is sometimes confirmed as well.
         self.status_cap = VerificationStatus.UNKNOWN
 
     def bubble_status(
         self, analysis: CallAnalysis
     ) -> Tuple[Optional[CallAnalysis], bool]:
         # In some cases, we might ignore an attempt while not at a leaf.
-        if self._search_position.is_stem():
-            self._search_position = self._search_position.grow_into(
-                SearchLeaf(analysis)
-            )
+        if isinstance(self._search_position, NodeStem):
+            self._search_position = self.grow_into(SearchLeaf(analysis))
         else:
-            self._search_position = self._search_position.simplify()
             assert isinstance(self._search_position, SearchTreeNode)
             self._search_position.exhausted = True
             self._search_position.result = analysis
@@ -1090,11 +1212,12 @@ class StateSpace:
             return (analysis, True)
         for node in reversed(self.choices_made):
             node.update_result(analysis)
-        if in_debug():
-            for line in debug_path_tree(
-                self._root, set(self.choices_made + [self._search_position])
-            ):
-                debug(line)
+        if False:  # this is more noise than it's worth (usually)
+            if in_debug():
+                for line in debug_path_tree(
+                    self._root, set(self.choices_made + [self._search_position])
+                ):
+                    debug(line)
         # debug('Path summary:', self.choices_made)
         first = self.choices_made[0]
         analysis = first.get_result()