crosshair-tool 0.0.99__cp312-cp312-macosx_10_13_x86_64.whl

crosshair/opcode_intercept_test.py

@@ -0,0 +1,304 @@
+import math
+import sys
+from abc import ABCMeta
+from typing import List, Set
+
+import pytest
+
+from crosshair.core_and_libs import NoTracing, proxy_for_type, standalone_statespace
+from crosshair.libimpl.builtinslib import (
+    ModelingDirector,
+    RealBasedSymbolicFloat,
+    SymbolicBool,
+    SymbolicInt,
+    SymbolicType,
+)
+from crosshair.statespace import POST_FAIL
+from crosshair.test_util import check_states
+from crosshair.tracers import ResumedTracing
+from crosshair.z3util import z3And
+
+
+def test_dict_index():
+    a = {"two": 2, "four": 4, "six": 6}
+
+    def numstr(x: str) -> int:
+        """
+        post: _ != 4
+        raises: KeyError
+        """
+        return a[x]
+
+    check_states(numstr, POST_FAIL)
+
+
+def test_dict_index_without_realization(space):
+    class WithMeta(metaclass=ABCMeta):
+        pass
+
+    space.extra(ModelingDirector).global_representations[float] = RealBasedSymbolicFloat
+    a = {
+        -1: WithMeta,
+        #   ^ tests regression: isinstance(WithMeta(), type) but type(WithMeta) != type
+        0: list,
+        1.0: 10.0,
+        2: 20,
+        3: 30,
+        4: 40,
+        ("complex", "key"): 50,
+        6: math.inf,
+        7: math.inf,
+    }
+    int_key = proxy_for_type(int, "int_key")
+    int_key2 = proxy_for_type(int, "int_key2")
+    int_key3 = proxy_for_type(int, "int_key3")
+    float_key = RealBasedSymbolicFloat("float_key")
+    float_key2 = RealBasedSymbolicFloat("float_key2")
+    with ResumedTracing():
+        # Try some concrete values out first:
+        assert a[("complex", "key")] == 50
+        assert a[6] == float("inf")
+        try:
+            a[42]
+            assert False, "Expected KeyError for missing key 42"
+        except KeyError:
+            pass
+
+        space.add(2 <= int_key)
+        space.add(int_key <= 4)
+        int_result = a[int_key]
+        assert space.is_possible(int_result == 20)
+        assert space.is_possible(int_result == 40)
+        assert not space.is_possible(int_result == 10)
+        space.add(float_key == 1.0)
+        float_result = a[float_key]
+        assert space.is_possible(float_result == 10.0)
+        assert not space.is_possible(float_result == 42.0)
+        space.add(float_key2 == 2.0)
+        float_result2 = a[float_key2]
+        assert space.is_possible(float_result2 == 20)
+        space.add(int_key2 == 0)
+        int_result2 = a[int_key2]
+        assert int_result2 == list
+        space.add(any([int_key3 == 6, int_key3 == 7]))
+        inf_result = a[int_key3]
+        assert inf_result is math.inf
+    assert isinstance(int_result, SymbolicInt)
+    assert isinstance(float_result, RealBasedSymbolicFloat)
+    assert isinstance(float_result2, SymbolicInt)
+    assert isinstance(int_result2, SymbolicType)
+
+
+def test_dict_symbolic_index_miss(space):
+    a = {6: 60, 7: 70}
+    x = proxy_for_type(int, "x")
+    with ResumedTracing():
+        space.add(x <= 4)
+        with pytest.raises(KeyError):
+            result = a[x]
+
+
+def test_concrete_list_with_symbolic_index_simple(space):
+    haystack = [False] * 13 + [True] + [False] * 11
+
+    idx = proxy_for_type(int, "idx")
+    with ResumedTracing():
+        space.add(0 <= idx)
+        space.add(idx < len(haystack))
+        ret = haystack[idx]
+    assert isinstance(ret, SymbolicBool)
+    with ResumedTracing():
+        assert space.is_possible(idx == 13)
+        assert space.is_possible(idx == 12)
+        space.add(ret)
+        assert not space.is_possible(idx == 12)
+
+
+def test_concrete_list_with_symbolic_index_unhashable_values(space):
+    o1 = dict()
+    options = [o1, o1, o1]
+    idx = proxy_for_type(int, "idx")
+    with ResumedTracing():
+        space.add(0 <= idx)
+        space.add(idx < 3)
+        ret = options[idx]
+        assert ret is o1
+        assert space.is_possible(idx == 0)
+        assert space.is_possible(idx == 2)
+
+
+def test_dict_key_containment():
+    abc = {"two": 2, "four": 4, "six": 6}
+
+    def numstr(x: str) -> bool:
+        """
+        post: _
+        """
+        return x not in abc
+
+    check_states(numstr, POST_FAIL)
+
+
+def test_dict_comprehension_basic():
+    with standalone_statespace as space:
+        with NoTracing():
+            x = proxy_for_type(int, "x")
+        space.add(x >= 40)
+        space.add(x < 50)
+        d = {k: v for k, v in ((35, 3), (x, 4))}
+        with NoTracing():
+            assert type(d) is not dict
+        for k in d:
+            if k == 35:
+                continue
+            with NoTracing():
+                assert type(k) is not int
+            assert space.is_possible(k == 43)
+            assert space.is_possible(k == 48)
+
+
+def test_dict_comprehension_traces_during_custom_hash():
+    class FancyCompare:
+        def __init__(self, mystr: str):
+            self.mystr = mystr
+
+        def __eq__(self, other):
+            return (
+                isinstance(other, FancyCompare)
+                and "".join([self.mystr, ""]) == other.mystr
+            )
+
+        def __hash__(self):
+            return hash(self.mystr)
+
+    with standalone_statespace as space:
+        with NoTracing():
+            mystr = proxy_for_type(str, "mystr")
+        # NOTE: If tracing isn't on when we call FancyCompare.__eq__, we'll get an
+        # exception here:
+        d = {x: 42 for x in [FancyCompare(mystr), FancyCompare(mystr)]}
+        # There is only one item:
+        assert len(d) == 1
+        # TODO: In theory, we shouldn't need to realize the string here (but we are):
+        # assert space.is_possible(mystr.__len__() == 0)
+        # assert space.is_possible(mystr.__len__() == 1)
+
+
+def test_dict_comprehension_e2e():
+    def f(ls: List[int]) -> dict:
+        """
+        post: 4321 not in __return__
+        """
+        return {i: i for i in ls}
+
+    check_states(f, POST_FAIL)
+
+
+@pytest.mark.skipif(
+    sys.version_info >= (3, 13), reason="Negation opcode changed; TODO: fix!"
+)
+def test_not_operator_on_bool():
+    with standalone_statespace as space:
+        with NoTracing():
+            boolval = proxy_for_type(bool, "boolval")
+        inverseval = not boolval
+        assert space.is_possible(inverseval)
+        with NoTracing():
+            assert type(inverseval) is not bool
+            assert not space.is_possible(z3And(boolval.var, inverseval.var))
+
+
+def test_not_operator_on_non_bool():
+    with standalone_statespace as space:
+        with NoTracing():
+            intlist = proxy_for_type(List[int], "intlist")
+        space.add(intlist.__len__() == 0)
+        notList = not intlist
+        with NoTracing():
+            assert notList
+
+
+def test_set_comprehension_basic():
+    with standalone_statespace as space:
+        with NoTracing():
+            x = proxy_for_type(int, "x")
+        space.add(x >= 40)
+        space.add(x < 50)
+        result_set = {k for k in (35, x)}
+        with NoTracing():
+            assert type(result_set) is not set
+        for k in result_set:
+            if k == 35:
+                continue
+            with NoTracing():
+                assert type(k) is not int
+            assert space.is_possible(k == 43)
+            assert space.is_possible(k == 48)
+
+
+def test_set_comprehension_e2e():
+    def f(s: Set[int]) -> Set:
+        """
+        post: 4321 not in __return__
+        """
+        return {i for i in s}
+
+    check_states(f, POST_FAIL)
+
+
+def test_trace_disabling_at_jump_targets(space):
+    # This replicates a corruption of the interpreter stack in 3.12
+    # under a specific bytecode layout.
+    #
+    # The origial issue was caused by neglecting to keep sys.monitor probes
+    # alive (for post-op callbacks) that could be jumped to from other
+    # locations.
+    _global_type_lookupx = {
+        1: 1,
+        bool: 2,
+        3: 3,
+    }
+    with ResumedTracing():
+        _ = {
+            k: v
+            for k, v in _global_type_lookupx.items()  # <- a new line has to be here (yes, the generated bytecode differs!)
+            if k == bool  # The iteration filter needs to alternate
+        }
+
+
+# TODO: we could implement identity comparisons on 3.8 by intercepting COMPARE_OP
+@pytest.mark.skipif(sys.version_info < (3, 9), reason="IS_OP is new in Python 3.9")
+def test_identity_operator_on_booleans():
+    with standalone_statespace as space:
+        with NoTracing():
+            b1 = proxy_for_type(bool, "b1")
+        space.add(b1)
+        assert b1 is True
+
+
+@pytest.mark.skipif(sys.version_info < (3, 9), reason="IS_OP is new in Python 3.9")
+def test_identity_operator_does_not_realize_on_differing_types():
+    with standalone_statespace as space:
+        with NoTracing():
+            b1 = proxy_for_type(bool, "b1")
+            choices_made_at_start = len(space.choices_made)
+        space.add(b1)
+        fourty_two = 42  # assignment just to avoid lint errors
+        b1 is fourty_two
+        assert len(space.choices_made) == choices_made_at_start
+
+
+class IExplodeOnRepr:
+    def __repr__(self):
+        raise ValueError("boom")
+
+
+def test_postop_callback_skipped_on_exception_handler_jump(space):
+    with ResumedTracing():
+        elements = IExplodeOnRepr()
+        try:
+            ret = f"these are them: {elements!r}"
+        except ValueError:  # pragma: no cover
+            ret = None
+        # need to do something(anything) with elements so that it's on the stack:
+        type(elements)

crosshair/options.py

@@ -0,0 +1,218 @@
+import collections
+import enum
+import math
+import re
+import sys
+from dataclasses import dataclass, replace
+from typing import Any, Callable, Mapping, Optional, Sequence, Tuple, get_type_hints
+
+
+class AnalysisKind(enum.Enum):
+    asserts = "asserts"
+    PEP316 = "PEP316"
+    icontract = "icontract"
+    deal = "deal"
+
+    def __repr__(self):
+        return f"AnalysisKind.{self.name}"
+
+
+def _parse_analysis_kind(argstr: str) -> Sequence[AnalysisKind]:
+    try:
+        return [AnalysisKind[part.strip()] for part in argstr.split(",")]
+    except KeyError:
+        raise ValueError
+
+
+def _parse_bool(argstr: str) -> Optional[bool]:
+    match = re.fullmatch(r"(1|true|y(?:es)?)|(0|false|no?)", argstr, re.I)
+    if match:
+        yes, _no = match.groups()
+        return bool(yes)
+    return None
+
+
+@dataclass
+class AnalysisOptionSet:
+    """
+    Encodes some set of partially-specified options.
+
+    This class is used while parsing options from various places.
+    It is very similar to `AnalysisOptions` (which is used during execution) but allows
+    None values everywhere so that options can correctly override each other.
+    """
+
+    analysis_kind: Optional[Sequence[AnalysisKind]] = None
+    enabled: Optional[bool] = None
+    specs_complete: Optional[bool] = None
+    per_condition_timeout: Optional[float] = None
+    per_path_timeout: Optional[float] = None
+    max_iterations: Optional[int] = None
+    report_all: Optional[bool] = None
+    report_verbose: Optional[bool] = None
+    timeout: Optional[float] = None
+    max_uninteresting_iterations: Optional[int] = None
+
+    # TODO: move stats out of options
+    stats: Optional[collections.Counter] = None
+
+    # These options are the ones allowed in directives
+    directive_fields = frozenset(
+        {
+            "enabled",
+            "analysis_kind",
+            "specs_complete",
+            "max_iterations",
+            "per_condition_timeout",
+            "per_path_timeout",
+            "max_uninteresting_iterations",
+        }
+    )
+
+    def overlay(self, overrides: "AnalysisOptionSet") -> "AnalysisOptionSet":
+        kw = {k: v for (k, v) in overrides.__dict__.items() if v is not None}
+        return replace(self, **kw)
+
+    @classmethod
+    def parser_for(cls, field: str) -> Optional[Callable[[str], Any]]:
+        if field == "analysis_kind":
+            return _parse_analysis_kind
+        hints = get_type_hints(AnalysisOptions)
+        if field not in hints:
+            return None
+        ctor = hints[field]
+        if ctor is bool:
+            return _parse_bool
+        return ctor
+
+    @classmethod
+    def parse_field(cls, field: str, strval: str) -> Any:
+        parser = cls.parser_for(field)
+        if parser is None:
+            return None
+        try:
+            return parser(strval)
+        except ValueError:
+            return None
+
+
+def option_set_from_dict(source: Mapping[str, object]) -> AnalysisOptionSet:
+    options = AnalysisOptionSet()
+    for optname in (
+        "analysis_kind",
+        "specs_complete",
+        "per_path_timeout",
+        "per_condition_timeout",
+        "max_uninteresting_iterations",
+        "report_all",
+        "report_verbose",
+    ):
+        arg_val = source.get(optname, None)
+        if arg_val is not None:
+            setattr(options, optname, arg_val)
+    return options
+
+
+@dataclass
+class AnalysisOptions:
+    """Encodes the options for use while running CrossHair."""
+
+    analysis_kind: Sequence[AnalysisKind]
+    enabled: bool
+    specs_complete: bool
+    per_condition_timeout: float
+    max_iterations: int
+    report_all: bool
+    report_verbose: bool
+    timeout: float
+    per_path_timeout: float
+    max_uninteresting_iterations: int
+
+    # Transient members (not user-configurable):
+    deadline: float = float("NaN")
+    stats: Optional[collections.Counter] = None
+
+    def get_max_uninteresting_iterations(self):
+        max_uninteresting_iterations = self.max_uninteresting_iterations
+        if max_uninteresting_iterations == sys.maxsize and (
+            not math.isfinite(self.per_condition_timeout)
+        ):
+            return 5
+        elif max_uninteresting_iterations == 0:
+            return sys.maxsize
+        else:
+            return max_uninteresting_iterations
+
+    def get_per_path_timeout(self):
+        if math.isnan(self.per_path_timeout):
+            if math.isfinite(self.per_condition_timeout):
+                if self.per_condition_timeout > 1.0:
+                    return self.per_condition_timeout**0.5
+                else:
+                    return self.per_condition_timeout
+            max_uninteresting_iterations = self.get_max_uninteresting_iterations()
+            if max_uninteresting_iterations < sys.maxsize:
+                return max(max_uninteresting_iterations, 1)
+            return float("inf")
+        else:
+            return self.per_path_timeout
+
+    def overlay(
+        self, overrides: Optional[AnalysisOptionSet] = None, **kw
+    ) -> "AnalysisOptions":
+        if overrides is not None:
+            assert not kw
+            kw = overrides.__dict__
+        kw = {k: v for (k, v) in kw.items() if v is not None}
+        ret = replace(self, **kw)
+        assert type(ret) is AnalysisOptions
+        return ret
+
+    def split_limits(
+        self, priority: float
+    ) -> Tuple["AnalysisOptions", "AnalysisOptions"]:
+        """
+        Divide resource allotments into two.
+
+        Namely, the resource allotments (timeouts, iteration caps) are split
+        into allotments for two stages of analysis.
+
+        pre: 0.0 <= priority <= 1.0
+        post: _[0].max_iterations + _[1].max_iterations == self.max_iterations
+        """
+        options1 = replace(
+            self,
+            per_condition_timeout=self.per_condition_timeout * priority,
+            per_path_timeout=self.per_path_timeout * priority,
+            max_iterations=round(self.max_iterations * priority),
+        )
+        inv_priority = 1.0 - priority
+        options2 = replace(
+            self,
+            per_condition_timeout=self.per_condition_timeout * inv_priority,
+            per_path_timeout=self.per_path_timeout * inv_priority,
+            max_iterations=self.max_iterations - options1.max_iterations,
+        )
+        return (options1, options2)
+
+    def incr(self, key: str):
+        if self.stats is not None:
+            self.stats[key] += 1
+
+
+DEFAULT_OPTIONS = AnalysisOptions(
+    analysis_kind=(
+        AnalysisKind.PEP316,
+        AnalysisKind.icontract,
+        AnalysisKind.deal,
+    ),
+    enabled=True,
+    specs_complete=False,
+    per_condition_timeout=float("inf"),
+    max_iterations=sys.maxsize,
+    report_all=False,
+    report_verbose=True,
+    timeout=float("inf"),
+    per_path_timeout=float("NaN"),
+    max_uninteresting_iterations=sys.maxsize,
+)

crosshair/options_test.py

@@ -0,0 +1,10 @@
+from crosshair.options import DEFAULT_OPTIONS
+
+
+def test_AnalysisOptions_split_limits() -> None:
+    options = DEFAULT_OPTIONS.overlay(per_path_timeout=10.0, max_iterations=16)
+    part1, part2 = options.split_limits(0.1)
+    assert part1.per_path_timeout == 1.0
+    assert part2.per_path_timeout == 9.0
+    assert part1.max_iterations == 2
+    assert part2.max_iterations == 14

crosshair/patch_equivalence_test.py

@@ -0,0 +1,75 @@
+import copy
+import itertools
+import operator
+import re
+import sys
+from dataclasses import dataclass
+from typing import Callable, List, Mapping, Optional, Sequence, Tuple
+
+import pytest  # type: ignore
+
+from crosshair.core import _PATCH_REGISTRATIONS
+from crosshair.core_and_libs import standalone_statespace
+from crosshair.test_util import ExecutionResult, summarize_execution
+from crosshair.util import ch_stack, debug
+
+"""
+Tests that the builtin and standard library patches behave like their
+counterparts, for native python input values.
+Equivalence under symbolic inputs is tested in "_ch_test.py" files.
+"""
+
+
+possible_args = [
+    (),
+    (0,),
+    (None,),
+    ("a",),
+    ("ab", "b"),  # str endwith, index
+    (b"a",),
+    (b"ab", "little"),  # int.from_bytes
+    ("ǔ", "latin-1", "replace"),  # encode, bytes constructor
+    (b"ab", b"b"),  # bytes endwith, index
+    (b"x", [b"a", b"b"]),  # bytes join
+    (bytearray(b"x"), [b"a", bytearray(b"b")]),  # mixed bytearray join
+    ([2, 1],),  # min, max
+    (1, 2),
+    (int, object),  # issubclass
+    (int, (str, (tuple, list))),  # wacky multiply-nested issubclass checking
+    (int, 42),  # issubclass error
+    (42, int),  # isinstance
+    (re.compile("(ab|a|b)"), r"\n", ""),  # re methods
+    (bool, [1, 1, 0]),  # itertools.takewhile and friends
+    (operator.add, [1, 0], [1, 1]),  # multi-iterable map
+    ([(1, 2), (3, 4)]),  # key-value pairs
+    ([(1, 2), ([], 4)]),  # key-value pairs w/ unhashable key
+]
+
+untested_patches = {
+    itertools.groupby,  # the return value has nested iterators that break comparisons
+}
+
+comparisons: List[Tuple[Callable, Callable]] = []
+for native_fn, patched_fn in _PATCH_REGISTRATIONS.items():
+    if native_fn in untested_patches:
+        continue
+    patch_name = native_fn.__name__
+    comparisons.append(
+        pytest.param(native_fn, patched_fn, id=patch_name)  # type: ignore
+    )
+
+
+@pytest.mark.parametrize("native_fn,patched_fn", comparisons)
+@pytest.mark.parametrize(
+    "args", possible_args, ids=lambda t: re.sub(r"[\W_]", "_", str(t))
+)
+def test_patch(native_fn: Callable, patched_fn: Callable, args: Sequence[object]):
+    debug("Patch test:", native_fn, patched_fn)
+    debug("Args:", args)
+    args2 = copy.deepcopy(args)
+    native_result = summarize_execution(native_fn, args, {}, detach_path=False)
+    debug("Native result:  ", native_result)
+    with standalone_statespace:
+        patched_result = summarize_execution(patched_fn, args2, {}, detach_path=False)
+    debug("Patched result: ", patched_result)
+    assert native_result == patched_result