crosshair-tool 0.0.56__cp39-cp39-macosx_11_0_arm64.whl → 0.0.100__cp39-cp39-macosx_11_0_arm64.whl

crosshair/diff_behavior.py

@@ -1,13 +1,15 @@
 import copy
 import dataclasses
 import dis
+import enum
 import inspect
 import sys
 import time
-from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
+from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
 
+from crosshair import IgnoreAttempt
 from crosshair.condition_parser import condition_parser
-from crosshair.core import ExceptionFilter, Patched, deep_realize, gen_args, realize
+from crosshair.core import ExceptionFilter, Patched, deep_realize, gen_args
 from crosshair.fnutil import FunctionInfo
 from crosshair.options import AnalysisOptions
 from crosshair.statespace import (
@@ -17,14 +19,22 @@ from crosshair.statespace import (
     StateSpaceContext,
     VerificationStatus,
 )
+from crosshair.test_util import flexible_equal
 from crosshair.tracers import (
     COMPOSITE_TRACER,
     CoverageResult,
     CoverageTracingModule,
     NoTracing,
     PushedModule,
+    ResumedTracing,
 )
-from crosshair.util import IgnoreAttempt, UnexploredPath, debug
+from crosshair.util import CrosshairUnsupported, IgnoreAttempt, UnexploredPath, debug
+
+
+class ExceptionEquivalenceType(enum.Enum):
+    ALL = "ALL"
+    SAME_TYPE = "SAME_TYPE"
+    TYPE_AND_MESSAGE = "TYPE_AND_MESSAGE"
 
 
 @dataclasses.dataclass
@@ -65,16 +75,16 @@ class Result:
 
 def describe_behavior(
     fn: Callable, args: inspect.BoundArguments
-) -> Tuple[object, Optional[str]]:
+) -> Tuple[Any, Optional[BaseException]]:
     with ExceptionFilter() as efilter:
         ret = fn(*args.args, **args.kwargs)
         return (ret, None)
     if efilter.user_exc is not None:
         exc = efilter.user_exc[0]
         debug("user-level exception found", repr(exc), *efilter.user_exc[1])
-        return (None, repr(exc))
+        return (None, exc)
     if efilter.ignore:
-        return (None, "IgnoreAttempt")
+        return (None, IgnoreAttempt())
     assert False
 
 
@@ -118,21 +128,30 @@ def diff_scorer(
 
 
 def diff_behavior(
-    ctxfn1: FunctionInfo, ctxfn2: FunctionInfo, options: AnalysisOptions
+    ctxfn1: FunctionInfo,
+    ctxfn2: FunctionInfo,
+    options: AnalysisOptions,
+    exception_equivalence: ExceptionEquivalenceType = ExceptionEquivalenceType.TYPE_AND_MESSAGE,
 ) -> Union[str, List[BehaviorDiff]]:
     fn1, sig1 = ctxfn1.callable()
     fn2, sig2 = ctxfn2.callable()
     debug("Resolved signature:", sig1)
     all_diffs: List[BehaviorDiff] = []
     half1, half2 = options.split_limits(0.5)
-    with condition_parser(options.analysis_kind), Patched(), COMPOSITE_TRACER:
+    with condition_parser(
+        options.analysis_kind
+    ), Patched(), COMPOSITE_TRACER, NoTracing():
         # We attempt both orderings of functions. This helps by:
         # (1) avoiding code path explosions in one of the functions
         # (2) using both signatures (in case they differ)
-        all_diffs.extend(diff_behavior_with_signature(fn1, fn2, sig1, half1))
+        all_diffs.extend(
+            diff_behavior_with_signature(fn1, fn2, sig1, half1, exception_equivalence)
+        )
         all_diffs.extend(
             diff.reverse()
-            for diff in diff_behavior_with_signature(fn2, fn1, sig2, half2)
+            for diff in diff_behavior_with_signature(
+                fn2, fn1, sig2, half2, exception_equivalence
+            )
         )
     debug("diff candidates:", all_diffs)
     # greedily pick results:
@@ -156,7 +175,11 @@ def diff_behavior(
 
 
 def diff_behavior_with_signature(
-    fn1: Callable, fn2: Callable, sig: inspect.Signature, options: AnalysisOptions
+    fn1: Callable,
+    fn2: Callable,
+    sig: inspect.Signature,
+    options: AnalysisOptions,
+    exception_equivalence: ExceptionEquivalenceType,
 ) -> Iterable[BehaviorDiff]:
     search_root = RootNode()
     condition_start = time.monotonic()
@@ -180,7 +203,10 @@ def diff_behavior_with_signature(
         with StateSpaceContext(space):
             output = None
             try:
-                (verification_status, output) = run_iteration(fn1, fn2, sig, space)
+                with ResumedTracing():
+                    (verification_status, output) = run_iteration(
+                        fn1, fn2, sig, space, exception_equivalence
+                    )
             except IgnoreAttempt:
                 verification_status = None
             except UnexploredPath:
@@ -209,8 +235,30 @@ def diff_behavior_with_signature(
                     break
 
 
+def check_exception_equivalence(
+    exception_equivalence_type: ExceptionEquivalenceType,
+    exc1: Optional[BaseException],
+    exc2: Optional[BaseException],
+) -> bool:
+    if exc1 is not None and exc2 is not None:
+        if exception_equivalence_type == ExceptionEquivalenceType.ALL:
+            return True
+        elif exception_equivalence_type == ExceptionEquivalenceType.SAME_TYPE:
+            return type(exc1) == type(exc2)
+        elif exception_equivalence_type == ExceptionEquivalenceType.TYPE_AND_MESSAGE:
+            return repr(exc1) == repr(exc2)
+        else:
+            raise CrosshairUnsupported("Invalid exception_equivalence type")
+    else:
+        return (exc1 is None) and (exc2 is None)
+
+
 def run_iteration(
-    fn1: Callable, fn2: Callable, sig: inspect.Signature, space: StateSpace
+    fn1: Callable,
+    fn2: Callable,
+    sig: inspect.Signature,
+    space: StateSpace,
+    exception_equivalence: ExceptionEquivalenceType,
 ) -> Tuple[Optional[VerificationStatus], Optional[BehaviorDiff]]:
     with NoTracing():
         original_args = gen_args(sig)
@@ -220,12 +268,19 @@ def run_iteration(
     with NoTracing():
         coverage_manager = CoverageTracingModule(fn1, fn2)
     with ExceptionFilter() as efilter, PushedModule(coverage_manager):
-        result1 = describe_behavior(fn1, args1)
-        result2 = describe_behavior(fn2, args2)
-        space.detach_path()
-        if result1 == result2 and args1 == args2:
+        return1, exc1 = describe_behavior(fn1, args1)
+        return2, exc2 = describe_behavior(fn2, args2)
+        if (
+            flexible_equal(return1, return2)
+            and flexible_equal(args1.arguments, args2.arguments)
+            and check_exception_equivalence(exception_equivalence, exc1, exc2)
+        ):
+            # Functions are equivalent if both have the same result,
+            # and deemed to have the same kind of error.
+            space.detach_path()
             debug("Functions equivalent")
             return (VerificationStatus.CONFIRMED, None)
+        space.detach_path()
         debug("Functions differ")
         realized_args = {
             k: repr(deep_realize(v)) for (k, v) in original_args.arguments.items()
@@ -239,13 +294,13 @@ def run_iteration(
         diff = BehaviorDiff(
             realized_args,
             Result(
-                repr(deep_realize(result1[0])),
-                realize(result1[1]),
+                repr(deep_realize(return1)),
+                repr(deep_realize(exc1)) if exc1 is not None else None,
                 post_execution_args1,
             ),
             Result(
-                repr(deep_realize(result2[0])),
-                realize(result2[1]),
+                repr(deep_realize(return2)),
+                repr(deep_realize(exc2)) if exc2 is not None else None,
                 post_execution_args2,
             ),
             coverage_manager.get_results(fn1),

crosshair/diff_behavior_test.py

@@ -1,11 +1,13 @@
-import sys
-import unittest
 from typing import Callable, List, Optional
 
-from crosshair.diff_behavior import BehaviorDiff, diff_behavior
+from crosshair.diff_behavior import (
+    BehaviorDiff,
+    ExceptionEquivalenceType,
+    diff_behavior,
+)
 from crosshair.fnutil import FunctionInfo, walk_qualname
 from crosshair.options import DEFAULT_OPTIONS
-from crosshair.util import IgnoreAttempt, debug, set_debug
+from crosshair.util import debug, set_debug
 
 
 def _foo1(x: int) -> int:
@@ -50,7 +52,34 @@ class Derived(Base):
         return 11
 
 
-class BehaviorDiffTest(unittest.TestCase):
+def _sum_list_original(int_list):
+    count = 0
+    for i in int_list:
+        count += i
+    return count
+
+
+def _sum_list_rewrite(int_list):
+    count = 0
+    for i in range(len(int_list)):
+        count += int_list[i]
+    return count
+
+
+def _sum_list_rewrite_2(int_list):
+    class CustomException(Exception):
+        pass
+
+    try:
+        count = 0
+        for i in range(len(int_list)):
+            count += int_list[i]
+    except:  # noqa E722
+        raise CustomException()
+    return count
+
+
+class TestBehaviorDiff:
     def test_diff_method(self):
         diffs = diff_behavior(
             walk_qualname(Base, "foo"),
@@ -58,10 +87,9 @@ class BehaviorDiffTest(unittest.TestCase):
             DEFAULT_OPTIONS.overlay(max_iterations=10),
         )
         assert isinstance(diffs, list)
-        self.assertEqual(
-            [(d.result1.return_repr, d.result2.return_repr) for d in diffs],
-            [("10", "11")],
-        )
+        assert [(d.result1.return_repr, d.result2.return_repr) for d in diffs] == [
+            ("10", "11")
+        ]
 
     def test_diff_staticmethod(self):
         diffs = diff_behavior(
@@ -69,20 +97,20 @@ class BehaviorDiffTest(unittest.TestCase):
             foo2,
             DEFAULT_OPTIONS.overlay(max_iterations=10),
         )
-        self.assertEqual(diffs, [])
+        assert diffs == []
 
     def test_diff_behavior_same(self) -> None:
         diffs = diff_behavior(foo1, foo2, DEFAULT_OPTIONS.overlay(max_iterations=10))
-        self.assertEqual(diffs, [])
+        assert diffs == []
 
     def test_diff_behavior_different(self) -> None:
         diffs = diff_behavior(foo1, foo3, DEFAULT_OPTIONS.overlay(max_iterations=10))
-        self.assertEqual(len(diffs), 1)
+        assert len(diffs) == 1
         diff = diffs[0]
         assert isinstance(diff, BehaviorDiff)
-        self.assertGreater(int(diff.args["x"]), 1000)
-        self.assertEqual(diff.result1.return_repr, "100")
-        self.assertEqual(diff.result2.return_repr, "1000")
+        assert int(diff.args["x"]) > 1000
+        assert diff.result1.return_repr == "100"
+        assert diff.result2.return_repr == "1000"
 
     def test_diff_behavior_mutation(self) -> None:
         def cut_out_item1(a: List[int], i: int):
@@ -99,10 +127,10 @@ class BehaviorDiffTest(unittest.TestCase):
             opts,
         )
         assert not isinstance(diffs, str)
-        self.assertEqual(len(diffs), 1)
+        assert len(diffs) == 1
         diff = diffs[0]
-        self.assertGreater(len(diff.args["a"]), 1)
-        self.assertEqual(diff.args["i"], "-1")
+        assert len(diff.args["a"]) > 1
+        assert diff.args["i"] == "-1"
 
     def test_example_coverage(self) -> None:
         # Try to get examples that highlist the differences in the code.
@@ -128,7 +156,7 @@ class BehaviorDiffTest(unittest.TestCase):
         debug("diffs=", diffs)
         assert not isinstance(diffs, str)
         return_vals = set((d.result1.return_repr, d.result2.return_repr) for d in diffs)
-        self.assertEqual(return_vals, {("False", "None"), ("False", "True")})
+        assert return_vals == {("False", "None"), ("False", "True")}
 
 
 def test_diff_behavior_lambda() -> None:
@@ -146,7 +174,88 @@ def test_diff_behavior_lambda() -> None:
     assert diffs == []
 
 
-if __name__ == "__main__":
-    if ("-v" in sys.argv) or ("--verbose" in sys.argv):
-        set_debug(True)
-    unittest.main()
+def test_diffbehavior_exceptions_default() -> None:
+    """
+    Default behavior of `diffbehavior` - treating exceptions as different.
+    """
+
+    diffs = diff_behavior(
+        FunctionInfo.from_fn(_sum_list_original),
+        FunctionInfo.from_fn(_sum_list_rewrite),
+        DEFAULT_OPTIONS,
+    )
+    debug("diffs=", diffs)
+    assert len(diffs) == 1  # finds a counter-example
+    assert isinstance(diffs[0], BehaviorDiff)
+    assert diffs[0].result1
+    assert isinstance(diffs[0].result1.error, str)
+    assert isinstance(diffs[0].result2.error, str)
+    assert diffs[0].result1.error.startswith("TypeError")
+    assert diffs[0].result2.error.startswith("TypeError")
+    assert (
+        diffs[0].result1.error != diffs[0].result2.error
+    )  # Both code-blocks raise a different type error
+
+
+def test_diffbehavior_exceptions_same_type() -> None:
+    """
+    Treat exceptions of the same type as equivalent.
+    """
+
+    diffs = diff_behavior(
+        FunctionInfo.from_fn(_sum_list_original),
+        FunctionInfo.from_fn(_sum_list_rewrite),
+        DEFAULT_OPTIONS,
+        exception_equivalence=ExceptionEquivalenceType.SAME_TYPE,
+    )
+    debug("diffs=", diffs)
+    assert len(diffs) == 0  # No-counter example, because all TypeErrors are equal
+
+
+def test_diffbehavior_exceptions_all() -> None:
+    """
+    Treat exceptions of all types as equivalent.
+    """
+
+    diffs = diff_behavior(
+        FunctionInfo.from_fn(_sum_list_original),
+        FunctionInfo.from_fn(_sum_list_rewrite_2),
+        DEFAULT_OPTIONS,
+        exception_equivalence=ExceptionEquivalenceType.ALL,
+    )
+    debug("diffs=", diffs)
+    assert len(diffs) == 0  # No-counter example, because all TypeErrors are equal
+
+
+def test_diffbehavior_exceptions_same_type_different() -> None:
+    """
+    Find a counter-example when raising different exception types.
+    """
+
+    diffs = diff_behavior(
+        FunctionInfo.from_fn(_sum_list_original),
+        FunctionInfo.from_fn(_sum_list_rewrite_2),
+        DEFAULT_OPTIONS,
+        exception_equivalence=ExceptionEquivalenceType.SAME_TYPE,
+    )
+    debug("diffs=", diffs)
+    assert (
+        len(diffs) == 1
+    )  # finds a counter-example, because TypeError!=CustomException
+    assert isinstance(diffs[0], BehaviorDiff)
+    assert isinstance(diffs[0].result1.error, str)
+    assert isinstance(diffs[0].result2.error, str)
+    assert diffs[0].result1.error.startswith("TypeError")
+    assert diffs[0].result2.error.startswith("CustomException")
+
+
+def test_diff_behavior_nan() -> None:
+    def f(x: float):
+        return x
+
+    diffs = diff_behavior(
+        FunctionInfo.from_fn(f),
+        FunctionInfo.from_fn(f),
+        DEFAULT_OPTIONS,
+    )
+    assert diffs == []

crosshair/dynamic_typing.py

@@ -1,9 +1,14 @@
 import collections.abc
+import sys
 import typing
-from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Type
+from inspect import Parameter, Signature
+from itertools import zip_longest
+from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Type
 
 import typing_inspect  # type: ignore
 
+from crosshair.util import debug  # type: ignore
+
 _EMPTYSET: frozenset = frozenset()
 
 
@@ -54,7 +59,7 @@ def unify_callable_args(
         return True
     if len(value_types) != len(recv_types):
         return False
-    for (varg, rarg) in zip(value_types, recv_types):
+    for varg, rarg in zip(value_types, recv_types):
         # note reversal here: Callable is contravariant in argument types
         if not unify(rarg, varg, bindings):
             return False
@@ -201,7 +206,7 @@ def unify(
                 vargs = [object for _ in rargs]
             else:
                 return False
-        for (varg, targ) in zip(vargs, rargs):
+        for varg, targ in zip(vargs, rargs):
             if not unify(varg, targ, bindings):
                 return False
         return True
@@ -219,18 +224,123 @@ def get_bindings_from_type_arguments(pytype: Type) -> Mapping[object, type]:
     return {}
 
 
-def realize(pytype: Type, bindings: Mapping[object, type]) -> object:
-    if typing_inspect.is_typevar(pytype):
-        return bindings[pytype]
-    if not hasattr(pytype, "__args__"):
-        return pytype
-    newargs: List = []
-    for arg in pytype.__args__:  # type:ignore
-        newargs.append(realize(arg, bindings))
-    # print('realizing pytype', repr(pytype), 'newargs', repr(newargs))
-    pytype_origin = origin_of(pytype)
-    if not hasattr(pytype_origin, "_name"):
-        pytype_origin = getattr(typing, pytype._name)  # type:ignore
-    if pytype_origin is Callable:  # Callable args get flattened
-        newargs = [newargs[:-1], newargs[-1]]
-    return pytype_origin.__getitem__(tuple(newargs))
+if sys.version_info >= (3, 9):
+
+    def realize(pytype: Type, bindings: Mapping[object, type]) -> object:
+        if typing_inspect.is_typevar(pytype):
+            return bindings[pytype]
+        if not hasattr(pytype, "__args__"):
+            return pytype
+        newargs: List = []
+        for arg in pytype.__args__:  # type:ignore
+            newargs.append(realize(arg, bindings))
+        pytype_origin = origin_of(pytype)
+        if pytype_origin in (
+            collections.abc.Callable,
+            typing.Callable,
+        ):  # Callable args get flattened
+            newargs = [newargs[:-1], newargs[-1]]
+        return pytype_origin.__class_getitem__(tuple(newargs))
+
+else:
+
+    def realize(pytype: Type, bindings: Mapping[object, type]) -> object:
+        if typing_inspect.is_typevar(pytype):
+            return bindings[pytype]
+        if not hasattr(pytype, "__args__"):
+            return pytype
+        newargs: List = []
+        for arg in pytype.__args__:  # type:ignore
+            newargs.append(realize(arg, bindings))
+        # print('realizing pytype', repr(pytype), 'newargs', repr(newargs))
+        pytype_origin = origin_of(pytype)
+        if not hasattr(pytype_origin, "_name"):
+            pytype_origin = getattr(typing, pytype._name)  # type:ignore
+        if pytype_origin is Callable:  # Callable args get flattened
+            newargs = [newargs[:-1], newargs[-1]]
+        return pytype_origin.__getitem__(tuple(newargs))
+
+
+def isolate_var_params(
+    sig: Signature,
+) -> Tuple[
+    List[Parameter], Dict[str, Parameter], Optional[Parameter], Optional[Parameter]
+]:
+    pos_only_params: List[Parameter] = []
+    keyword_params: Dict[str, Parameter] = {}
+    var_positional: Optional[Parameter] = None
+    var_keyword: Optional[Parameter] = None
+    for name, param in sig.parameters.items():
+        if param.kind == Parameter.VAR_POSITIONAL:
+            var_positional = param
+        elif param.kind == Parameter.VAR_KEYWORD:
+            var_keyword = param
+        elif param.kind == Parameter.POSITIONAL_ONLY:
+            pos_only_params.append(param)
+        else:
+            keyword_params[name] = param
+    return pos_only_params, keyword_params, var_positional, var_keyword
+
+
+def intersect_signatures(
+    sig1: Signature,
+    sig2: Signature,
+) -> Signature:
+    """
+    Approximate the intersection of two signatures.
+    The resulting signature may be overly loose
+    (matching some inputs that neither of the original signatures would match),
+    but it should cover all the inputs for each original signature.
+
+    One minor exception: All arguments that are allowed to be called as
+    keyword arguments will be converted to keyword-only arguments.
+    We do this to resolve the abiguity when position-or-keyword arguments
+    appear in the same position but with different names.
+    """
+    pos1, key1, var_pos1, var_key1 = isolate_var_params(sig1)
+    pos2, key2, var_pos2, var_key2 = isolate_var_params(sig2)
+    is_squishy1 = var_pos1 is not None or var_key1 is not None
+    is_squishy2 = var_pos2 is not None or var_key2 is not None
+    out_params: Dict[str, Parameter] = {}
+    for p1, p2 in zip_longest(pos1, pos2):
+        if p1 is None:
+            if is_squishy1:
+                out_params[p2.name] = p2
+        elif p2 is None:
+            if is_squishy2:
+                out_params[p1.name] = p1
+        elif unify(p1.annotation, p2.annotation):
+            out_params[p1.name] = p1
+        else:
+            out_params[p2.name] = p2
+    for key in [
+        k
+        for pair in zip_longest(key1.keys(), key2.keys())
+        for k in pair
+        if k is not None
+    ]:
+        if key not in key2:
+            if is_squishy2:
+                out_params[key] = key1[key].replace(kind=Parameter.KEYWORD_ONLY)
+            continue
+        if key not in key1:
+            if is_squishy1:
+                out_params[key] = key2[key].replace(kind=Parameter.KEYWORD_ONLY)
+            continue
+        if unify(key1[key].annotation, key2[key].annotation):
+            out_params[key] = key1[key].replace(kind=Parameter.KEYWORD_ONLY)
+        else:
+            out_params[key] = key2[key].replace(kind=Parameter.KEYWORD_ONLY)
+    if var_pos1 and var_pos2:
+        out_params[var_pos1.name] = var_pos1
+    if var_key1 and var_key2:
+        out_params[var_key1.name] = var_key1
+    if unify(sig1.return_annotation, sig2.return_annotation):
+        out_return_annotation = sig1.return_annotation
+    else:
+        out_return_annotation = sig2.return_annotation
+    result = Signature(
+        parameters=list(out_params.values()), return_annotation=out_return_annotation
+    )
+    debug("Combined __init__ and __new__ signatures", sig1, "and", sig2, "into", result)
+    return result