crosshair-tool 0.0.56__cp39-cp39-macosx_11_0_arm64.whl → 0.0.100__cp39-cp39-macosx_11_0_arm64.whl

crosshair/tracers.py

@@ -3,7 +3,9 @@
 import ctypes
 import dataclasses
 import dis
+import os
 import sys
+import types
 from collections import defaultdict
 from sys import _getframe
 from types import CodeType
@@ -12,6 +14,7 @@ from typing import (
     Callable,
     DefaultDict,
     Dict,
+    FrozenSet,
     Iterable,
     List,
     Optional,
@@ -20,8 +23,11 @@ from typing import (
     TypeVar,
 )
 
-from _crosshair_tracers import CTracer, TraceSwap
+from _crosshair_tracers import CTracer, TraceSwap, supported_opcodes  # type: ignore
 
+CROSSHAIR_EXTRA_ASSERTS = os.environ.get("CROSSHAIR_EXTRA_ASSERTS", "0") == "1"
+
+SYS_MONITORING_TOOL_ID = 4
 USE_C_TRACER = True
 
 PyObjPtr = ctypes.POINTER(ctypes.py_object)
@@ -42,12 +48,12 @@ _debug_header: Tuple[Tuple[str, type], ...] = (
 from _crosshair_tracers import frame_stack_read, frame_stack_write
 
 CALL_FUNCTION = dis.opmap.get("CALL_FUNCTION", 256)
-CALL_FUNCTION_KW = dis.opmap.get("CALL_FUNCTION_KW", 256)
+CALL_FUNCTION_KW = dis.opmap.get("CALL_FUNCTION_KW", 256)  # Removed as of 3.11
 CALL_FUNCTION_EX = dis.opmap.get("CALL_FUNCTION_EX", 256)
 CALL_METHOD = dis.opmap.get("CALL_METHOD", 256)
 BUILD_TUPLE_UNPACK_WITH_CALL = dis.opmap.get("BUILD_TUPLE_UNPACK_WITH_CALL", 256)
 CALL = dis.opmap.get("CALL", 256)
-CALL_KW = dis.opmap.get("CALL_KW", 256)
+CALL_KW = dis.opmap.get("CALL_KW", 256)  # New in 3.13
 
 
 class RawNullPointer:
@@ -55,89 +61,113 @@ class RawNullPointer:
 
 
 NULL_POINTER = RawNullPointer()
+CallStackInfo = (
+    Tuple[  # Information about the interpreter stack just before calling a function
+        int,  # stack index of the callable
+        Callable,  # the callable object itself
+        Optional[int],  # index of kwargs dict (if used in this call)
+    ]
+)
 
 
-def handle_build_tuple_unpack_with_call(frame) -> Optional[Tuple[int, Callable]]:
+def handle_build_tuple_unpack_with_call(frame) -> CallStackInfo:
     idx = -(
         frame.f_code.co_code[frame.f_lasti + 1] + 1
     )  # TODO: account for EXTENDED_ARG, here and elsewhere
     try:
-        return (idx, frame_stack_read(frame, idx))
+        return (idx, frame_stack_read(frame, idx), None)
     except ValueError:
         return (idx, NULL_POINTER)  # type: ignore
 
 
-def handle_call_3_11(frame) -> Optional[Tuple[int, Callable]]:
+def handle_call_3_11(frame) -> CallStackInfo:
     idx = -(frame.f_code.co_code[frame.f_lasti + 1] + 1)
     try:
-        ret = (idx - 1, frame_stack_read(frame, idx - 1))
+        ret = (idx - 1, frame_stack_read(frame, idx - 1), None)
     except ValueError:
-        ret = (idx, frame_stack_read(frame, idx))
+        ret = (idx, frame_stack_read(frame, idx), None)
     return ret
 
 
-def handle_call_3_13(frame) -> Optional[Tuple[int, Callable]]:
+def handle_call_3_13(frame) -> CallStackInfo:
     idx = -(frame.f_code.co_code[frame.f_lasti + 1] + 2)
-    return (idx, frame_stack_read(frame, idx))
+    return (idx, frame_stack_read(frame, idx), None)
 
 
-def handle_call_function(frame) -> Optional[Tuple[int, Callable]]:
+def handle_call_function(frame) -> CallStackInfo:
     idx = -(frame.f_code.co_code[frame.f_lasti + 1] + 1)
     try:
-        return (idx, frame_stack_read(frame, idx))
+        return (idx, frame_stack_read(frame, idx), None)
     except ValueError:
         return (idx, NULL_POINTER)  # type: ignore
 
 
-def handle_call_function_kw(frame) -> Optional[Tuple[int, Callable]]:
+def handle_call_function_kw(frame) -> CallStackInfo:
     idx = -(frame.f_code.co_code[frame.f_lasti + 1] + 2)
     try:
-        return (idx, frame_stack_read(frame, idx))
+        return (idx, frame_stack_read(frame, idx), None)
     except ValueError:
         return (idx, NULL_POINTER)  # type: ignore
 
 
-def handle_call_kw(frame) -> Optional[Tuple[int, Callable]]:
+def handle_call_kw(frame) -> CallStackInfo:
     idx = -(frame.f_code.co_code[frame.f_lasti + 1] + 3)
-    return (idx, frame_stack_read(frame, idx))
+    return (idx, frame_stack_read(frame, idx), None)
 
 
-def handle_call_function_ex_3_6(frame) -> Optional[Tuple[int, Callable]]:
-    idx = -((frame.f_code.co_code[frame.f_lasti + 1] & 1) + 2)
+def handle_call_function_ex_3_6(frame) -> CallStackInfo:
+    has_kwargs = frame.f_code.co_code[frame.f_lasti + 1] & 1
+    idx = -(has_kwargs + 2)
+    kwargs_idx = -1 if has_kwargs else None
     try:
-        # print("fetch @ idx", idx)
-        return (idx, frame_stack_read(frame, idx))
+        return (idx, frame_stack_read(frame, idx), kwargs_idx)
     except ValueError:
-        return (idx, NULL_POINTER)  # type: ignore
+        return (idx, NULL_POINTER, kwargs_idx)  # type: ignore
 
 
-def handle_call_function_ex_3_13(frame) -> Optional[Tuple[int, Callable]]:
-    idx = -((frame.f_code.co_code[frame.f_lasti + 1] & 1) + 3)
+def handle_call_function_ex_3_13(frame) -> CallStackInfo:
+    has_kwargs = frame.f_code.co_code[frame.f_lasti + 1] & 1
+    idx = -(has_kwargs + 3)
+    kwargs_idx = -1 if has_kwargs else None
     try:
-        return (idx, frame_stack_read(frame, idx))
+        return (idx, frame_stack_read(frame, idx), kwargs_idx)
     except ValueError:
-        return (idx, NULL_POINTER)  # type: ignore
+        return (idx, NULL_POINTER, kwargs_idx)  # type: ignore
+
+
+def handle_call_function_ex_3_14(frame) -> CallStackInfo:
+    callable_idx, kwargs_idx = -4, -1
+    try:
+        return (callable_idx, frame_stack_read(frame, callable_idx), kwargs_idx)
+    except ValueError:
+        return (callable_idx, NULL_POINTER, kwargs_idx)  # type: ignore
 
 
-def handle_call_method(frame) -> Optional[Tuple[int, Callable]]:
+def handle_call_method(frame) -> CallStackInfo:
     idx = -(frame.f_code.co_code[frame.f_lasti + 1] + 2)
     try:
-        return (idx, frame_stack_read(frame, idx))
+        return (idx, frame_stack_read(frame, idx), None)
     except ValueError:
         # not a sucessful method lookup; no call happens here
         idx += 1
-        return (idx, frame_stack_read(frame, idx))
+        return (idx, frame_stack_read(frame, idx), None)
 
 
-_CALL_HANDLERS: Dict[int, Callable[[object], Optional[Tuple[int, Callable]]]] = {
+_CALL_HANDLERS: Dict[int, Callable[[object], CallStackInfo]] = {
     BUILD_TUPLE_UNPACK_WITH_CALL: handle_build_tuple_unpack_with_call,
     CALL: handle_call_3_13 if sys.version_info >= (3, 13) else handle_call_3_11,
     CALL_KW: handle_call_kw,
     CALL_FUNCTION: handle_call_function,
     CALL_FUNCTION_KW: handle_call_function_kw,
-    CALL_FUNCTION_EX: handle_call_function_ex_3_13
-    if sys.version_info >= (3, 13)
-    else handle_call_function_ex_3_6,
+    CALL_FUNCTION_EX: (
+        handle_call_function_ex_3_14
+        if sys.version_info >= (3, 14)
+        else (
+            handle_call_function_ex_3_13
+            if sys.version_info >= (3, 13)
+            else handle_call_function_ex_3_6
+        )
+    ),
     CALL_METHOD: handle_call_method,
 }
 
@@ -152,6 +182,35 @@ class TraceException(BaseException):
     pass
 
 
+def check_opcode_support(opcodes: FrozenSet[int]):
+    if sys.version_info < (3, 12):
+        return
+    missing_opcodes = opcodes - set(supported_opcodes())
+    if missing_opcodes:
+        raise TraceException(
+            f"The C-level tracer does not support these opcodes: {','.join(map(dis.opname.__getitem__, missing_opcodes))}"
+        )
+
+
+check_opcode_support(frozenset(_CALL_HANDLERS.keys()))
+
+
+wrapper_descriptor_type = type(int.__bool__)
+assert str(wrapper_descriptor_type) == "<class 'wrapper_descriptor'>"
+
+_NORMAL_CALLABLE_TYPES = (
+    type,
+    types.FunctionType,  #': <class 'function'>,
+    types.MethodDescriptorType,  #': <class 'method_descriptor'>,
+    types.MethodType,  #': <class 'method'>,
+    types.MethodWrapperType,  #': <class 'method-wrapper'>}
+    types.BuiltinFunctionType,  #': <class 'builtin_function_or_method'>,
+    types.BuiltinMethodType,  #: <class 'builtin_function_or_method'>,
+    types.ClassMethodDescriptorType,  #': <class 'classmethod_descriptor'>,
+    wrapper_descriptor_type,
+)
+
+
 class TracingModule:
     # override these!:
     opcodes_wanted = frozenset(_CALL_HANDLERS.keys())
@@ -165,18 +224,21 @@ class TracingModule:
         call_handler = _CALL_HANDLERS.get(opcodenum)
         if not call_handler:
             return None
-        maybe_call_info = call_handler(frame)
-        if maybe_call_info is None:
-            # TODO: this cannot happen?
-            return None
-        (fn_idx, target) = maybe_call_info
+        (fn_idx, target, kwargs_idx) = call_handler(frame)
         binding_target = None
 
+        __self = None
         try:
             __self = object.__getattribute__(target, "__self__")
         except AttributeError:
             pass
-        else:
+        if (__self is None) and (not isinstance(target, _NORMAL_CALLABLE_TYPES)):
+            try:
+                target = object.__getattribute__(target, "__call__")
+                __self = object.__getattribute__(target, "__self__")
+            except AttributeError:
+                pass
+        if __self is not None:
             try:
                 __func = object.__getattribute__(target, "__func__")
             except AttributeError:
@@ -190,6 +252,27 @@ class TracingModule:
                 binding_target = __self
                 target = __func
 
+        if kwargs_idx is not None:
+            try:
+                kwargs_dict = frame_stack_read(frame, kwargs_idx)
+            except ValueError:
+                pass
+            else:
+                replacement_kwargs = {}
+                for key, val in kwargs_dict.items():
+                    if isinstance(key, str):
+                        replacement_kwargs[key] = val
+                        continue
+                    # circular import:
+                    from crosshair.libimpl.builtinslib import AnySymbolicStr
+
+                    if isinstance(key, AnySymbolicStr):
+                        # NOTE: We need to ensure symbolic strings don't need tracing for realization
+                        replacement_kwargs[key.__ch_realize__()] = val
+                    else:
+                        raise TypeError("keywords must be strings")
+                frame_stack_write(frame, kwargs_idx, replacement_kwargs)
+
         if isinstance(target, Untracable):
             return None
         replacement = self.trace_call(frame, target, binding_target)
@@ -222,7 +305,6 @@ class PatchingModule(TracingModule):
     def __init__(
         self,
         overrides: Optional[Dict[Callable, Callable]] = None,
-        fn_type_overrides: Optional[Dict[type, Callable]] = None,
     ):
         # NOTE: you might imagine that we should use an IdKeyedDict for self.overrides
         # However, some builtin bound methods have no way to get identity for their code:
@@ -234,11 +316,13 @@ class PatchingModule(TracingModule):
         self.nextfn: Dict[object, Callable] = {}  # code object to next, lower layer
         if overrides:
             self.add(overrides)
-        self.fn_type_overrides = fn_type_overrides or {}
 
     def add(self, new_overrides: Dict[Callable, Callable]):
         for orig, new_override in new_overrides.items():
             prev_override = self.overrides.get(orig, orig)
+            assert (
+                prev_override is not new_override
+            ), f"Function patch {new_override} has already been applied"
             self.nextfn[(new_override.__code__, orig)] = prev_override
             self.overrides[orig] = new_override
 
@@ -265,11 +349,7 @@ class PatchingModule(TracingModule):
             # not properly unbound by `TracingModule.trace_op`.
             return None
         if target is None:
-            fn_type_override = self.fn_type_overrides.get(type(fn))
-            if fn_type_override is None:
-                return None
-            else:
-                target = fn_type_override(fn)
+            return None
         caller_code = frame.f_code
         if caller_code.co_name == "_crosshair_wrapper":
             return None
@@ -306,7 +386,7 @@ class CompositeTracer:
 
         def __enter__(self) -> object:
             self.ctracer.push_module(self.patching_module)
-            tool_id = 4
+            tool_id = SYS_MONITORING_TOOL_ID
             sys.monitoring.use_tool_id(tool_id, "CrossHair")
             sys.monitoring.register_callback(
                 tool_id,
@@ -321,7 +401,7 @@ class CompositeTracer:
             return self
 
         def __exit__(self, _etype, exc, _etb):
-            tool_id = 4
+            tool_id = SYS_MONITORING_TOOL_ID
             sys.monitoring.register_callback(
                 tool_id, sys.monitoring.events.INSTRUCTION, None
             )
@@ -380,6 +460,9 @@ class CoverageResult:
 class CoverageTracingModule(TracingModule):
     opcodes_wanted = frozenset(i for i in range(256))
 
+    # TODO: this needs to be moved into a separate kind of monitor to
+    # support threading (sys.monitoring probes are global)
+
     def __init__(self, *fns: Callable):
         assert not is_tracing()
         self.fns = fns
@@ -431,8 +514,17 @@ def NoTracing():
     return TraceSwap(COMPOSITE_TRACER.ctracer, True)
 
 
-def ResumedTracing():
-    return TraceSwap(COMPOSITE_TRACER.ctracer, False)
+if CROSSHAIR_EXTRA_ASSERTS:
+
+    def ResumedTracing():
+        if COMPOSITE_TRACER.ctracer.is_handling():
+            raise TraceException("Cannot resume tracing while opcode handling")
+        return TraceSwap(COMPOSITE_TRACER.ctracer, False)
+
+else:
+
+    def ResumedTracing():
+        return TraceSwap(COMPOSITE_TRACER.ctracer, False)
 
 
 _T = TypeVar("_T")

crosshair/type_repo.py

@@ -5,6 +5,8 @@ from typing import Dict, List, Optional, Type
 
 import z3  # type: ignore
 
+from crosshair.tracers import NoTracing
+from crosshair.util import CrossHairInternal, CrossHairValue, is_hashable
 from crosshair.z3util import z3Eq, z3Not
 
 _MAP: Optional[Dict[type, List[type]]] = None
@@ -18,6 +20,8 @@ _IGNORED_MODULE_ROOTS = {
     "pkg_resources",
     "pytest",
     "py",  # (part of pytest)
+    # Disabled because the import attempt can cause problems:
+    "numpy",  # importing numpy/testing/_private/utils.py attempts a subprocess call
 }
 
 
@@ -48,7 +52,7 @@ def get_subclass_map() -> Dict[type, List[type]]:
     Crawl all types presently in memory and makes a map from parent to child classes.
 
     Only direct children are included.
-    Does not yet handle "protocol" subclassing (eg "Iterator", "Mapping", etc).
+    TODO: Does not yet handle "protocol" subclassing (eg "Iterator", "Mapping", etc).
     """
     global _MAP
     if _MAP is None:
@@ -57,15 +61,13 @@ def get_subclass_map() -> Dict[type, List[type]]:
             if module_name.split(".", 1)[0] in _IGNORED_MODULE_ROOTS:
                 continue
             if module is None:
-                # We set the internal _datetime module to None, ensuring that
-                # we don't load the C implementation.
                 continue
             try:
                 module_classes = inspect.getmembers(module, inspect.isclass)
             except ImportError:
                 continue
             for _, cls in module_classes:
-                if _class_known_to_be_copyable(cls):
+                if _class_known_to_be_copyable(cls) and is_hashable(cls):
                     classes.add(cls)
         subclass = collections.defaultdict(list)
         for cls in classes:
@@ -120,6 +122,11 @@ class SymbolicTypeRepository:
         return SMT_SUBTYPE_FN(typ1, typ2) == MAYBE_SORT.yes
 
     def get_type(self, typ: Type) -> z3.ExprRef:
+        with NoTracing():
+            if issubclass(typ, CrossHairValue):
+                raise CrossHairInternal(
+                    "Type repo should not be queried with a symbolic"
+                )
         pytype_to_smt = self.pytype_to_smt
         if typ not in pytype_to_smt:
             stmts = []