crosshair-tool 0.0.56__cp39-cp39-macosx_11_0_arm64.whl → 0.0.100__cp39-cp39-macosx_11_0_arm64.whl

crosshair/__init__.py

@@ -15,7 +15,7 @@ from crosshair.statespace import StateSpace
 from crosshair.tracers import NoTracing, ResumedTracing
 from crosshair.util import IgnoreAttempt, debug
 
-__version__ = "0.0.56"  # Do not forget to update in setup.py!
+__version__ = "0.0.100"  # Do not forget to update in setup.py!
 __author__ = "Phillip Schanely"
 __license__ = "MIT"
 __status__ = "Alpha"

crosshair/_mark_stacks.h

@@ -3,7 +3,7 @@
 
 
 // This file includes a modified version of CPython's mark_stacks
-// implementation fomr:
+// implementation from:
 // https://github.com/python/cpython/blob/v3.12.0/Objects/frameobject.c
 
 // The shared source code is licensed under the PSF license and is
@@ -36,6 +36,7 @@ _ch_pop_to_level(int64_t stack, int level) {
 //    Python 3.13
 //    ===========
 
+// from Include/internal/pycore_opcode_metadata.h
 const uint8_t _ch_PyOpcode_Caches[256] = {
     [JUMP_BACKWARD] = 1,
     [TO_BOOL] = 3,
@@ -48,6 +49,7 @@ const uint8_t _ch_PyOpcode_Caches[256] = {
     [LOAD_SUPER_ATTR] = 1,
     [LOAD_ATTR] = 9,
     [COMPARE_OP] = 1,
+    [CONTAINS_OP] = 1,
     [POP_JUMP_IF_TRUE] = 1,
     [POP_JUMP_IF_FALSE] = 1,
     [POP_JUMP_IF_NONE] = 1,
@@ -57,6 +59,7 @@ const uint8_t _ch_PyOpcode_Caches[256] = {
     [BINARY_OP] = 1,
 };
 
+// from Include/internal/pycore_opcode_metadata.h
 const uint8_t _ch_PyOpcode_Deopt[256] = {
     [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH,
     [BEFORE_WITH] = BEFORE_WITH,
@@ -87,6 +90,7 @@ const uint8_t _ch_PyOpcode_Deopt[256] = {
     [CALL] = CALL,
     [CALL_ALLOC_AND_ENTER_INIT] = CALL,
     [CALL_BOUND_METHOD_EXACT_ARGS] = CALL,
+    [CALL_BOUND_METHOD_GENERAL] = CALL,
     [CALL_BUILTIN_CLASS] = CALL,
     [CALL_BUILTIN_FAST] = CALL,
     [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL,
@@ -102,8 +106,9 @@ const uint8_t _ch_PyOpcode_Deopt[256] = {
     [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL,
     [CALL_METHOD_DESCRIPTOR_NOARGS] = CALL,
     [CALL_METHOD_DESCRIPTOR_O] = CALL,
+    [CALL_NON_PY_GENERAL] = CALL,
     [CALL_PY_EXACT_ARGS] = CALL,
-    [CALL_PY_WITH_DEFAULTS] = CALL,
+    [CALL_PY_GENERAL] = CALL,
     [CALL_STR_1] = CALL,
     [CALL_TUPLE_1] = CALL,
     [CALL_TYPE_1] = CALL,
@@ -115,6 +120,8 @@ const uint8_t _ch_PyOpcode_Deopt[256] = {
     [COMPARE_OP_INT] = COMPARE_OP,
     [COMPARE_OP_STR] = COMPARE_OP,
     [CONTAINS_OP] = CONTAINS_OP,
+    [CONTAINS_OP_DICT] = CONTAINS_OP,
+    [CONTAINS_OP_SET] = CONTAINS_OP,
     [CONVERT_VALUE] = CONVERT_VALUE,
     [COPY] = COPY,
     [COPY_FREE_VARS] = COPY_FREE_VARS,
@@ -268,6 +275,7 @@ const uint8_t _ch_PyOpcode_Deopt[256] = {
     [YIELD_VALUE] = YIELD_VALUE,
 };
 
+// from Python/instrumentation.c
 static const uint8_t _ch_DE_INSTRUMENT[256] = {
     [INSTRUMENTED_RESUME] = RESUME,
     [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE,
@@ -530,26 +538,13 @@ static const uint8_t _ch_DE_INSTRUMENT[256] = {
 #endif
 #endif
 
-/* Get the underlying opcode, stripping instrumentation */
-int _ch_Py_GetBaseOpcode(PyCodeObject *code, int i)
-{
-    int opcode = _PyCode_CODE(code)[i].op.code;
-    if (opcode == INSTRUMENTED_LINE) {
-        opcode = code->_co_monitoring->lines[i].original_opcode;
-    }
-    if (opcode == INSTRUMENTED_INSTRUCTION) {
-        opcode = code->_co_monitoring->per_instruction_opcodes[i];
-    }
-    int deinstrumented = _ch_DE_INSTRUMENT[opcode];
-    if (deinstrumented) {
-        return deinstrumented;
-    }
-    return _ch_PyOpcode_Deopt[opcode];
-}
-
 static int64_t *
 _ch_mark_stacks(PyCodeObject *code_obj, int len)
 {
+    // This differs from the corresponding function in CPython in a few ways:
+    // 1) The returned stack values represent stack depths, not stack content types.
+    // 2) The low bit of each depth is a flag that indicates whether this is an
+    // instruction that we trace, or it could follow an instruction that we trace.
     PyObject *co_code = PyCode_GetCode(code_obj);
     // printf("co_code %d\n", co_code);
     if (co_code == NULL) {
@@ -557,34 +552,47 @@ _ch_mark_stacks(PyCodeObject *code_obj, int len)
     }
     _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code);
     int64_t *stacks = PyMem_New(int64_t, len+1);
-    int i, j, opcode;
-
     if (stacks == NULL) {
         PyErr_NoMemory();
         Py_DECREF(co_code);
         return NULL;
     }
+    uint8_t *enabled_tracing = PyMem_New(uint8_t, len+1);
+    if (enabled_tracing == NULL) {
+        PyErr_NoMemory();
+        PyMem_Free(stacks);
+        Py_DECREF(co_code);
+        return NULL;
+    }
+    int i, j, opcode;
+
     for (int i = 1; i <= len; i++) {
         stacks[i] = UNINITIALIZED;
+        enabled_tracing[i] = 0;
     }
     stacks[0] = EMPTY_STACK;
+#if PY_VERSION_HEX < 0x030D0000
+    // Python 3.12
     if (code_obj->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR))
     {
         // Generators get sent None while starting:
         stacks[0]++;
     }
+#endif
     int todo = 1;
     while (todo) {
         todo = 0;
         /* Scan instructions */
         for (i = 0; i < len;) {
             int64_t next_stack = stacks[i];
-            opcode = _ch_Py_GetBaseOpcode(code_obj, i);
+            opcode = code[i].op.code;
+            uint8_t trace_enabled_here = _ch_TRACABLE_INSTRUCTIONS[opcode];
+            enabled_tracing[i] |= trace_enabled_here;
             int oparg = 0;
             while (opcode == EXTENDED_ARG) {
                 oparg = (oparg << 8) | code[i].op.arg;
                 i++;
-                opcode = _ch_Py_GetBaseOpcode(code_obj, i);
+                opcode = code[i].op.code;
                 stacks[i] = next_stack;
             }
             int next_i = i + _ch_PyOpcode_Caches[opcode] + 1;
@@ -607,6 +615,7 @@ _ch_mark_stacks(PyCodeObject *code_obj, int len)
                     target_stack = next_stack;
                     assert(stacks[j] == UNINITIALIZED || stacks[j] == target_stack);
                     stacks[j] = target_stack;
+                    enabled_tracing[j] |= trace_enabled_here;
                     stacks[next_i] = next_stack;
                     break;
                 }
@@ -615,6 +624,7 @@ _ch_mark_stacks(PyCodeObject *code_obj, int len)
                     assert(j < len);
                     assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack);
                     stacks[j] = next_stack;
+                    enabled_tracing[j] |= trace_enabled_here;
                     stacks[next_i] = next_stack;
                     break;
                 case JUMP_FORWARD:
@@ -622,6 +632,7 @@ _ch_mark_stacks(PyCodeObject *code_obj, int len)
                     assert(j < len);
                     assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack);
                     stacks[j] = next_stack;
+                    enabled_tracing[j] |= trace_enabled_here;
                     break;
                 case JUMP_BACKWARD:
                 case JUMP_BACKWARD_NO_INTERRUPT:
@@ -633,6 +644,7 @@ _ch_mark_stacks(PyCodeObject *code_obj, int len)
                     }
                     assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack);
                     stacks[j] = next_stack;
+                    enabled_tracing[j] |= trace_enabled_here;
                     break;
                 case GET_ITER:
                 case GET_AITER:
@@ -646,10 +658,18 @@ _ch_mark_stacks(PyCodeObject *code_obj, int len)
                     assert(j < len);
                     assert(stacks[j] == UNINITIALIZED || stacks[j] == target_stack);
                     stacks[j] = target_stack;
+                    enabled_tracing[j] |= trace_enabled_here;
                     break;
                 }
                 case END_ASYNC_FOR:
+#if PY_VERSION_HEX < 0x030D0000
+                   // Python 3.12
+                    next_stack--;
+#else
+                   // Python 3.13+
                     next_stack--;
+                    next_stack--;
+#endif
                     stacks[next_i] = next_stack;
                     break;
                 case PUSH_EXC_INFO:
@@ -676,10 +696,10 @@ _ch_mark_stacks(PyCodeObject *code_obj, int len)
                 case LOAD_GLOBAL:
                 {
                     int j = oparg;
+                    next_stack++;
                     if (j & 1) {
                         next_stack++;
                     }
-                    next_stack++;
                     stacks[next_i] = next_stack;
                     break;
                 }
@@ -725,6 +745,7 @@ _ch_mark_stacks(PyCodeObject *code_obj, int len)
                     stacks[next_i] = next_stack;
                 }
             }
+            enabled_tracing[next_i] |= trace_enabled_here;
             i = next_i;
         }
         /* Scan exception table */
@@ -759,5 +780,11 @@ _ch_mark_stacks(PyCodeObject *code_obj, int len)
         }
     }
     Py_DECREF(co_code);
+    for (i = 0; i < len; i++) {
+        if (stacks[i] >= 0) {
+            stacks[i] = (stacks[i] << 1) | enabled_tracing[i];
+        }
+    }
+    PyMem_Free(enabled_tracing);
     return stacks;
 }

crosshair/_tracers.h

@@ -50,10 +50,11 @@ typedef struct HandlerTable {
 } HandlerTable;
 
 
-typedef struct FrameAndCallback {
-    PyObject* frame;
+typedef struct FrameNextIandCallback {
+    PyFrameObject* frame;
+    int expected_i;
     PyObject* callback;
-} FrameAndCallback;
+} FrameNextIandCallback;
 
 
 typedef struct CodeAndStacks {
@@ -62,7 +63,7 @@ typedef struct CodeAndStacks {
 } CodeAndStacks;
 
 
-DEFINE_VEC(FrameAndCallbackVec, FrameAndCallback, init_framecbvec, push_framecb);
+DEFINE_VEC(FrameNextIandCallbackVec, FrameNextIandCallback, init_framecbvec, push_framecb);
 DEFINE_VEC(ModuleVec, PyObject*, init_modulevec, push_module);
 DEFINE_VEC(TableVec, HandlerTable, init_tablevec, push_table_entry)
 
@@ -70,9 +71,10 @@ typedef struct CTracer {
     PyObject_HEAD
     ModuleVec modules;
     TableVec handlers;
-    FrameAndCallbackVec postop_callbacks;
+    FrameNextIandCallbackVec postop_callbacks;
     BOOL enabled;
     BOOL handling;
+    BOOL trace_all_opcodes;
     int thread_id;
 } CTracer;
 
@@ -87,4 +89,6 @@ typedef struct TraceSwap {
 
 extern PyTypeObject TraceSwapType;
 
+extern const uint8_t _ch_TRACABLE_INSTRUCTIONS[256];
+
 #endif /* _COVERAGE_TRACER_H */

crosshair/_tracers_test.py

@@ -5,7 +5,11 @@ from typing import List
 
 import pytest
 
-from _crosshair_tracers import CTracer, code_stack_depths, frame_stack_read
+from _crosshair_tracers import (  # type: ignore
+    CTracer,
+    code_stack_depths,
+    frame_stack_read,
+)
 from crosshair.util import mem_usage_kb
 
 
@@ -15,19 +19,19 @@ class ExampleModule:
 
 def test_CTracer_module_refcounts_dont_leak():
     mod = ExampleModule()
-    assert sys.getrefcount(mod) == 2
+    base_count = sys.getrefcount(mod)
     tracer = CTracer()
     tracer.push_module(mod)
-    assert sys.getrefcount(mod) == 3
+    assert sys.getrefcount(mod) == base_count + 1
     tracer.push_module(mod)
     tracer.start()
     tracer.stop()
-    assert sys.getrefcount(mod) == 4
+    assert sys.getrefcount(mod) == base_count + 2
     tracer.pop_module(mod)
-    assert sys.getrefcount(mod) == 3
+    assert sys.getrefcount(mod) == base_count + 1
     del tracer
     gc.collect()
-    assert sys.getrefcount(mod) == 2
+    assert sys.getrefcount(mod) == base_count
 
 
 def _get_depths(fn):
@@ -85,7 +89,10 @@ def _log_execution_stacks(fn, *a, **kw):
     return stacks
 
 
-@pytest.mark.skipif(sys.version_info < (3, 12), reason="stack depth on 3.12+")
+@pytest.mark.skipif(
+    sys.version_info < (3, 12) or sys.version_info >= (3, 14),
+    reason="stack depths only in 3.12 & 3.13",
+)
 def test_one_function_stack_depth():
     _E = (TypeError, KeyboardInterrupt)
 
@@ -96,7 +103,10 @@ def test_one_function_stack_depth():
     _log_execution_stacks(a, 4)
 
 
-@pytest.mark.skipif(sys.version_info < (3, 12), reason="stack depth on 3.12+")
+@pytest.mark.skipif(
+    sys.version_info < (3, 12) or sys.version_info >= (3, 14),
+    reason="stack depths only in 3.12 & 3.13",
+)
 def test_stack_get():
     def to_be_traced(x):
         r = 8 - x
@@ -125,4 +135,4 @@ def test_CTracer_does_not_leak_memory():
         if i == 100:
             usage = mem_usage_kb()
     usage_increase = mem_usage_kb() - usage
-    assert usage_increase < 35
+    assert usage_increase < 200

crosshair/auditwall.py

@@ -1,4 +1,5 @@
 import importlib
+import inspect
 import os
 import sys
 import traceback
@@ -28,9 +29,10 @@ def reject(event: str, args: Tuple) -> None:
 
 
 def inside_module(modules: Iterable[ModuleType]) -> bool:
-    files = {m.__file__ for m in modules}
-    for frame, lineno in traceback.walk_stack(None):
-        if frame.f_code.co_filename in files:
+    """Checks whether the current call stack is inside one of the given modules."""
+    for frame, _lineno in traceback.walk_stack(None):
+        frame_module = inspect.getmodule(frame)
+        if frame_module and frame_module in modules:
             return True
     return False
 
@@ -60,7 +62,7 @@ def check_msvcrt_open(event: str, args: Tuple) -> None:
 _MODULES_THAT_CAN_POPEN: Optional[Set[ModuleType]] = None
 
 
-def modules_with_allowed_popen():
+def modules_with_allowed_subprocess():
     global _MODULES_THAT_CAN_POPEN
     if _MODULES_THAT_CAN_POPEN is None:
         allowed_module_names = ("_aix_support", "ctypes", "platform", "uuid")
@@ -74,13 +76,14 @@ def modules_with_allowed_popen():
 
 
 def check_subprocess(event: str, args: Tuple) -> None:
-    if not inside_module(modules_with_allowed_popen()):
+    if not inside_module(modules_with_allowed_subprocess()):
         reject(event, args)
 
 
 _SPECIAL_HANDLERS = {
     "open": check_open,
     "subprocess.Popen": check_subprocess,
+    "os.posix_spawn": check_subprocess,
     "msvcrt.open_osfhandle": check_msvcrt_open,
 }
 
@@ -135,7 +138,6 @@ def make_handler(event: str) -> Callable[[str, Tuple], None]:
         "imaplib",
         "msvcrt",
         "nntplib",
-        "os",
         "pathlib",
         "poplib",
         "shutil",
@@ -179,8 +181,7 @@ def opened_auditwall() -> Generator:
 
 def engage_auditwall() -> None:
     sys.dont_write_bytecode = True  # disable .pyc file writing
-    if sys.version_info >= (3, 8):  # audithook is new in 3.8
-        sys.addaudithook(audithook)
+    sys.addaudithook(audithook)
 
 
 def disable_auditwall() -> None:

crosshair/auditwall_test.py

@@ -11,33 +11,44 @@ from crosshair.auditwall import SideEffectDetected, engage_auditwall
 # audit hooks cannot be uninstalled, and we don't want to wall off the
 # testing process. Spawn subprcoesses instead.
 
-if sys.version_info >= (3, 8):  # audithook is new in 3.8
-    pyexec = sys.executable
+pyexec = sys.executable
 
-    def test_fs_read_allowed():
-        assert call([pyexec, __file__, "read_open", "withwall"]) != 10
 
-    def test_scandir_allowed():
-        assert call([pyexec, __file__, "scandir", "withwall"]) == 0
+def test_fs_read_allowed():
+    assert call([pyexec, __file__, "read_open", "withwall"]) != 10
 
-    def test_import_allowed():
-        assert call([pyexec, __file__, "import", "withwall"]) == 0
 
-    def test_fs_write_disallowed():
-        assert call([pyexec, __file__, "write_open", "withwall"]) == 10
+def test_scandir_allowed():
+    assert call([pyexec, __file__, "scandir", "withwall"]) == 0
 
-    def test_http_disallowed():
-        assert call([pyexec, __file__, "http", "withwall"]) == 10
 
-    def test_unlink_disallowed():
-        assert call([pyexec, __file__, "unlink", "withwall"]) == 10
+def test_import_allowed():
+    assert call([pyexec, __file__, "import", "withwall"]) == 0
 
-    def test_popen_disallowed():
-        assert call([pyexec, __file__, "popen", "withwall"]) == 10
 
-    @pytest.mark.skipif(sys.version_info < (3, 9), reason="Python 3.9+ required")
-    def test_popen_via_platform_allowed():
-        assert call([pyexec, __file__, "popen_via_platform", "withwall"]) == 0
+def test_fs_write_disallowed():
+    assert call([pyexec, __file__, "write_open", "withwall"]) == 10
+
+
+def test_http_disallowed():
+    assert call([pyexec, __file__, "http", "withwall"]) == 10
+
+
+def test_unlink_disallowed():
+    assert call([pyexec, __file__, "unlink", "withwall"]) == 10
+
+
+def test_popen_disallowed():
+    assert call([pyexec, __file__, "popen", "withwall"]) == 10
+
+
+def test_chdir_allowed():
+    assert call([pyexec, __file__, "chdir", "withwall"]) == 0
+
+
+@pytest.mark.skipif(sys.version_info < (3, 9), reason="Python 3.9+ required")
+def test_popen_via_platform_allowed():
+    assert call([pyexec, __file__, "popen_via_platform", "withwall"]) == 0
 
 
 _ACTIONS = {
@@ -51,6 +62,7 @@ _ACTIONS = {
     "popen_via_platform": lambda: platform._syscmd_ver(  # type: ignore
         supported_platforms=(sys.platform,)
     ),
+    "chdir": lambda: os.chdir("."),
 }
 
 if __name__ == "__main__":

crosshair/codeconfig.py

@@ -1,4 +1,5 @@
 """Configure analysis options at different levels."""
+
 import importlib.resources
 import inspect
 import re
@@ -25,7 +26,7 @@ def get_directives(source_text: str) -> Iterable[Tuple[int, int, str]]:
     ret = []
     tokens = tokenize.generate_tokens(StringIO(source_text).readline)
     # TODO catch tokenize.TokenError ... just in case?
-    for (toktyp, tokval, begin, _, _) in tokens:
+    for toktyp, tokval, begin, _, _ in tokens:
         linenum, colnum = begin
         if toktyp == tokenize.COMMENT:
             directive = _COMMENT_TOKEN_RE.sub(r"\1", tokval)
@@ -39,7 +40,7 @@ class InvalidDirective(Exception):
 
 
 def parse_directives(
-    directive_lines: Iterable[Tuple[int, int, str]]
+    directive_lines: Iterable[Tuple[int, int, str]],
 ) -> AnalysisOptionSet:
     """
     Parse options from directives in comments.