offline-debug 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {offline_debug-0.1.0 → offline_debug-0.1.1}/PKG-INFO +1 -1
- {offline_debug-0.1.0 → offline_debug-0.1.1}/offline_debug/__init__.py +2 -1
- offline_debug-0.1.1/offline_debug/_inner/__init__.py +0 -0
- offline_debug-0.1.1/offline_debug/_inner/c_api/__init__.py +4 -0
- offline_debug-0.1.1/offline_debug/_inner/c_api/_create_frame.py +93 -0
- offline_debug-0.1.1/offline_debug/_inner/c_api/_link_frame.py +128 -0
- offline_debug-0.1.1/offline_debug/_inner/load_traceback.py +129 -0
- offline_debug-0.1.1/offline_debug/_inner/models.py +29 -0
- offline_debug-0.1.1/offline_debug/_inner/save_traceback.py +98 -0
- {offline_debug-0.1.0 → offline_debug-0.1.1}/pyproject.toml +2 -1
- offline_debug-0.1.0/offline_debug/serializer.py +0 -297
- {offline_debug-0.1.0 → offline_debug-0.1.1}/README.md +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Tool for serializing and reconstructing Python exceptions with full stack traces."""
|
|
2
2
|
|
|
3
|
-
from .
|
|
3
|
+
from ._inner.load_traceback import load_traceback
|
|
4
|
+
from ._inner.save_traceback import save_traceback
|
|
4
5
|
|
|
5
6
|
__all__ = ["load_traceback", "save_traceback"]
|
|
File without changes
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Abstraction of the CPython api for creating a traceback frame object.
|
|
3
|
+
|
|
4
|
+
The reason we have to implement it, is that cpython doesn't expose
|
|
5
|
+
the api to create a FrameType using pure python.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import ctypes
|
|
11
|
+
import sys
|
|
12
|
+
from functools import cache
|
|
13
|
+
from types import CodeType, FrameType
|
|
14
|
+
from typing import TYPE_CHECKING, Any
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
import _ctypes
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@cache
|
|
21
|
+
def _get_py_frame_new() -> ctypes._NamedFuncPointer:
|
|
22
|
+
"""Get the PyFrame_New C function configured with ctypes."""
|
|
23
|
+
func: ctypes._NamedFuncPointer = ctypes.pythonapi.PyFrame_New
|
|
24
|
+
func.argtypes = (
|
|
25
|
+
ctypes.c_void_p, # PyThreadState *tstate
|
|
26
|
+
ctypes.py_object, # PyCodeObject *code
|
|
27
|
+
ctypes.py_object, # PyObject *globals
|
|
28
|
+
ctypes.py_object, # PyObject *locals
|
|
29
|
+
)
|
|
30
|
+
func.restype = ctypes.py_object
|
|
31
|
+
|
|
32
|
+
def errcheck[T](
|
|
33
|
+
result: T | None,
|
|
34
|
+
_func: _ctypes.CFuncPtr,
|
|
35
|
+
_args: tuple[Any, ...],
|
|
36
|
+
) -> T: # pragma: no cover
|
|
37
|
+
if not result:
|
|
38
|
+
msg = "failed to create a new frame while calling"
|
|
39
|
+
raise RuntimeError(msg)
|
|
40
|
+
return result
|
|
41
|
+
|
|
42
|
+
func.errcheck = errcheck
|
|
43
|
+
return func
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@cache
|
|
47
|
+
def _get_py_thread_state_get() -> ctypes._NamedFuncPointer:
|
|
48
|
+
"""Get the PyThreadState_Get C function configured with ctypes."""
|
|
49
|
+
func = ctypes.pythonapi.PyThreadState_Get
|
|
50
|
+
func.argtypes = ()
|
|
51
|
+
func.restype = ctypes.c_void_p
|
|
52
|
+
|
|
53
|
+
def errcheck[T](
|
|
54
|
+
result: T | None, _func: _ctypes.CFuncPtr, _args: tuple[Any, ...]
|
|
55
|
+
) -> T: # pragma: no cover
|
|
56
|
+
if not result:
|
|
57
|
+
msg = "failed to get the current thread state"
|
|
58
|
+
raise RuntimeError(msg)
|
|
59
|
+
return result
|
|
60
|
+
|
|
61
|
+
func.errcheck = errcheck
|
|
62
|
+
return func
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def create_frame(
|
|
66
|
+
code: CodeType,
|
|
67
|
+
frame_globals: dict[str, Any],
|
|
68
|
+
frame_locals: dict[str, Any],
|
|
69
|
+
thread_state: int | None = None,
|
|
70
|
+
) -> FrameType:
|
|
71
|
+
"""
|
|
72
|
+
Create a FrameType object using the received parameters.
|
|
73
|
+
|
|
74
|
+
If the thread state received is None, use the current thread state automatically.
|
|
75
|
+
"""
|
|
76
|
+
py_frame_new = _get_py_frame_new()
|
|
77
|
+
py_thread_state_get = _get_py_thread_state_get()
|
|
78
|
+
|
|
79
|
+
if thread_state is None:
|
|
80
|
+
thread_state = py_thread_state_get()
|
|
81
|
+
|
|
82
|
+
frame: FrameType = py_frame_new(thread_state, code, frame_globals, frame_locals)
|
|
83
|
+
|
|
84
|
+
# In 3.13+, PEP 667 allows safe write-through access to locals.
|
|
85
|
+
# py_frame_new ignores the frame_locals argument,
|
|
86
|
+
# and must be assigned after the frame's creation in 3.13+
|
|
87
|
+
if sys.version_info >= (3, 13) and frame_locals:
|
|
88
|
+
frame.f_locals.update(frame_locals)
|
|
89
|
+
|
|
90
|
+
if not isinstance(frame, FrameType):
|
|
91
|
+
msg = f"Expected types.FrameType, but got {type(frame).__name__}"
|
|
92
|
+
raise TypeError(msg)
|
|
93
|
+
return frame
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Link 2 frames together, so that a linked chain of frames will eventually create a traceback.
|
|
3
|
+
|
|
4
|
+
The native cpython api does not expose a way to link frames together
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import ctypes
|
|
10
|
+
import sys
|
|
11
|
+
from functools import cache
|
|
12
|
+
from typing import TYPE_CHECKING, Any
|
|
13
|
+
|
|
14
|
+
from ._create_frame import create_frame
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
import _ctypes
|
|
18
|
+
from types import FrameType
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@cache
|
|
22
|
+
def _get_py_incref() -> ctypes._NamedFuncPointer:
|
|
23
|
+
"""
|
|
24
|
+
Get the Py_IncRef C function configured with ctypes.
|
|
25
|
+
|
|
26
|
+
Py_IncRef increases the reference count of an object.
|
|
27
|
+
We call this function when we want to "own" an object,
|
|
28
|
+
meaning that the object won't be deleted while we are still using it.
|
|
29
|
+
|
|
30
|
+
In our case, we use the function to hold the f_back reference to link between frames.
|
|
31
|
+
"""
|
|
32
|
+
func: ctypes._NamedFuncPointer = ctypes.pythonapi.Py_IncRef
|
|
33
|
+
func.argtypes = (ctypes.py_object,)
|
|
34
|
+
func.restype = None
|
|
35
|
+
|
|
36
|
+
def errcheck[T](
|
|
37
|
+
result: T,
|
|
38
|
+
_func: _ctypes.CFuncPtr,
|
|
39
|
+
_args: tuple[Any, ...],
|
|
40
|
+
) -> T: # pragma: no cover
|
|
41
|
+
if result is not None:
|
|
42
|
+
msg = f"Unexpected {result=}, expected None."
|
|
43
|
+
raise TypeError(msg)
|
|
44
|
+
return result
|
|
45
|
+
|
|
46
|
+
func.errcheck = errcheck
|
|
47
|
+
return func
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def link_frame(frame: FrameType, f_back: FrameType) -> None:
|
|
51
|
+
"""Link a frame to its parent frame using the discovered offset."""
|
|
52
|
+
_f_back_offset = _get_f_back_offset()
|
|
53
|
+
if _f_back_offset is None:
|
|
54
|
+
msg = "Failed discovering the offset for the f_back property."
|
|
55
|
+
raise RuntimeError(msg)
|
|
56
|
+
|
|
57
|
+
# In Python, setting f_back means the child frame now owns a reference
|
|
58
|
+
# to the parent frame. We must increment the reference count of the
|
|
59
|
+
# parent to reflect this.
|
|
60
|
+
py_incref = _get_py_incref()
|
|
61
|
+
py_incref(f_back)
|
|
62
|
+
|
|
63
|
+
# Use ctypes to write the address of the back frame into the discovered offset.
|
|
64
|
+
# We find the address of the f_back property, by finding the start of the frame struct,
|
|
65
|
+
# Then moving to the f_back offset in the struct.
|
|
66
|
+
ptr = ctypes.c_void_p.from_address(id(frame) + _f_back_offset)
|
|
67
|
+
ptr.value = id(f_back)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@cache
|
|
71
|
+
def _get_f_back_offset() -> int | None:
|
|
72
|
+
"""
|
|
73
|
+
Dynamically discover the memory offset of f_back in PyFrameObject.
|
|
74
|
+
|
|
75
|
+
The offset for the f_back can change between python versions and operating systems,
|
|
76
|
+
So we find the location of the f_back property dynamically.
|
|
77
|
+
The general idea of the algorithm is to create a mock frame,
|
|
78
|
+
And scanning its memory until we hit the f_back property field.
|
|
79
|
+
We know we hit the f_back property by
|
|
80
|
+
setting the value of each slot in the memeory to another frame, and checking if the
|
|
81
|
+
f_back property was populated after changing that memory location.
|
|
82
|
+
"""
|
|
83
|
+
try:
|
|
84
|
+
# Compile an empty code object that we can use to create a frame.
|
|
85
|
+
code = compile("pass", "<discovery>", "exec")
|
|
86
|
+
|
|
87
|
+
# Create 2 frames that we can attempt linking between to find the f_back property.
|
|
88
|
+
frame = create_frame(code=code, frame_globals={}, frame_locals={})
|
|
89
|
+
f_back_frame = create_frame(code=code, frame_globals={}, frame_locals={})
|
|
90
|
+
|
|
91
|
+
ptr_size = ctypes.sizeof(ctypes.c_void_p)
|
|
92
|
+
jump_size = ptr_size
|
|
93
|
+
|
|
94
|
+
ref_count_size = ptr_size # The amount of references to the object
|
|
95
|
+
type_object_size = ptr_size
|
|
96
|
+
frame_header_size = ref_count_size + type_object_size
|
|
97
|
+
start = frame_header_size # The header does not contain the f_back property
|
|
98
|
+
|
|
99
|
+
frame_size = sys.getsizeof(frame)
|
|
100
|
+
|
|
101
|
+
# We scan the memory ptr_size bytes forward, so we decrease it from the end.
|
|
102
|
+
# Since we want the last memory slot to be scanned as well,
|
|
103
|
+
# we add +1 because the for loop is exclusive to the end.
|
|
104
|
+
end = frame_size - ptr_size + 1
|
|
105
|
+
|
|
106
|
+
# We start scanning after the PyObject header (refcnt + type).
|
|
107
|
+
for offset in range(start, end, jump_size):
|
|
108
|
+
candidate_f_back_address = id(frame) + offset
|
|
109
|
+
candidate_f_back_ptr = ctypes.c_ssize_t.from_address(candidate_f_back_address)
|
|
110
|
+
|
|
111
|
+
# f_back is initially NULL (0) in a newly created frame,
|
|
112
|
+
# since no other frames are linked to it.
|
|
113
|
+
if candidate_f_back_ptr.value != 0:
|
|
114
|
+
continue
|
|
115
|
+
|
|
116
|
+
candidate_f_back_ptr.value = id(f_back_frame)
|
|
117
|
+
# If reading f_back via Python now returns our target, we found it.
|
|
118
|
+
if frame.f_back is f_back_frame:
|
|
119
|
+
# Success, but we must restore 0 so we don't mess up refcounts
|
|
120
|
+
# when 'frame' is eventually garbage collected.
|
|
121
|
+
candidate_f_back_ptr.value = 0
|
|
122
|
+
return offset
|
|
123
|
+
# Restore to 0 if this wasn't the correct offset.
|
|
124
|
+
candidate_f_back_ptr.value = 0
|
|
125
|
+
|
|
126
|
+
except Exception: # noqa: BLE001
|
|
127
|
+
return None
|
|
128
|
+
return None
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Load traceback object from a dump file."""
|
|
2
|
+
|
|
3
|
+
import marshal
|
|
4
|
+
import pickle
|
|
5
|
+
import sys
|
|
6
|
+
import types
|
|
7
|
+
from io import BytesIO
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from types import CodeType
|
|
10
|
+
from typing import Never
|
|
11
|
+
|
|
12
|
+
from offline_debug._inner.c_api import (
|
|
13
|
+
create_frame,
|
|
14
|
+
link_frame,
|
|
15
|
+
)
|
|
16
|
+
from offline_debug._inner.models import (
|
|
17
|
+
_ExceptionData,
|
|
18
|
+
_FrameData,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _reconstruct_exc_data(data: _ExceptionData) -> BaseException:
|
|
23
|
+
"""
|
|
24
|
+
Recursively reconstruct an exception from its serialized data.
|
|
25
|
+
|
|
26
|
+
Note on Python Locals:
|
|
27
|
+
Python uses two ways to store local variables:
|
|
28
|
+
1. "Slow" locals: A dictionary used for module-level code and class definitions.
|
|
29
|
+
2. "Fast" locals: A fixed-size array used for functions. This is faster than
|
|
30
|
+
dictionary lookups because variables are accessed by index.
|
|
31
|
+
|
|
32
|
+
During reconstruction, we must explicitly synchronize these because PyFrame_New
|
|
33
|
+
does not automatically populate the "fast" locals array from a dictionary.
|
|
34
|
+
"""
|
|
35
|
+
exc: BaseException = pickle.loads(data.exc_pickle) # noqa: S301
|
|
36
|
+
if not isinstance(exc, BaseException):
|
|
37
|
+
msg = f"Expected BaseException, but got {type(exc).__name__}"
|
|
38
|
+
raise TypeError(msg)
|
|
39
|
+
|
|
40
|
+
reconstructed_frames: list[tuple[types.FrameType, _FrameData]] = []
|
|
41
|
+
for f_data in data.tb_frames:
|
|
42
|
+
code: CodeType = marshal.loads(f_data.code) # noqa: S302
|
|
43
|
+
|
|
44
|
+
# In Python 3.11 and 3.12, accessing f_locals on a frame created via
|
|
45
|
+
# PyFrame_New for optimized code (functions) causes a segmentation fault
|
|
46
|
+
# because the internal 'fast' locals array is not initialized.
|
|
47
|
+
# As a workaround, we create a 'non-optimized' version of the code object
|
|
48
|
+
# by compiling a dummy string. This ensures the bytecode is safe
|
|
49
|
+
# (no LOAD_FAST) while preserving metadata like name and filename.
|
|
50
|
+
if sys.version_info < (3, 13):
|
|
51
|
+
# A simple module-level code object never has fast locals.
|
|
52
|
+
# Since the source is empty, no optimized locals will be created.
|
|
53
|
+
# Instead, python will go to the unoptimized dictionary we set under frame_locals later.
|
|
54
|
+
unoptimized_code = compile("", code.co_filename, "exec")
|
|
55
|
+
code = unoptimized_code.replace(
|
|
56
|
+
co_name=code.co_name,
|
|
57
|
+
co_firstlineno=code.co_firstlineno,
|
|
58
|
+
co_qualname=code.co_qualname,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# PyFrame_New returns a new reference to a PyFrameObject.
|
|
62
|
+
if f_data.module_name:
|
|
63
|
+
f_data.globals["__name__"] = f_data.module_name
|
|
64
|
+
|
|
65
|
+
frame: types.FrameType = create_frame(
|
|
66
|
+
code=code, frame_globals=f_data.globals, frame_locals=f_data.locals
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
if reconstructed_frames:
|
|
70
|
+
# link the frame back to the previously constructed frame.
|
|
71
|
+
link_frame(frame, reconstructed_frames[-1][0])
|
|
72
|
+
|
|
73
|
+
reconstructed_frames.append((frame, f_data))
|
|
74
|
+
|
|
75
|
+
tb_next: types.TracebackType | None = None
|
|
76
|
+
for frame, f_data in reversed(reconstructed_frames):
|
|
77
|
+
tb = types.TracebackType(
|
|
78
|
+
tb_next=tb_next,
|
|
79
|
+
tb_frame=frame,
|
|
80
|
+
tb_lasti=f_data.lasti,
|
|
81
|
+
tb_lineno=f_data.lineno,
|
|
82
|
+
)
|
|
83
|
+
tb_next = tb
|
|
84
|
+
|
|
85
|
+
exc = exc.with_traceback(tb_next)
|
|
86
|
+
|
|
87
|
+
if data.cause:
|
|
88
|
+
exc.__cause__ = _reconstruct_exc_data(data.cause)
|
|
89
|
+
if data.context:
|
|
90
|
+
exc.__context__ = _reconstruct_exc_data(data.context)
|
|
91
|
+
|
|
92
|
+
return exc
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def load_traceback(file: Path | BytesIO) -> Never:
|
|
96
|
+
"""Load an exception and its traceback from a file and raise it."""
|
|
97
|
+
if isinstance(file, Path):
|
|
98
|
+
with file.open("rb") as f:
|
|
99
|
+
data = pickle.load(f) # noqa: S301
|
|
100
|
+
else:
|
|
101
|
+
data = pickle.load(file) # noqa: S301
|
|
102
|
+
|
|
103
|
+
if not isinstance(data, _ExceptionData):
|
|
104
|
+
msg = f"Expected _ExceptionData, but got {type(data).__name__}"
|
|
105
|
+
raise TypeError(msg)
|
|
106
|
+
|
|
107
|
+
exc = _reconstruct_exc_data(data)
|
|
108
|
+
|
|
109
|
+
current_frames: list[types.FrameType] = []
|
|
110
|
+
curr: types.FrameType | None = sys._getframe(1) # noqa: SLF001
|
|
111
|
+
while curr:
|
|
112
|
+
current_frames.append(curr)
|
|
113
|
+
curr = curr.f_back
|
|
114
|
+
|
|
115
|
+
if exc.__traceback__ and current_frames:
|
|
116
|
+
reconstructed_outer = exc.__traceback__.tb_frame
|
|
117
|
+
link_frame(reconstructed_outer, current_frames[0])
|
|
118
|
+
|
|
119
|
+
tb_chain: types.TracebackType | None = exc.__traceback__
|
|
120
|
+
for frame in current_frames:
|
|
121
|
+
tb_chain = types.TracebackType(
|
|
122
|
+
tb_next=tb_chain,
|
|
123
|
+
tb_frame=frame,
|
|
124
|
+
tb_lasti=frame.f_lasti,
|
|
125
|
+
tb_lineno=frame.f_lineno,
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
exc = exc.with_traceback(tb_chain)
|
|
129
|
+
raise exc
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Functions for serializing and reconstructing exceptions with their tracebacks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class _FrameData:
|
|
11
|
+
"""Serialized data for a single stack frame."""
|
|
12
|
+
|
|
13
|
+
code: bytes
|
|
14
|
+
globals: dict[str, Any]
|
|
15
|
+
locals: dict[str, Any]
|
|
16
|
+
lasti: int
|
|
17
|
+
lineno: int
|
|
18
|
+
stack_depth: int
|
|
19
|
+
module_name: str | None = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class _ExceptionData:
|
|
24
|
+
"""Serialized data for an exception and its traceback."""
|
|
25
|
+
|
|
26
|
+
exc_pickle: bytes
|
|
27
|
+
tb_frames: list[_FrameData]
|
|
28
|
+
cause: _ExceptionData | None = None
|
|
29
|
+
context: _ExceptionData | None = None
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Save traceback to a file."""
|
|
2
|
+
|
|
3
|
+
import marshal
|
|
4
|
+
import pickle
|
|
5
|
+
import types
|
|
6
|
+
from io import BytesIO
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from offline_debug._inner.models import _ExceptionData, _FrameData
|
|
10
|
+
|
|
11
|
+
# Internal attributes that are either unpicklable or redundant in a new process.
|
|
12
|
+
# We exclude these specifically because they are automatically recreated
|
|
13
|
+
# when the new frame is initialized or when the module is imported.
|
|
14
|
+
_INTERNAL_ATTRIBUTES_TO_SKIP = ("__builtins__", "__doc__", "__loader__", "__package__", "__spec__")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _get_stack_depth(frame: types.FrameType) -> int:
|
|
18
|
+
"""Calculate the depth of the current stack frame."""
|
|
19
|
+
depth = 0
|
|
20
|
+
curr: types.FrameType | None = frame
|
|
21
|
+
while curr:
|
|
22
|
+
depth += 1
|
|
23
|
+
curr = curr.f_back
|
|
24
|
+
return depth
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _filter_dict(d: dict) -> dict:
|
|
28
|
+
"""Filter dictionary to include only picklable items."""
|
|
29
|
+
result = {}
|
|
30
|
+
for k, v in d.items():
|
|
31
|
+
if k in _INTERNAL_ATTRIBUTES_TO_SKIP:
|
|
32
|
+
continue
|
|
33
|
+
try:
|
|
34
|
+
# We must verify if the value is picklable because many globals
|
|
35
|
+
# (like open file handles, database connections, or modules)
|
|
36
|
+
# cannot be saved to disk.
|
|
37
|
+
pickle.dumps(v)
|
|
38
|
+
result[k] = v
|
|
39
|
+
except Exception: # noqa: BLE001
|
|
40
|
+
result[k] = f"<unpicklable {type(v).__name__}: {v!r}>"
|
|
41
|
+
return result
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _serialize_exc_data(exc: BaseException) -> _ExceptionData:
|
|
45
|
+
"""Recursively serialize exception data into dataclasses."""
|
|
46
|
+
tb_frames: list[_FrameData] = []
|
|
47
|
+
curr_tb = exc.__traceback__
|
|
48
|
+
while curr_tb:
|
|
49
|
+
f = curr_tb.tb_frame
|
|
50
|
+
|
|
51
|
+
# Try to get the "real" module name. If the module was run as a script,
|
|
52
|
+
# __name__ will be "__main__", but __spec__.name might contain the
|
|
53
|
+
# actual module path if run via `python -m`.
|
|
54
|
+
mod_name = f.f_globals.get("__name__")
|
|
55
|
+
if mod_name == "__main__":
|
|
56
|
+
spec = f.f_globals.get("__spec__")
|
|
57
|
+
if spec and hasattr(spec, "name"):
|
|
58
|
+
mod_name = spec.name
|
|
59
|
+
|
|
60
|
+
tb_frames.append(
|
|
61
|
+
_FrameData(
|
|
62
|
+
code=marshal.dumps(f.f_code),
|
|
63
|
+
globals=_filter_dict(f.f_globals),
|
|
64
|
+
locals=_filter_dict(f.f_locals),
|
|
65
|
+
lasti=curr_tb.tb_lasti,
|
|
66
|
+
lineno=curr_tb.tb_lineno,
|
|
67
|
+
stack_depth=_get_stack_depth(f),
|
|
68
|
+
module_name=mod_name,
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
curr_tb = curr_tb.tb_next
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
exc_pickle = pickle.dumps(exc)
|
|
75
|
+
except Exception: # noqa: BLE001
|
|
76
|
+
exc_pickle = pickle.dumps(
|
|
77
|
+
RuntimeError(f"Unpicklable exception {type(exc).__name__}: {exc!s}")
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
return _ExceptionData(
|
|
81
|
+
exc_pickle=exc_pickle,
|
|
82
|
+
tb_frames=tb_frames,
|
|
83
|
+
cause=_serialize_exc_data(exc.__cause__) if exc.__cause__ else None,
|
|
84
|
+
context=_serialize_exc_data(exc.__context__) if exc.__context__ else None,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def save_traceback(exc: BaseException, file: Path | BytesIO) -> None:
|
|
89
|
+
"""Serialize an exception and its traceback to a file."""
|
|
90
|
+
data = _serialize_exc_data(exc)
|
|
91
|
+
if isinstance(file, Path):
|
|
92
|
+
with file.open("wb") as f:
|
|
93
|
+
pickle.dump(data, f)
|
|
94
|
+
elif isinstance(file, BytesIO):
|
|
95
|
+
pickle.dump(data, file)
|
|
96
|
+
else:
|
|
97
|
+
msg = f"Unexpected type for file {type(file).__name__}"
|
|
98
|
+
raise TypeError(msg)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "offline-debug"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.1"
|
|
4
4
|
description = "Debug exceptions offline by saving them to a dump and raising them at a later point."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
@@ -60,6 +60,7 @@ dev = [
|
|
|
60
60
|
[tool.coverage.report]
|
|
61
61
|
exclude_lines = [
|
|
62
62
|
"pragma: no cover",
|
|
63
|
+
"if TYPE_CHECKING:",
|
|
63
64
|
"if sys.version_info < \\(3, 13\\).*",
|
|
64
65
|
"if sys.version_info >= \\(3, 13\\).*",
|
|
65
66
|
]
|
|
@@ -1,297 +0,0 @@
|
|
|
1
|
-
"""Functions for serializing and reconstructing exceptions with their tracebacks."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import ctypes
|
|
6
|
-
import marshal
|
|
7
|
-
import pickle
|
|
8
|
-
import sys
|
|
9
|
-
import types
|
|
10
|
-
from dataclasses import dataclass
|
|
11
|
-
from pathlib import Path
|
|
12
|
-
from typing import Any, Never
|
|
13
|
-
|
|
14
|
-
# Define C API for frame creation
|
|
15
|
-
_py_frame_new = ctypes.pythonapi.PyFrame_New
|
|
16
|
-
_py_frame_new.argtypes = (
|
|
17
|
-
ctypes.c_void_p, # PyThreadState *tstate
|
|
18
|
-
ctypes.py_object, # PyCodeObject *code
|
|
19
|
-
ctypes.py_object, # PyObject *globals
|
|
20
|
-
ctypes.py_object, # PyObject *locals
|
|
21
|
-
)
|
|
22
|
-
_py_frame_new.restype = ctypes.py_object
|
|
23
|
-
|
|
24
|
-
_py_thread_state_get = ctypes.pythonapi.PyThreadState_Get
|
|
25
|
-
_py_thread_state_get.restype = ctypes.c_void_p
|
|
26
|
-
|
|
27
|
-
_py_incref = ctypes.pythonapi.Py_IncRef
|
|
28
|
-
_py_incref.argtypes = (ctypes.py_object,)
|
|
29
|
-
|
|
30
|
-
_py_decref = ctypes.pythonapi.Py_DecRef
|
|
31
|
-
_py_decref.argtypes = (ctypes.py_object,)
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def _get_f_back_offset() -> int | None:
|
|
35
|
-
"""Dynamically discover the memory offset of f_back in PyFrameObject."""
|
|
36
|
-
try:
|
|
37
|
-
tstate = _py_thread_state_get()
|
|
38
|
-
# Compile a dummy code object that we can use to create a frame.
|
|
39
|
-
code = compile("pass", "<discovery>", "exec")
|
|
40
|
-
# Create a new, detached frame object using the C API.
|
|
41
|
-
frame = _py_frame_new(tstate, code, {}, {})
|
|
42
|
-
if not isinstance(frame, types.FrameType):
|
|
43
|
-
return None
|
|
44
|
-
|
|
45
|
-
# We need a target frame object to point to.
|
|
46
|
-
target = sys._getframe() # noqa: SLF001
|
|
47
|
-
target_addr = id(target)
|
|
48
|
-
|
|
49
|
-
# We scan the frame object's memory for the f_back pointer.
|
|
50
|
-
# We cap the scan at the object's actual size to avoid out-of-bounds reads.
|
|
51
|
-
limit = sys.getsizeof(frame)
|
|
52
|
-
ptr_size = ctypes.sizeof(ctypes.c_void_p)
|
|
53
|
-
|
|
54
|
-
# We start scanning after the PyObject header (refcnt + type).
|
|
55
|
-
for offset in range(2 * ptr_size, limit - ptr_size + 1, ptr_size):
|
|
56
|
-
try:
|
|
57
|
-
# We use c_ssize_t to read the raw value at the offset.
|
|
58
|
-
current_val = ctypes.c_ssize_t.from_address(id(frame) + offset).value
|
|
59
|
-
# f_back is initially NULL (0) in a newly created frame.
|
|
60
|
-
if current_val == 0:
|
|
61
|
-
ctypes.c_ssize_t.from_address(id(frame) + offset).value = target_addr
|
|
62
|
-
# If reading f_back via Python now returns our target, we found it.
|
|
63
|
-
if frame.f_back is target:
|
|
64
|
-
# Success, but we must restore 0 so we don't mess up refcounts
|
|
65
|
-
# when 'frame' is eventually garbage collected.
|
|
66
|
-
ctypes.c_ssize_t.from_address(id(frame) + offset).value = 0
|
|
67
|
-
return offset
|
|
68
|
-
# Restore to 0 if this wasn't the correct offset.
|
|
69
|
-
ctypes.c_ssize_t.from_address(id(frame) + offset).value = 0
|
|
70
|
-
except (AttributeError, ValueError, TypeError, RuntimeError):
|
|
71
|
-
continue
|
|
72
|
-
except Exception: # noqa: BLE001
|
|
73
|
-
return None
|
|
74
|
-
return None
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
_F_BACK_OFFSET = _get_f_back_offset()
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
def _link_frame(frame: types.FrameType, back: types.FrameType) -> None:
|
|
81
|
-
"""Link a frame to its parent frame using the discovered offset."""
|
|
82
|
-
if _F_BACK_OFFSET is None:
|
|
83
|
-
return
|
|
84
|
-
|
|
85
|
-
# In Python, setting f_back means the child frame now owns a reference
|
|
86
|
-
# to the parent frame. We must increment the reference count of the
|
|
87
|
-
# parent to reflect this.
|
|
88
|
-
_py_incref(back)
|
|
89
|
-
|
|
90
|
-
# Use ctypes to write the address of the back frame into the discovered offset.
|
|
91
|
-
ptr = ctypes.c_void_p.from_address(id(frame) + _F_BACK_OFFSET)
|
|
92
|
-
ptr.value = id(back)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
# Internal attributes that are either unpicklable or redundant in a new process.
|
|
96
|
-
# We exclude these specifically because they are automatically recreated
|
|
97
|
-
# when the new frame is initialized or when the module is imported.
|
|
98
|
-
_INTERNAL_ATTRIBUTES_TO_SKIP = ("__builtins__", "__doc__", "__loader__", "__package__", "__spec__")
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
@dataclass
|
|
102
|
-
class _FrameData:
|
|
103
|
-
"""Serialized data for a single stack frame."""
|
|
104
|
-
|
|
105
|
-
code: bytes
|
|
106
|
-
globals: dict[str, Any]
|
|
107
|
-
locals: dict[str, Any]
|
|
108
|
-
lasti: int
|
|
109
|
-
lineno: int
|
|
110
|
-
stack_depth: int
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
@dataclass
|
|
114
|
-
class _ExceptionData:
|
|
115
|
-
"""Serialized data for an exception and its traceback."""
|
|
116
|
-
|
|
117
|
-
exc_pickle: bytes
|
|
118
|
-
tb_frames: list[_FrameData]
|
|
119
|
-
cause: _ExceptionData | None = None
|
|
120
|
-
context: _ExceptionData | None = None
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def _get_stack_depth(frame: types.FrameType) -> int:
|
|
124
|
-
"""Calculate the depth of the current stack frame."""
|
|
125
|
-
depth = 0
|
|
126
|
-
curr: types.FrameType | None = frame
|
|
127
|
-
while curr:
|
|
128
|
-
depth += 1
|
|
129
|
-
curr = curr.f_back
|
|
130
|
-
return depth
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def _filter_dict(d: dict) -> dict:
|
|
134
|
-
"""Filter dictionary to include only picklable items."""
|
|
135
|
-
result = {}
|
|
136
|
-
for k, v in d.items():
|
|
137
|
-
if k in _INTERNAL_ATTRIBUTES_TO_SKIP:
|
|
138
|
-
continue
|
|
139
|
-
try:
|
|
140
|
-
# We must verify if the value is picklable because many globals
|
|
141
|
-
# (like open file handles, database connections, or modules)
|
|
142
|
-
# cannot be saved to disk.
|
|
143
|
-
pickle.dumps(v)
|
|
144
|
-
result[k] = v
|
|
145
|
-
except Exception: # noqa: BLE001
|
|
146
|
-
result[k] = f"<unpicklable {type(v).__name__}: {v!r}>"
|
|
147
|
-
return result
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
def _serialize_exc_data(exc: BaseException) -> _ExceptionData:
|
|
151
|
-
"""Recursively serialize exception data into dataclasses."""
|
|
152
|
-
tb_frames: list[_FrameData] = []
|
|
153
|
-
curr_tb = exc.__traceback__
|
|
154
|
-
while curr_tb:
|
|
155
|
-
f = curr_tb.tb_frame
|
|
156
|
-
tb_frames.append(
|
|
157
|
-
_FrameData(
|
|
158
|
-
code=marshal.dumps(f.f_code),
|
|
159
|
-
globals=_filter_dict(f.f_globals),
|
|
160
|
-
locals=_filter_dict(f.f_locals),
|
|
161
|
-
lasti=curr_tb.tb_lasti,
|
|
162
|
-
lineno=curr_tb.tb_lineno,
|
|
163
|
-
stack_depth=_get_stack_depth(f),
|
|
164
|
-
)
|
|
165
|
-
)
|
|
166
|
-
curr_tb = curr_tb.tb_next
|
|
167
|
-
|
|
168
|
-
try:
|
|
169
|
-
exc_pickle = pickle.dumps(exc)
|
|
170
|
-
except Exception: # noqa: BLE001
|
|
171
|
-
exc_pickle = pickle.dumps(
|
|
172
|
-
RuntimeError(f"Unpicklable exception {type(exc).__name__}: {exc!s}")
|
|
173
|
-
)
|
|
174
|
-
|
|
175
|
-
return _ExceptionData(
|
|
176
|
-
exc_pickle=exc_pickle,
|
|
177
|
-
tb_frames=tb_frames,
|
|
178
|
-
cause=_serialize_exc_data(exc.__cause__) if exc.__cause__ else None,
|
|
179
|
-
context=_serialize_exc_data(exc.__context__) if exc.__context__ else None,
|
|
180
|
-
)
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
def save_traceback(exc: BaseException, file_path: str | Path) -> None:
|
|
184
|
-
"""Serialize an exception and its traceback to a file."""
|
|
185
|
-
data = _serialize_exc_data(exc)
|
|
186
|
-
with Path(file_path).open("wb") as f:
|
|
187
|
-
pickle.dump(data, f)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
def _reconstruct_exc_data(data: _ExceptionData) -> BaseException:
|
|
191
|
-
"""
|
|
192
|
-
Recursively reconstruct an exception from its serialized data.
|
|
193
|
-
|
|
194
|
-
Note on Python Locals:
|
|
195
|
-
Python uses two ways to store local variables:
|
|
196
|
-
1. "Slow" locals: A dictionary used for module-level code and class definitions.
|
|
197
|
-
2. "Fast" locals: A fixed-size array used for functions. This is faster than
|
|
198
|
-
dictionary lookups because variables are accessed by index.
|
|
199
|
-
|
|
200
|
-
During reconstruction, we must explicitly synchronize these because PyFrame_New
|
|
201
|
-
does not automatically populate the "fast" locals array from a dictionary.
|
|
202
|
-
"""
|
|
203
|
-
exc = pickle.loads(data.exc_pickle) # noqa: S301
|
|
204
|
-
if not isinstance(exc, BaseException):
|
|
205
|
-
msg = f"Expected BaseException, but got {type(exc).__name__}"
|
|
206
|
-
raise TypeError(msg)
|
|
207
|
-
|
|
208
|
-
tstate = _py_thread_state_get()
|
|
209
|
-
|
|
210
|
-
reconstructed_frames: list[tuple[types.FrameType, _FrameData]] = []
|
|
211
|
-
prev_frame: types.FrameType | None = None
|
|
212
|
-
for f_data in data.tb_frames:
|
|
213
|
-
code = marshal.loads(f_data.code) # noqa: S302
|
|
214
|
-
|
|
215
|
-
# In Python 3.11 and 3.12, accessing f_locals on a frame created via
|
|
216
|
-
# PyFrame_New for optimized code (functions) causes a segmentation fault
|
|
217
|
-
# because the internal 'fast' locals array is not initialized.
|
|
218
|
-
# As a workaround, we create a 'non-optimized' version of the code object
|
|
219
|
-
# by compiling a dummy string. This ensures the bytecode is safe
|
|
220
|
-
# (no LOAD_FAST) while preserving metadata like name and filename.
|
|
221
|
-
if sys.version_info < (3, 13):
|
|
222
|
-
# A simple module-level code object never has fast locals.
|
|
223
|
-
dummy_code = compile("", code.co_filename, "exec")
|
|
224
|
-
code = dummy_code.replace(
|
|
225
|
-
co_name=code.co_name,
|
|
226
|
-
co_firstlineno=code.co_firstlineno,
|
|
227
|
-
co_qualname=code.co_qualname,
|
|
228
|
-
)
|
|
229
|
-
|
|
230
|
-
# PyFrame_New returns a new reference to a PyFrameObject.
|
|
231
|
-
frame = _py_frame_new(tstate, code, f_data.globals, f_data.locals)
|
|
232
|
-
if not isinstance(frame, types.FrameType):
|
|
233
|
-
msg = f"Expected types.FrameType, but got {type(frame).__name__}"
|
|
234
|
-
raise TypeError(msg)
|
|
235
|
-
|
|
236
|
-
# In 3.13+, PEP 667 allows safe write-through access to locals.
|
|
237
|
-
if sys.version_info >= (3, 13) and f_data.locals:
|
|
238
|
-
frame.f_locals.update(f_data.locals)
|
|
239
|
-
|
|
240
|
-
if prev_frame:
|
|
241
|
-
_link_frame(frame, prev_frame)
|
|
242
|
-
|
|
243
|
-
reconstructed_frames.append((frame, f_data))
|
|
244
|
-
prev_frame = frame
|
|
245
|
-
|
|
246
|
-
tb_next: types.TracebackType | None = None
|
|
247
|
-
for frame, f_data in reversed(reconstructed_frames):
|
|
248
|
-
tb = types.TracebackType(
|
|
249
|
-
tb_next=tb_next,
|
|
250
|
-
tb_frame=frame,
|
|
251
|
-
tb_lasti=f_data.lasti,
|
|
252
|
-
tb_lineno=f_data.lineno,
|
|
253
|
-
)
|
|
254
|
-
tb_next = tb
|
|
255
|
-
|
|
256
|
-
exc = exc.with_traceback(tb_next)
|
|
257
|
-
|
|
258
|
-
if data.cause:
|
|
259
|
-
exc.__cause__ = _reconstruct_exc_data(data.cause)
|
|
260
|
-
if data.context:
|
|
261
|
-
exc.__context__ = _reconstruct_exc_data(data.context)
|
|
262
|
-
|
|
263
|
-
return exc
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
def load_traceback(file_path: str | Path) -> Never:
|
|
267
|
-
"""Load an exception and its traceback from a file and raise it."""
|
|
268
|
-
with Path(file_path).open("rb") as f:
|
|
269
|
-
data = pickle.load(f) # noqa: S301
|
|
270
|
-
|
|
271
|
-
if not isinstance(data, _ExceptionData):
|
|
272
|
-
msg = f"Expected _ExceptionData, but got {type(data).__name__}"
|
|
273
|
-
raise TypeError(msg)
|
|
274
|
-
|
|
275
|
-
exc = _reconstruct_exc_data(data)
|
|
276
|
-
|
|
277
|
-
current_frames: list[types.FrameType] = []
|
|
278
|
-
curr: types.FrameType | None = sys._getframe(1) # noqa: SLF001
|
|
279
|
-
while curr:
|
|
280
|
-
current_frames.append(curr)
|
|
281
|
-
curr = curr.f_back
|
|
282
|
-
|
|
283
|
-
if exc.__traceback__ and current_frames:
|
|
284
|
-
reconstructed_outer = exc.__traceback__.tb_frame
|
|
285
|
-
_link_frame(reconstructed_outer, current_frames[0])
|
|
286
|
-
|
|
287
|
-
tb_chain: types.TracebackType | None = exc.__traceback__
|
|
288
|
-
for frame in current_frames:
|
|
289
|
-
tb_chain = types.TracebackType(
|
|
290
|
-
tb_next=tb_chain,
|
|
291
|
-
tb_frame=frame,
|
|
292
|
-
tb_lasti=frame.f_lasti,
|
|
293
|
-
tb_lineno=frame.f_lineno,
|
|
294
|
-
)
|
|
295
|
-
|
|
296
|
-
exc = exc.with_traceback(tb_chain)
|
|
297
|
-
raise exc
|
|
File without changes
|