pystack 1.3.0__tar.gz → 1.4.1__tar.gz

{pystack-1.3.0 → pystack-1.4.1}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.3.0
+current_version = 1.4.1
 commit = True
 message = 
 	Prepare for {new_version} release

{pystack-1.3.0 → pystack-1.4.1}/Makefile

@@ -93,6 +93,11 @@ docs:  ## Generate documentation
 	$(MAKE) -C docs clean
 	$(MAKE) -C docs html
 
+.PHONY: docs-live
+docs-live:  ## Serve documentation on localhost:8000, with live-reload
+	$(MAKE) -C docs clean
+	$(MAKE) -C docs livehtml
+
 .PHONY: gh-pages
 gh-pages:  ## Publish documentation on GitHub Pages
 	$(eval GIT_REMOTE := $(shell git remote get-url $(UPSTREAM_GIT_REMOTE)))

{pystack-1.3.0 → pystack-1.4.1}/NEWS.rst

@@ -8,6 +8,35 @@ Changelog
 
 .. towncrier release notes start
 
+pystack 1.4.1 (2024-10-04)
+--------------------------
+
+Features
+~~~~~~~~
+
+- Improve our ability to debug Python 3.13 processes when the ``_Py_DebugOffsets`` cannot be located by accounting for a structure layout change in CPython 3.13.0rc3 (the reversion of the incremental cyclic garbage collector). (#213)
+
+
+pystack 1.4.0 (2024-09-11)
+--------------------------
+
+Features
+~~~~~~~~
+
+- Add support for Python 3.13 (#186)
+- Add support for gzip compressed corefiles (#171)
+- Add a new ``--version`` command line flag to show the version of pystack (#203)
+- Support debugging free-threading (a.k.a. "nogil") Python 3.13 builds. Note that PyStack can't itself be run with ``python3.13t``, it can only attach to a ``python3.13t`` process or core file from another interpreter. (#206)
+
+
+Bug Fixes
+~~~~~~~~~
+
+- Fix a bug that was causing Python scripts executed directly via shebang to report the shell script as the executable. (#184)
+- Heap corruption could cause PyStack to fail to generate a stack when ``--locals`` mode was used. This has been fixed by falling back to a reasonable default when attempting to format the repr of a local variable causes a dereference of an invalid pointer. (#194)
+- Fix a crash when analysing processes where the eval loop has a shim frame at the bottom of the stack (#204)
+
+
 pystack 1.3.0 (2023-11-28)
 --------------------------
 

{pystack-1.3.0 → pystack-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pystack
-Version: 1.3.0
+Version: 1.4.1
 Summary: Analysis of the stack of remote python processes
 Home-page: https://github.com/bloomberg/pystack
 Author: Pablo Galindo Salgado
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Software Development :: Debuggers
 Requires-Python: >=3.7.0

{pystack-1.3.0 → pystack-1.4.1}/pyproject.toml

@@ -46,6 +46,7 @@ underlines = "-~"
 build = ["cp38-*", "cp39-*", "cp310-*", "cp311-*"]
 manylinux-x86_64-image = "manylinux2014"
 manylinux-i686-image = "manylinux2014"
+musllinux-x86_64-image = "musllinux_1_1"
 skip = "*-musllinux_aarch64"
 
 [tool.cibuildwheel.linux]
@@ -115,3 +116,8 @@ omit = [
 
 [tool.coverage.report]
 show_missing = true
+
+[tool.pytest.ini_options]
+# pytest retains all temp files from the last 3 test suite runs by default.
+# Keep only ones for failed tests to avoid filling up a disk.
+tmp_path_retention_policy = "failed"

{pystack-1.3.0 → pystack-1.4.1}/setup.py

@@ -115,6 +115,7 @@ setuptools.setup(
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "Programming Language :: Python :: Implementation :: CPython",
         "Topic :: Software Development :: Debuggers",
     ],

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/__main__.py

@@ -12,8 +12,11 @@ from typing import NoReturn
 from typing import Optional
 from typing import Set
 
+from pystack import __version__
 from pystack.errors import InvalidPythonProcess
+from pystack.process import decompress_gzip
 from pystack.process import is_elf
+from pystack.process import is_gzip
 
 from . import errors
 from . import print_thread
@@ -116,6 +119,9 @@ def generate_cli_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "-v", "--verbose", action="count", default=0, dest="global_verbose"
     )
+    parser.add_argument(
+        "--version", action="version", version=__version__, help="Show version"
+    )
     parser.add_argument(
         "--no-color",
         action="store_true",
@@ -319,9 +325,32 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N
     if not corefile.exists():
         parser.error(f"Core {corefile} does not exist")
 
+    if is_gzip(corefile):
+        corefile = decompress_gzip(corefile)
+
     if args.executable is None:
         corefile_analyzer = CoreFileAnalyzer(corefile)
         executable = pathlib.Path(corefile_analyzer.extract_executable())
+        if not executable.exists() or not is_elf(executable):
+            first_map = next(
+                (
+                    map
+                    for map in corefile_analyzer.extract_maps()
+                    if map.path is not None
+                ),
+                None,
+            )
+            if (
+                first_map is not None
+                and first_map.path is not None
+                and first_map.path.exists()
+                and is_elf(first_map.path)
+            ):
+                executable = first_map.path
+                LOGGER.info(
+                    "Setting executable automatically to the first map in the core: %s",
+                    executable,
+                )
         if not executable.exists():
             raise errors.DetectedExecutableNotFound(
                 f"Detected executable doesn't exist: {executable}"

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/_pystack/corefile.cpp

@@ -1,3 +1,4 @@
+#include <algorithm>
 #include <cassert>
 #include <cstring>
 #include <fstream>

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/_pystack/cpython/code.h

@@ -160,13 +160,41 @@ typedef struct
 } PyCodeObject;
 }  // namespace Python3_12
 
-typedef union {
-    Python2::PyCodeObject v2;
-    Python3_3::PyCodeObject v3_3;
-    Python3_6::PyCodeObject v3_6;
-    Python3_8::PyCodeObject v3_8;
-    Python3_11::PyCodeObject v3_11;
-    Python3_12::PyCodeObject v3_12;
+namespace Python3_13 {
+typedef uint16_t _Py_CODEUNIT;
+typedef struct
+{
+    PyObject_VAR_HEAD;
+    PyObject* co_consts;
+    PyObject* co_names;
+    PyObject* co_exceptiontable;
+    int co_flags;
+    int co_argcount;
+    int co_posonlyargcount;
+    int co_kwonlyargcount;
+    int co_stacksize;
+    int co_firstlineno;
+    int co_nlocalsplus;
+    int co_framesize;
+    int co_nlocals;
+    int co_ncellvars;
+    int co_nfreevars;
+    uint32_t co_version;
+    PyObject* co_localsplusnames;
+    PyObject* co_localspluskinds;
+    PyObject* co_filename;
+    PyObject* co_name;
+    PyObject* co_qualname;
+    PyObject* co_linetable;
+    PyObject* co_weakreflist;
+    void* co_executors;
+    void* _co_cached;
+    uintptr_t _co_instrumentation_version;
+    void* _co_monitoring;
+    int _co_firsttraceable;
+    void* co_extra;
+    char co_code_adaptive[1];
 } PyCodeObject;
+}  // namespace Python3_13
 
 }  // namespace pystack

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/_pystack/cpython/dict.h

@@ -47,6 +47,11 @@ typedef struct
     PyObject** ma_values;
 } PyDictObject;
 
+typedef struct _dictvalues
+{
+    PyObject* values[1];
+} PyDictValuesObject;
+
 }  // namespace Python3
 namespace Python3_3 {
 typedef struct _dictkeysobject
@@ -77,8 +82,21 @@ typedef struct _dictkeysobject
     uint32_t dk_version;
     Py_ssize_t dk_usable;
     Py_ssize_t dk_nentries;
-    char dk_indices[]; /* char is required to avoid strict aliasing. */
+    char dk_indices[1]; /* char is required to avoid strict aliasing. */
 } PyDictKeysObject;
 }  // namespace Python3_11
 
+namespace Python3_13 {
+
+typedef struct _dictvalues
+{
+    uint8_t capacity;
+    uint8_t size;
+    uint8_t embedded;
+    uint8_t valid;
+    PyObject* values[1];
+} PyDictValuesObject;
+
+}  // namespace Python3_13
+
 }  // namespace pystack

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/_pystack/cpython/frame.h

@@ -39,7 +39,7 @@ namespace Python3_7 {
 typedef struct _pyframeobject
 {
     PyObject_VAR_HEAD struct _pyframeobject* f_back;
-    PyCodeObject* f_code;
+    PyObject* f_code;
     PyObject* f_builtins;
     PyObject* f_globals;
     PyObject* f_locals;
@@ -64,7 +64,7 @@ typedef signed char PyFrameState;
 typedef struct _pyframeobject
 {
     PyObject_VAR_HEAD struct _pyframeobject* f_back;
-    PyCodeObject* f_code;
+    PyObject* f_code;
     PyObject* f_builtins;
     PyObject* f_globals;
     PyObject* f_locals;
@@ -126,12 +126,4 @@ typedef struct _interpreter_frame
 
 }  // namespace Python3_12
 
-typedef union {
-    Python2::PyFrameObject v2;
-    Python3_7::PyFrameObject v3_7;
-    Python3_10::PyFrameObject v3_10;
-    Python3_11::PyFrameObject v3_11;
-    Python3_12::PyFrameObject v3_12;
-} PyFrameObject;
-
 }  // namespace pystack

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/_pystack/cpython/gc.h

@@ -82,9 +82,4 @@ struct _gc_runtime_state
 };
 
 }  // namespace Python3_8
-
-typedef union {
-    struct Python3_7::_gc_runtime_state v3_7;
-    struct Python3_8::_gc_runtime_state v3_8;
-} GCRuntimeState;
 }  // namespace pystack

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/_pystack/cpython/interpreter.h

@@ -247,13 +247,95 @@ typedef struct _is
 } PyInterpreterState;
 }  // namespace Python3_12
 
-typedef union {
-    Python2::PyInterpreterState v2;
-    Python3_5::PyInterpreterState v3_5;
-    Python3_7::PyInterpreterState v3_7;
-    Python3_8::PyInterpreterState v3_8;
-    Python3_9::PyInterpreterState v3_9;
-    Python3_11::PyInterpreterState v3_11;
-    Python3_12::PyInterpreterState v3_12;
+namespace Python3_13 {
+
+struct _pythreadstate; /* Forward */
+struct PyMutex
+{
+    uint8_t v;
+};
+typedef int (*_Py_pending_call_func)(void*);
+struct _pending_call
+{
+    _Py_pending_call_func func;
+    void* arg;
+    int flags;
+};
+struct _pending_calls
+{
+    void* handling_thread;
+    PyMutex mutex;
+    int32_t npending;
+    int32_t max;
+    int32_t maxloop;
+    struct _pending_call calls[300];
+    int first;
+    int next;
+};
+
+struct _ceval_state
+{
+    uintptr_t instrumentation_version;
+    int recursion_limit;
+    struct _gil_runtime_state* gil;
+    int own_gil;
+    struct _pending_calls pending;
+};
+
+struct _import_state
+{
+    PyObject* modules;
+    PyObject* modules_by_index;
+    PyObject* importlib;
+    int override_frozen_modules;
+    int override_multi_interp_extensions_check;
+    int dlopenflags;
+    PyObject* import_func;
+    struct
+    {
+        PyThread_type_lock mutex;
+        unsigned long thread;
+        int level;
+    } lock;
+    struct
+    {
+        int import_level;
+        int64_t accumulated;
+        int header;
+    } find_and_load;
+};
+
+typedef struct _is
+{
+    struct _ceval_state ceval;
+    struct _is* next;
+
+    int64_t id;
+    int64_t id_refcount;
+    int requires_idref;
+    PyThread_type_lock id_mutex;
+    long _whence;
+    int _initialized;
+    int _ready;
+    int finalizing;
+    uintptr_t last_restart_version;
+    struct pythreads
+    {
+        uint64_t next_unique_id;
+        _pythreadstate* head;
+        _pythreadstate* main;
+        Py_ssize_t count;
+        size_t stacksize;
+    } threads;
+    struct pyruntimestate* runtime;
+    void* _finalizing;
+    unsigned long _finalizing_id;
+    struct Python3_8::_gc_runtime_state gc;
+    // Dictionary of the sys module
+    PyObject* sysdict;
+    // Dictionary of the builtins module
+    PyObject* builtins;
+    struct _import_state imports;
 } PyInterpreterState;
+}  // namespace Python3_13
 }  // namespace pystack

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/_pystack/cpython/object.h

@@ -205,12 +205,6 @@ typedef struct _typeobject
 } PyTypeObject;
 }  // namespace Python3_8
 
-typedef union {
-    Python2::PyTypeObject v2;
-    Python3_3::PyTypeObject v3_3;
-    Python3_8::PyTypeObject v3_8;
-} PyTypeObject;
-
 /* These flags are used to determine if a type is a subclass. */
 constexpr long Pystack_TPFLAGS_INT_SUBCLASS = 1ul << 23u;
 constexpr long Pystack_TPFLAGS_LONG_SUBCLASS = 1ul << 24u;

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/_pystack/cpython/runtime.h

@@ -105,13 +105,15 @@ struct _ceval_runtime_state
     struct _gil_runtime_state gil;
 };
 
+struct PyThreadState;
+
 typedef struct pyruntimestate
 {
     int preinitializing;
     int preinitialized;
     int core_initialized;
     int initialized;
-    PyThreadState* finalizing;
+    void* finalizing;
 
     struct pyinterpreters
     {
@@ -171,7 +173,7 @@ typedef struct pyruntimestate
     int preinitialized;
     int core_initialized;
     int initialized;
-    PyThreadState* finalizing;
+    void* finalizing;
 
     struct pyinterpreters
     {
@@ -246,12 +248,187 @@ typedef struct pyruntimestate
 } PyRuntimeState;
 }  // namespace Python3_12
 
+namespace Python3_13 {
+
+typedef struct _Py_DebugOffsets
+{
+    char cookie[8];
+    uint64_t version;
+    uint64_t free_threaded;
+    // Runtime state offset;
+    struct _runtime_state
+    {
+        uint64_t size;
+        uint64_t finalizing;
+        uint64_t interpreters_head;
+    } runtime_state;
+
+    // Interpreter state offset;
+    struct _interpreter_state
+    {
+        uint64_t size;
+        uint64_t id;
+        uint64_t next;
+        uint64_t threads_head;
+        uint64_t gc;
+        uint64_t imports_modules;
+        uint64_t sysdict;
+        uint64_t builtins;
+        uint64_t ceval_gil;
+        uint64_t gil_runtime_state;
+        uint64_t gil_runtime_state_enabled;
+        uint64_t gil_runtime_state_locked;
+        uint64_t gil_runtime_state_holder;
+    } interpreter_state;
+
+    // Thread state offset;
+    struct _thread_state
+    {
+        uint64_t size;
+        uint64_t prev;
+        uint64_t next;
+        uint64_t interp;
+        uint64_t current_frame;
+        uint64_t thread_id;
+        uint64_t native_thread_id;
+        uint64_t datastack_chunk;
+        uint64_t status;
+    } thread_state;
+
+    // InterpreterFrame offset;
+    struct _interpreter_frame
+    {
+        uint64_t size;
+        uint64_t previous;
+        uint64_t executable;
+        uint64_t instr_ptr;
+        uint64_t localsplus;
+        uint64_t owner;
+    } interpreter_frame;
+
+    // Code object offset;
+    struct _code_object
+    {
+        uint64_t size;
+        uint64_t filename;
+        uint64_t name;
+        uint64_t qualname;
+        uint64_t linetable;
+        uint64_t firstlineno;
+        uint64_t argcount;
+        uint64_t localsplusnames;
+        uint64_t localspluskinds;
+        uint64_t co_code_adaptive;
+    } code_object;
+
+    // PyObject offset;
+    struct _pyobject
+    {
+        uint64_t size;
+        uint64_t ob_type;
+    } pyobject;
+
+    // PyTypeObject object offset;
+    struct _type_object
+    {
+        uint64_t size;
+        uint64_t tp_name;
+        uint64_t tp_repr;
+        uint64_t tp_flags;
+    } type_object;
+
+    // PyTuple object offset;
+    struct _tuple_object
+    {
+        uint64_t size;
+        uint64_t ob_item;
+        uint64_t ob_size;
+    } tuple_object;
+
+    // PyList object offset;
+    struct _list_object
+    {
+        uint64_t size;
+        uint64_t ob_item;
+        uint64_t ob_size;
+    } list_object;
+
+    // PyDict object offset;
+    struct _dict_object
+    {
+        uint64_t size;
+        uint64_t ma_keys;
+        uint64_t ma_values;
+    } dict_object;
+
+    // PyFloat object offset;
+    struct _float_object
+    {
+        uint64_t size;
+        uint64_t ob_fval;
+    } float_object;
+
+    // PyLong object offset;
+    struct _long_object
+    {
+        uint64_t size;
+        uint64_t lv_tag;
+        uint64_t ob_digit;
+    } long_object;
+
+    // PyBytes object offset;
+    struct _bytes_object
+    {
+        uint64_t size;
+        uint64_t ob_size;
+        uint64_t ob_sval;
+    } bytes_object;
+
+    // Unicode object offset;
+    struct _unicode_object
+    {
+        uint64_t size;
+        uint64_t state;
+        uint64_t length;
+        uint64_t asciiobject_size;
+    } unicode_object;
+
+    // GC runtime state offset;
+    struct _gc
+    {
+        uint64_t size;
+        uint64_t collecting;
+    } gc;
+} _Py_DebugOffsets;
+
+typedef struct pyruntimestate
+{
+    _Py_DebugOffsets debug_offsets;
+    int _initialized;
+    int preinitializing;
+    int preinitialized;
+    int core_initialized;
+    int initialized;
+    PyThreadState* finalizing;
+    unsigned long _finalizing_id;
+
+    struct pyinterpreters
+    {
+        PyMutex mutex;
+        PyInterpreterState* head;
+        PyInterpreterState* main;
+        int64_t next_id;
+    } interpreters;
+} PyRuntimeState;
+
+}  // namespace Python3_13
 typedef union {
     Python3_7::PyRuntimeState v3_7;
     Python3_8::PyRuntimeState v3_8;
     Python3_9::PyRuntimeState v3_9;
     Python3_11::PyRuntimeState v3_11;
     Python3_12::PyRuntimeState v3_12;
+    Python3_13::PyRuntimeState v3_13;
 } PyRuntimeState;
 
 }  // namespace pystack

{pystack-1.3.0 → pystack-1.4.1}/src/pystack/_pystack/cpython/string.h

@@ -12,11 +12,13 @@ typedef uint8_t Py_UCS1;
 typedef Py_UCS4 Py_UNICODE;
 typedef Py_ssize_t Py_hash_t;
 
+namespace Python3 {
 typedef struct
 {
     PyObject_VAR_HEAD Py_hash_t ob_shash;
     char ob_sval[1];
 } PyBytesObject;
+}  // namespace Python3
 
 namespace Python2 {
 typedef struct
@@ -37,19 +39,21 @@ typedef struct
 
 namespace Python3 {
 
+struct _PyUnicode_State
+{
+    unsigned int interned : 2;
+    unsigned int kind : 3;
+    unsigned int compact : 1;
+    unsigned int ascii : 1;
+    unsigned int ready : 1;
+    unsigned int : 24;
+};
+
 typedef struct
 {
     PyObject_HEAD Py_ssize_t length;
     Py_hash_t hash;
-    struct
-    {
-        unsigned int interned : 2;
-        unsigned int kind : 3;
-        unsigned int compact : 1;
-        unsigned int ascii : 1;
-        unsigned int ready : 1;
-        unsigned int : 24;
-    } state;
+    _PyUnicode_State state;
     wchar_t* wstr;
 } PyASCIIObject;
 
@@ -76,19 +80,13 @@ typedef struct
 
 namespace Python3_12 {
 
+using Python3::_PyUnicode_State;
+
 typedef struct
 {
     PyObject_HEAD Py_ssize_t length;
     Py_hash_t hash;
-    struct
-    {
-        unsigned int interned : 2;
-        unsigned int kind : 3;
-        unsigned int compact : 1;
-        unsigned int ascii : 1;
-        unsigned int ready : 1;
-        unsigned int : 24;
-    } state;
+    _PyUnicode_State state;
 } PyASCIIObject;
 
 typedef struct
@@ -100,6 +98,7 @@ typedef struct
 
 typedef struct
 {
+    PyCompactUnicodeObject _base;
     union {
         void* any;
         Py_UCS1* latin1;
@@ -110,10 +109,4 @@ typedef struct
 
 }  // namespace Python3_12
 
-typedef union {
-    Python2::PyUnicodeObject v2;
-    Python3::PyUnicodeObject v3;
-    Python3_12::PyUnicodeObject v3_12;
-} PyUnicodeObject;
-
 }  // namespace pystack