midden 0.1.0__tar.gz

midden-0.1.0/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: midden
+Version: 0.1.0
+Summary: A simple heap analysis and visualization tool.
+License-Expression: MIT
+Requires-Dist: flask>=3.1.3 ; extra == 'ui'
+Requires-Dist: psutil>=7.2.2 ; extra == 'ui'
+Requires-Dist: midden-analysis==0.1.0 ; extra == 'ui'
+Requires-Dist: cheroot>=11.1.2 ; extra == 'ui'
+Maintainer: James Pickering
+Maintainer-email: James Pickering <james_pic@hotmail.com>
+Requires-Python: >=3.10
+Project-URL: Repository, https://github.com/jamespic/midden
+Project-URL: Issues, https://github.com/jamespic/midden/issues
+Provides-Extra: ui
+Description-Content-Type: text/markdown
+
+Midden
+======
+
+Midden is a tool for dumping and analysing heaps from Python programs.
+
+It takes a "dump first, ask questions later" approach, making it easy to grab a heap dump from a running
+application, then analyse it offline later in its UI, potentially on a different machine entirely.
+
+Installing
+----------
+
+If you just want to grab a heap dump, you can install everything you need with:
+
+```
+pip install midden
+# Or if you're using uv
+uv add midden
+```
+
+If you want to analyse the data, you'll need the extra UI dependencies, which you can install with
+
+```
+pip install midden[ui]
+# Or if you're using uv
+uv add midden[ui]
+```
+
+Grabbing a Heap Dump
+--------------------
+
+Grabbing a heap dump can be as simple as:
+
+```
+# Assuming pid we want to grab heap from is pid 12345
+midden-inject 12345 --output-file /tmp/dump.jsonl
+```
+
+On Python 3.14 and newer, this will use `sys.remote_exec`, which means no extra dependencies.
+You will need appropriate permissions [as documented here](https://docs.python.org/3/howto/remote_debugging.html#permission-requirements).
+Roughly speaking, you either need to be root/administrator, or be running on Linux with ptrace protection disabled.
+
+On Python 3.10 to 3.13, injection is done with gdb, which means gdb needs to be installed.
+Gdb-based injection is only tested on Linux.
+
+Analysing a Heap Dump
+---------------------
+
+You can run the analysis UI with
+
+```
+midden-ui
+```
+
+This will start a web application, and pop up a web browser pointing at the analysis application. Upload a heap dump
+generated with `midden-inject` to get started.

midden-0.1.0/README.md

@@ -0,0 +1,55 @@
+Midden
+======
+
+Midden is a tool for dumping and analysing heaps from Python programs.
+
+It takes a "dump first, ask questions later" approach, making it easy to grab a heap dump from a running
+application, then analyse it offline later in its UI, potentially on a different machine entirely.
+
+Installing
+----------
+
+If you just want to grab a heap dump, you can install everything you need with:
+
+```
+pip install midden
+# Or if you're using uv
+uv add midden
+```
+
+If you want to analyse the data, you'll need the extra UI dependencies, which you can install with
+
+```
+pip install midden[ui]
+# Or if you're using uv
+uv add midden[ui]
+```
+
+Grabbing a Heap Dump
+--------------------
+
+Grabbing a heap dump can be as simple as:
+
+```
+# Assuming pid we want to grab heap from is pid 12345
+midden-inject 12345 --output-file /tmp/dump.jsonl
+```
+
+On Python 3.14 and newer, this will use `sys.remote_exec`, which means no extra dependencies.
+You will need appropriate permissions [as documented here](https://docs.python.org/3/howto/remote_debugging.html#permission-requirements).
+Roughly speaking, you either need to be root/administrator, or be running on Linux with ptrace protection disabled.
+
+On Python 3.10 to 3.13, injection is done with gdb, which means gdb needs to be installed.
+Gdb-based injection is only tested on Linux.
+
+Analysing a Heap Dump
+---------------------
+
+You can run the analysis UI with
+
+```
+midden-ui
+```
+
+This will start a web application, and pop up a web browser pointing at the analysis application. Upload a heap dump
+generated with `midden-inject` to get started.

midden-0.1.0/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "midden"
+version = "0.1.0"
+description = "A simple heap analysis and visualization tool."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+license = "MIT"
+maintainers = [
+    { name = "James Pickering", email = "james_pic@hotmail.com" } 
+]
+
+
+[project.urls]
+Repository = "https://github.com/jamespic/midden"
+Issues = "https://github.com/jamespic/midden/issues"
+
+[project.optional-dependencies]
+ui = [
+    "flask>=3.1.3",
+    "psutil>=7.2.2",
+    "midden-analysis==0.1.0",
+    "cheroot>=11.1.2",
+]
+
+[project.scripts]
+midden-inject = "midden.dump.inject:main"
+midden-ui = "midden.wsgi:main"
+
+[build-system]
+requires = ["uv_build>=0.9.22,<0.10.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = ["docker>=7.1.0", "pytest>=9.0.3", "ruff>=0.15.7", "ty>=0.0.24"]
+
+[tool.pytest.ini_options]
+markers = [
+    "linux: tests that require Linux",
+    "min_python(python_ver): tests that require a minimum Python version",
+]

midden-0.1.0/src/midden/__init__.py

@@ -0,0 +1 @@
+"""Web UI and heap-dump tooling for exploring Python process memory."""

midden-0.1.0/src/midden/dump/__init__.py

@@ -0,0 +1 @@
+"""Helpers for capturing heap dumps from running Python processes."""

midden-0.1.0/src/midden/dump/dump_heap.py

@@ -0,0 +1,210 @@
+"""Script to be injected with either Pyrasite or sys.remote_exec to dump the heap of a running Python process to a file.
+
+The output is a JSONL file where each line is a JSON object representing an object in the heap,
+with its id, type, referers, references, and optionally its value (for simple types)."""
+
+from types import (
+    ModuleType,
+    FunctionType,
+    BuiltinFunctionType,
+    MethodType,
+    WrapperDescriptorType,
+    MethodWrapperType,
+    MethodDescriptorType,
+    ClassMethodDescriptorType,
+    GetSetDescriptorType,
+    MemberDescriptorType,
+)
+
+import gc
+import json
+import os
+import sys
+
+# Max length for string representations of values
+_max_value_len = 1000
+
+DEFAULT_DUMP_FILE_NAME = "/tmp/dump.jsonl"
+
+
+def _dump_heap():
+    """Write a JSONL heap snapshot for the current process to the fixed dump path."""
+    # Get all objects tracked by gc
+    all_objects = gc.get_objects()
+    print(f"Found {len(all_objects)} objects from gc.get_objects()", file=sys.stderr)
+    extra_objects = []
+    object_ids_tracked = set(id(obj) for obj in all_objects)
+
+    # Collect ids of our own data structures so we can exclude them
+    exclude_ids = set(
+        [id(all_objects), id(extra_objects), id(object_ids_tracked), id(_dump_heap)]
+    )
+    exclude_ids.add(
+        id(exclude_ids)
+    )  # Don't forget to exclude the set of excluded ids itself!
+
+    def _maybe_add_ref(ref_obj, refs):
+        ref_id = id(ref_obj)
+        if ref_id not in exclude_ids:
+            refs.append(ref_id)
+            if ref_id not in object_ids_tracked:
+                extra_objects.append(ref_obj)
+                object_ids_tracked.add(ref_id)
+
+    exclude_ids.add(id(_maybe_add_ref))
+
+    def _get_all_references(obj):
+        """Get references from an object, including non-gc-tracked immutables."""
+        refs = []
+        exclude_ids.add(id(refs))
+
+        obj_type = type(obj)
+        if obj_type is dict:
+            for k, v in obj.items():
+                _maybe_add_ref(k, refs)
+                _maybe_add_ref(v, refs)
+        elif obj_type in (list, tuple, frozenset, set):
+            for item in obj:
+                _maybe_add_ref(item, refs)
+        else:
+            # Fall back to gc.get_referents for other types,
+            # which catches __dict__, slots, etc.
+            gc_refs = gc.get_referents(obj)
+            exclude_ids.add(id(gc_refs))
+            for r in gc_refs:
+                _maybe_add_ref(r, refs)
+
+        return refs
+
+    exclude_ids.add(id(_get_all_references))
+
+    try:
+        with open(f"{DEFAULT_DUMP_FILE_NAME}.partial", "w") as f:
+            exclude_ids.add(id(f))
+
+            def dump_object(obj):
+                """Serialize one object unless it belongs to the dumper's bookkeeping."""
+                obj_id = id(obj)
+
+                # Skip our own bookkeeping objects
+                if obj_id in exclude_ids:
+                    return
+
+                # Get references (including non-gc-tracked children)
+                references = _get_all_references(obj)
+
+                type_name = _get_type_name(obj)
+
+                record = {
+                    "id": obj_id,
+                    "type": type_name,
+                    "references": references,
+                    "size": sys.getsizeof(
+                        obj, 0
+                    ),  # Get size of object, excluding referents
+                    # Don't get referrers - it's too slow. We'll index it offline later.
+                }
+
+                # Only include value for whitelisted types
+                if extractor := _value_extractors.get(type(obj)):
+                    try:
+                        record["value"] = extractor(obj)
+                    except Exception as e:
+                        record["value"] = f"<error extracting value: {e}>"
+
+                exclude_ids.add(id(record))
+                exclude_ids.add(id(record.get("references")))
+
+                line = json.dumps(record, default=str)
+                exclude_ids.add(id(line))
+                f.write(line)
+                f.write("\n")
+
+            exclude_ids.add(id(dump_object))
+
+            for obj in all_objects:
+                dump_object(obj)
+            for obj in extra_objects:
+                dump_object(obj)
+
+        os.rename(f"{DEFAULT_DUMP_FILE_NAME}.partial", DEFAULT_DUMP_FILE_NAME)
+
+    except Exception as e:
+        sys.stderr.write(f"dump_heap error: {e}\n")
+
+
+def _get_qualname(obj):
+    """Get a qualified name for an object, if possible."""
+    if qualname := getattr(obj, "__qualname__", None):
+        return qualname
+    elif name := getattr(obj, "__name__", None):
+        return name
+    else:
+        return repr(obj)
+
+
+def _get_prefix(obj):
+    """Get the module name or class name for an object, if possible."""
+    if module := getattr(obj, "__module__", None):
+        return f"{module}."
+    elif obj_class := getattr(obj, "__objclass__", None):
+        return f"{_get_qualname(obj_class)}."
+    else:
+        return ""
+
+
+def _name_extractor(obj):
+    """Return a readable name for callables, descriptors, and types."""
+    return _get_prefix(obj) + _get_qualname(obj)
+
+
+def _get_type_name(obj):
+    """Get a friendly type name for an object."""
+    t = type(obj)
+    return _name_extractor(t)
+
+
+def _string_extractor(obj):
+    """Keep long strings readable by truncating them in the dump."""
+    if len(obj) > _max_value_len:
+        return obj[:_max_value_len] + "...<truncated>"
+    return obj
+
+
+def _bytes_extractor(obj):
+    """Render bytes as repr output, truncating long values."""
+    if len(obj) > _max_value_len:
+        return repr(obj[:_max_value_len]) + "...<truncated>"
+    return repr(obj)
+
+
+def _module_extractor(obj):
+    """Render modules by name rather than by their default repr."""
+    return f"module {obj.__name__}"
+
+
+_value_extractors = {
+    str: _string_extractor,
+    bytes: _bytes_extractor,
+    int: str,
+    float: str,
+    complex: str,
+    bool: str,
+    type(None): lambda x: "None",
+    ModuleType: _module_extractor,
+    FunctionType: _name_extractor,
+    BuiltinFunctionType: _name_extractor,
+    MethodType: _name_extractor,
+    staticmethod: _name_extractor,
+    classmethod: _name_extractor,
+    WrapperDescriptorType: _name_extractor,
+    MethodWrapperType: _name_extractor,
+    MethodDescriptorType: _name_extractor,
+    ClassMethodDescriptorType: _name_extractor,
+    GetSetDescriptorType: _name_extractor,
+    MemberDescriptorType: _name_extractor,
+    type: _name_extractor,
+}
+
+
+# _dump_heap()  # Replaced by the injector so the dump only runs inside the target process.

midden-0.1.0/src/midden/dump/inject.py

@@ -0,0 +1,411 @@
+import datetime
+from contextlib import contextmanager
+import io
+import shutil
+import re
+import tarfile
+import time
+import tempfile
+from subprocess import Popen, PIPE
+import os
+import pathlib
+import sys
+import subprocess
+
+try:
+    from sys import remote_exec  # ty: ignore[unresolved-import]
+except ImportError:
+    remote_exec = None
+
+try:
+    from os import setns  # ty: ignore[unresolved-import]
+except ImportError:
+    setns = None
+
+try:
+    import psutil
+
+    _psutil_available = True
+except ImportError:
+    _psutil_available = False
+
+GIL_ENABLED = True
+try:
+    from sys import _is_gil_enabled  # ty: ignore[unresolved-import]
+
+    GIL_ENABLED = _is_gil_enabled()
+except ImportError:
+    pass
+
+DEFAULT_DUMP_FILE = "/tmp/dump.jsonl"
+
+_FILES_NEEDED_FOR_INJECTION = ["dump_heap.py", "inject.py"]
+
+
+def _build_tarball_of_dumping_code():
+    """Bundle the dump scripts so they can be copied into another namespace."""
+    file_obj = io.BytesIO()
+    with tarfile.open(fileobj=file_obj, mode="w:gz") as tar:
+        for filename in _FILES_NEEDED_FOR_INJECTION:
+            file_path = pathlib.Path(__file__).parent / filename
+            tar.add(file_path, arcname=filename)
+    return file_obj.getvalue()
+
+
+# We do this at import time so that if we fork into another namespace, it's still available
+_TARBALL = _build_tarball_of_dumping_code()
+
+
+def dump_heap_from_pid(
+    pid,
+    output_file=DEFAULT_DUMP_FILE,
+    can_use_namespace_injection=True,
+    can_use_alternate_python_interpreter=True,
+):
+    """Dump the heap of a running Python process given its PID."""
+    _dump_heap_from_pid_possibly_in_namespace(
+        pid,
+        output_file,
+        can_use_namespace_injection,
+        can_use_alternate_python_interpreter,
+    )
+
+
+_DUMP_SCRIPT = (pathlib.Path(__file__).parent / "dump_heap.py").read_text()
+
+
+def _build_dump_heap_code(output_file):
+    """Rewrite the injected script so it writes to the requested output path."""
+    return _DUMP_SCRIPT.replace(f'"{DEFAULT_DUMP_FILE}"', repr(output_file)).replace(
+        "# _dump_heap()", "_dump_heap()"
+    )
+
+
+def _dump_heap_from_pid_possibly_using_an_alternate_python_interpreter(
+    pid,
+    output_file=DEFAULT_DUMP_FILE,
+    can_use_alternate_python_interpreter=True,
+):
+    """Pick a compatible Python interpreter before injecting, if needed."""
+    if can_use_alternate_python_interpreter and (
+        alternate_python := _should_use_alternate_python_interpreter(pid)
+    ):
+        print(
+            f"Using alternate Python interpreter {alternate_python} for injection",
+            file=sys.stderr,
+        )
+        _dump_heap_from_pid_using_alternate_python_interpreter(
+            pid, alternate_python, output_file
+        )
+    else:
+        _dump_heap_from_pid(pid, output_file)
+
+
+def _dump_heap_from_pid_using_alternate_python_interpreter(
+    pid, alternate_python, output_file=DEFAULT_DUMP_FILE
+):
+    """Run the injector with a different Python executable inside a temp copy of the code."""
+    # The target mount namespace may not be able to see our original source tree.
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tarball_file_obj = io.BytesIO(_TARBALL)
+        with tarfile.open(fileobj=tarball_file_obj, mode="r:gz") as tar:
+            tar.extractall(path=tmpdir)
+
+        cmd = [
+            alternate_python,
+            tmpdir + "/inject.py",
+            str(pid),
+            "--output-file",
+            output_file,
+            "--no-alternate-python-interpreter",  # Don't recurse into trying to find another alternate Python interpreter
+            "--no-namespace-injection",  # Don't use namespace injection - this function should only be called after we've sorted out namespaces
+        ]
+        print(f"Running injection command: {' '.join(cmd)}", file=sys.stderr)
+        subprocess.run(cmd, check=True)
+
+
+def _should_use_alternate_python_interpreter(pid) -> str | None:
+    """Return a better-matched Python executable for injection, if one is needed."""
+    try:
+        exe, maps = _get_exe_and_maps(pid)
+    except Exception as e:
+        print(
+            f"Could not determine target process executable and maps, so proceeding with this interpreter: {e!r}",
+            file=sys.stderr,
+        )
+        return None
+    if not _can_this_python_inject(exe):
+        if os.path.basename(exe).startswith("python"):
+            print(
+                f"Using {exe} as alternate Python interpreter based on process exe",
+                file=sys.stderr,
+            )
+            return exe
+        else:
+            # Check maps for a python library to identify Python version
+            for m in maps:
+                if match := re.match(
+                    r".*lib(python\d\.\d.*)\.so", m
+                ):  # Look for a python library in the maps
+                    python_exe_wanted = match.group(1)
+                    if python_exe_wanted == _this_effective_executable_name():
+                        print(
+                            f"Mapped library {m} suggests target process is running same Python version, so no alternate interpreter needed",
+                            file=sys.stderr,
+                        )
+                        return None
+                    # Check if there's an executable with the same name on the path
+                    if python_exe := shutil.which(python_exe_wanted):
+                        print(
+                            f"Using {python_exe} as alternate Python interpreter based on mapped library {m}",
+                            file=sys.stderr,
+                        )
+                        return python_exe
+            print(
+                "Couldn't find an alternate Python interpreter to use for injection,"
+                " but target process is running a different Python version, so injection may fail",
+                file=sys.stderr,
+            )
+
+
+def _get_exe_and_maps(pid):
+    """Inspect the target process executable and mapped libraries."""
+    if _psutil_available:
+        psutil_process = psutil.Process(pid)
+        exe = psutil_process.exe()
+        try:
+            maps = [map.path for map in psutil_process.memory_maps(grouped=False)]
+        except Exception:
+            maps = []
+        return exe, maps
+    elif sys.platform == "linux":
+        # Linux-only procfs fallback
+        exe = os.readlink(f"/proc/{pid}/exe")
+        maps = []
+        try:
+            with open(f"/proc/{pid}/maps") as f:
+                for line in f:
+                    parts = line.split()
+                    if len(parts) >= 6:
+                        path = parts[-1]
+                        maps.append(path)
+        except Exception:
+            pass
+        return exe, maps
+    else:
+        raise Exception(
+            "Can't determine process executable on this platform without psutil"
+        )
+
+
+def _can_this_python_inject(exe):
+    """Return whether the current interpreter likely matches the target runtime."""
+    if exe == sys.executable:
+        return True
+    if _get_effective_executable_name(exe) == _this_effective_executable_name():
+        return True
+
+    return False
+
+
+def _get_effective_executable_name(exe):
+    """Return the versioned Python executable name for a given path, if it looks like Python."""
+    basename = os.path.basename(exe)
+    match = re.match(r"python\d\.\d.+t?", basename)
+    if match:
+        return match.group(0)
+    if os.path.islink(exe):
+        # If it's a symlink, check if the target looks like Python
+        target = os.readlink(exe)
+        return _get_effective_executable_name(target)
+    return None
+
+
+def _this_effective_executable_name():
+    """Return the versioned Python executable name for the current runtime."""
+
+    name = f"python{sys.version_info.major}.{sys.version_info.minor}"
+    if GIL_ENABLED:
+        return name
+    else:
+        # For GIL-less Python, the executable is suffixed with -gil0
+        return name + "t"
+
+
+def _dump_heap_from_pid_possibly_in_namespace(
+    pid,
+    output_file=DEFAULT_DUMP_FILE,
+    can_use_namespace_injection=True,
+    can_use_alternate_python_interpreter=True,
+):
+    """Decide whether the dump has to run from inside the target mount namespace."""
+    if can_use_namespace_injection and _should_use_namespace(pid):
+        print(
+            "Target process is in a different mount namespace, using namespace injection method",
+            file=sys.stderr,
+        )
+        _dump_heap_from_pid_in_namespace(
+            pid, output_file, can_use_alternate_python_interpreter
+        )
+    else:
+        _dump_heap_from_pid_possibly_using_an_alternate_python_interpreter(
+            pid, output_file, can_use_alternate_python_interpreter
+        )
+
+
+def _dump_heap_from_pid_in_namespace(
+    pid, output_file=DEFAULT_DUMP_FILE, can_use_alternate_python_interpreter=True
+):
+    """Copy the dump file out of the target mount namespace after injection."""
+    dump_loc_in_namespace = f"/tmp/dump_{pid}.jsonl"
+    out = os.open(output_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
+
+    with _in_namespace(pid) as inner_pid:
+        if inner_pid is not None:
+            _dump_heap_from_pid_possibly_using_an_alternate_python_interpreter(
+                inner_pid, dump_loc_in_namespace, can_use_alternate_python_interpreter
+            )
+            while not os.path.exists(dump_loc_in_namespace):
+                time.sleep(0.1)
+            with open(dump_loc_in_namespace) as f, os.fdopen(out, "w") as out_f:
+                for line in f:
+                    out_f.write(line)
+            os.remove(dump_loc_in_namespace)
+
+
+@contextmanager
+def _in_namespace(pid):
+    """Fork into the target mount and PID namespaces for the duration of the block."""
+    if setns is None:
+        raise NotImplementedError(
+            "Namespace injection method is not available on this platform"
+        )
+    inner_pid = _identify_pid_within_namespace(pid)
+    if (forked_pid := os.fork()) == 0:
+        setns(os.open(f"/proc/{pid}/ns/pid", 0), 0)
+        setns(os.open(f"/proc/{pid}/ns/mnt", 0), 0)
+        # Must fork again after setns to actually be in the new PID namespace
+        if (inner_forked_pid := os.fork()) == 0:
+            yield inner_pid
+            os._exit(0)
+        else:
+            yield None
+            pid, status = os.waitpid(inner_forked_pid, 0)
+
+        os._exit(status)
+    else:
+        yield None
+        pid, status = os.waitpid(forked_pid, 0)
+        if status != 0:
+            print(f"Child process failed with status {status}", file=sys.stderr)
+
+
+def _identify_pid_within_namespace(pid):
+    """Translate a host PID into the PID seen inside the target namespace."""
+    with open(f"/proc/{pid}/status") as f:
+        for line in f:
+            if line.startswith("NSpid:"):
+                return int(line.split()[-1])
+
+
+def _dump_heap_from_pid(
+    pid, output_file=DEFAULT_DUMP_FILE, inactivity_timeout=datetime.timedelta(seconds=5)
+):
+    """Build the payload script and inject it into the target process.
+
+    Use remote_exec when available, else fall back to gdb."""
+    code = _build_dump_heap_code(output_file)
+
+    script_file = tempfile.NamedTemporaryFile(suffix=".py", mode="w", delete=False)
+    script_file.write(code)
+    script_file.close()
+    if remote_exec is not None:
+        print("Using remote_exec to inject code", file=sys.stderr)
+        remote_exec(pid, script_file.name)
+        last_progress = time.time()
+        while not os.path.exists(output_file):
+            partial_file = output_file + ".partial"
+            try:
+                last_progress = os.stat(partial_file).st_mtime
+            except FileNotFoundError:
+                pass
+            if time.time() - last_progress > inactivity_timeout.total_seconds():
+                raise PermissionError(
+                    "Timed out waiting for dump file to be created, injection may have failed"
+                )
+
+    else:
+        print("remote_exec not available, falling back to gdb method", file=sys.stderr)
+        gdb_cmds = [
+            "(char *) PyGILState_Ensure()",
+            '(void) PyRun_SimpleString("'
+            rf'exec(open(\"{script_file.name}\").read())")',
+            "(void) PyGILState_Release($1)",
+        ]
+        p = Popen(
+            [
+                "gdb",
+                "-p",
+                str(pid),
+                "--batch",
+                *(f"--eval-command=call {cmd}" for cmd in gdb_cmds),
+            ],
+            stdout=PIPE,
+            stderr=PIPE,
+            text=True,
+        )
+        out, err = p.communicate()
+        print(out, file=sys.stderr)
+        print(err, file=sys.stderr)
+
+
+def _should_use_namespace(pid):
+    """Return whether the target process lives in a different mount namespace."""
+    try:
+        import os
+
+        own_ns = os.readlink("/proc/self/ns/mnt")
+        target_ns = os.readlink(f"/proc/{pid}/ns/mnt")
+        return own_ns != target_ns
+    except Exception:
+        # This can fail for a number of reasons, but they're all cases where we can't join the processes namespace
+        return False
+
+
+def main():
+    """CLI entry point for dumping a live Python process by PID."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Dump the heap of a running Python process."
+    )
+    parser.add_argument("pid", type=int, help="PID of the target Python process")
+    parser.add_argument(
+        "--output-file",
+        "-o",
+        default=DEFAULT_DUMP_FILE,
+        help=f"Path to output file (default: {DEFAULT_DUMP_FILE})",
+    )
+    parser.add_argument(
+        "--no-namespace-injection",
+        action="store_false",
+        dest="can_use_namespace_injection",
+        help="Don't attempt to use namespace injection method.",
+    )
+    parser.add_argument(
+        "--no-alternate-python-interpreter",
+        action="store_false",
+        dest="can_use_alternate_python_interpreter",
+        help="Don't attempt to use an alternate Python interpreter for injection, even if the target process is running a different Python version.",
+    )
+    args = parser.parse_args()
+    dump_heap_from_pid(
+        args.pid,
+        args.output_file,
+        args.can_use_namespace_injection,
+        args.can_use_alternate_python_interpreter,
+    )
+
+
+if __name__ == "__main__":
+    main()

midden-0.1.0/src/midden/static/style.css

@@ -0,0 +1,14 @@
+body { font-family: Arial, sans-serif; margin: 2em; }
+table { border-collapse: collapse; width: 50%; }
+th, td { border: 1px solid #ccc; padding: 8px 12px; text-align: left; }
+th { background: #f4f4f4; }
+h1 { margin-bottom: 1em; }
+ul { margin-bottom: 2em; }
+li { margin: 0.5em 0; }
+form.upload-form { border: 1px solid #ccc; padding: 1em; width: 350px; background: #f9f9f9; }
+label { display: block; margin-bottom: 0.5em; }
+input[type="text"], input[type="file"], select { width: 100%; margin-bottom: 1em; }
+input[type="submit"] { padding: 0.5em 1em; }
+dl { margin-bottom: 2em; }
+dt { font-weight: bold; margin-top: 1em; }
+dd { margin-left: 1.5em; margin-bottom: 0.5em; }

midden-0.1.0/src/midden/templates/components.html

@@ -0,0 +1,35 @@
+{% macro object_list(objects, dump_name, sortable=False) %}
+<table>
+    <tr>
+        <th>{% if sortable %}
+                <a href="?sort=id">ID</a>
+            {% else %}
+                ID
+            {% endif %}
+        </th>
+        <th>Type</th>
+        <th>
+            {% if sortable %}
+                <a href="?sort=size">Size</a>
+            {% else %}
+                Size
+            {% endif %}
+        </th>
+        <th>
+            {% if sortable %}
+                <a href="?sort=subtree_size">Subtree Size</a>
+            {% else %}
+                Subtree Size
+            {% endif %}
+        </th>
+        <th>Value</th></tr>
+    {% for obj in objects %}
+        <tr>
+            <td><a href="{{ url_for('explore_object', dump_name=dump_name, obj_id=obj.id) }}">{{ obj.id }}</a></td>
+            <td>{{ obj.type }}</td>
+            <td>{{ obj.size }} bytes</td>
+            <td>{% if obj.subtree_size is not none %}{{ obj.subtree_size }} bytes{% else %}Unknown{% endif %}</td>
+            <td>{% if obj.value is not none %}{{ obj.value }}{% endif %}</td></tr>
+    {% endfor %}
+</table>
+{% endmacro %}

midden-0.1.0/src/midden/templates/explore.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Heap Dump Overview - {{ dump_name }}</title>
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
+</head>
+<body>
+    <h1>Heap Dump Overview: {{ dump_name }}</h1>
+    <p><a href="{{ url_for('index') }}">Back to heap dump list</a></p>
+    <table>
+        <tr>
+            <th><a href="{{ url_for('explore_dump', dump_name=dump_name, sort_by='type') }}">Type</a></th>
+            <th><a href="{{ url_for('explore_dump', dump_name=dump_name, sort_by='count') }}">Instance Count</a></th>
+            <th><a href="{{ url_for('explore_dump', dump_name=dump_name, sort_by='size') }}">Total Size</a></th>
+        </tr>
+        {% for type_name, summary in type_summaries %}
+        <tr>
+            <td><a href="{{ url_for('explore_type', dump_name=dump_name, type_name=type_name) }}">{{ type_name }}</a></td>
+            <td>{{ summary.count }}</td>
+            <td>{{ summary.total_size }}</td>
+        </tr>
+        {% endfor %}
+    </table>
+</body>
+</html>

midden-0.1.0/src/midden/templates/index.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Heap Dump Explorer</title>
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
+</head>
+<body>
+    <h1>Heap Dump Explorer</h1>
+
+    <h2>Existing Heap Dumps</h2>
+    {% if dump_names %}
+        <ul>
+        {% for name in dump_names %}
+            <li>
+                <a href="{{ url_for('explore_dump', dump_name=name) }}">{{ name }}</a>
+            </li>
+        {% endfor %}
+        </ul>
+    {% else %}
+        <p>No heap dumps loaded yet.</p>
+    {% endif %}
+
+    <h2>Upload New Heap Dump</h2>
+    <form action="{{ url_for('upload_dump') }}" method="post" enctype="multipart/form-data" class="upload-form">
+        <label for="dump_name">Heap Dump Name:</label>
+        <input type="text" id="dump_name" name="dump_name">
+        <label for="estimator_precision">Subtree Size Estimate Precision</label>
+        <select id="estimator_precision" name="estimator_precision" required>
+            <option value="no_estimates">No Estimates (fastest, lowest memory)</option>
+            <option value="low">Low Precision</option>
+            <option value="medium" selected>Medium Precision</option>
+            <option value="high">High Precision</option>
+            <option value="exact">Exact (slowest, highest memory)</option>
+        </select>
+
+        <label for="dump_file">Heap Dump File (.jsonl):</label>
+        <input type="file" id="dump_file" name="dump_file" accept=".jsonl" required>
+
+        <input type="submit" value="Upload">
+    </form>
+</body>
+</html>

midden-0.1.0/src/midden/templates/object.html

@@ -0,0 +1,52 @@
+{% import 'components.html' as components %}
+<!DOCTYPE html>
+<html>
+<head>
+    <title>{{ obj.type }} instance {{ obj.id }} - {{ dump_name }}</title>
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
+</head>
+<body>
+    <h1>{{ obj.type }} instance {{ obj.id }} in Heap Dump: {{ dump_name }}</h1>
+    <p><a href="{{ url_for('explore_dump', dump_name=dump_name) }}">Back to overview</a></p>
+    {% if current_from_id == obj.id %}
+        <p><strong>This object will be used as the starting point if you start a path search.</strong></p>
+    {% else %}
+        <form action="{{ url_for('set_path_finding_endpoint', dump_name=dump_name) }}" method="post">
+            <input type="hidden" name="from_id" value="{{ obj.id }}">
+            <button type="submit">Can you find a path from here to {% if current_to_id %}object {{ current_to_id }}{% else %}...{% endif %}?</button>
+        </form>
+    {% endif %}
+    {% if current_to_id == obj.id %}
+        <p><strong>This object will be used as the ending point if you start a path search.</strong></p>
+    {% else %}
+        <form action="{{ url_for('set_path_finding_endpoint', dump_name=dump_name) }}" method="post">
+            <input type="hidden" name="to_id" value="{{ obj.id }}">
+            <button type="submit">Can you find a path from {% if current_from_id %}object {{ current_from_id }}{% else %}...{% endif %} to here?</button>
+        </form>
+    {% endif %}
+    <dl>
+        <dt>Type:</dt>
+        <dd><a href="{{ url_for('explore_type', type_name=obj.type, dump_name=dump_name) }}">{{ obj.type }}</a></dd>
+        <dt>ID:</dt>
+        <dd>{{ obj.id }}</dd>
+        <dt>Size:</dt>
+        <dd>{{ obj.size }} bytes</dd>
+        {% if obj.subtree_size is not none %}
+        <dt>Subtree Size:</dt>
+        <dd>{{ obj.subtree_size }} bytes</dd>
+        {% endif %}
+        {% if obj.value is not none %}
+        <dt>Value:</dt>
+        <dd>{{ obj.value }}</dd>
+        {% endif %}
+    </dl>
+    {% if obj.referrers %}
+    <h2>Referrers</h2>
+    {{ components.object_list(obj.referrers, dump_name) }}
+    {% endif %}
+    {% if obj.references %}
+    <h2>References</h2>
+    {{ components.object_list(obj.references, dump_name) }}
+    {% endif %}
+</body>
+</html>

midden-0.1.0/src/midden/templates/path.html

@@ -0,0 +1,46 @@
+{% import 'components.html' as components %}
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Path from {{ from_id }} to {{ to_id }} - {{ dump_name }}</title>
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
+</head>
+<body>
+    <h1>Path from {{ from_id }} to {{ to_id }} in Heap Dump: {{ dump_name }}</h1>
+    <p><a href="{{ url_for('explore_dump', dump_name=dump_name) }}">Back to overview</a></p>
+    {% if path is none %}
+        <p><strong>No path found.</strong></p>
+    {% else %}
+        <table>
+    <tr>
+        <th>ID</th>
+        <th>Type</th>
+        <th>Size</th>
+        <th>Subtree Size</th>
+        <th>Value</th>
+        <th>Avoid</th>
+    </tr>
+
+    {% for obj in path %}
+        <tr>
+            <td><a href="{{ url_for('explore_object', dump_name=dump_name, obj_id=obj.id) }}">{{ obj.id }}</a></td>
+            <td>{{ obj.type }}</td>
+            <td>{{ obj.size }} bytes</td>
+            <td>{% if obj.subtree_size is not none %}{{ obj.subtree_size }} bytes{% else %}Unknown{% endif %}</td>
+            <td>{% if obj.value is not none %}{{ obj.value }}{% endif %}</td>
+            <td>
+                {% if obj.id not in avoid_ids %}
+                    <a
+                        href="{{ url_for('find_path', dump_name=dump_name, from_id=from_id, to_id=to_id, avoid_id=avoid_ids + [obj.id]) }}"
+                        title="Find a new path that avoids this object">
+                        Avoid
+                    </a>
+                {% endif %}
+            </td>
+        </tr>
+    {% endfor %}
+</table>
+    {% endif %}
+    
+</body>
+</html>

midden-0.1.0/src/midden/templates/type.html

@@ -0,0 +1,27 @@
+{% import 'components.html' as components %}
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Instances of type {{ type_name }} - {{ dump_name }}</title>
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
+</head>
+<body>
+    <h1>Instances of type {{ type_name }} in Heap Dump: {{ dump_name }}</h1>
+    <p><a href="{{ url_for('explore_dump', dump_name=dump_name) }}">Back to overview</a></p>
+    <p>
+        <form action="{{ url_for('explore_type', dump_name=dump_name, type_name=type_name) }}" method="get">
+            <a title="first page" href="{{ url_for('explore_type', dump_name=dump_name, type_name=type_name, page=1, sort=sort) }}">⏮️</a>
+            {% if page > 1 %}
+                <a title="previous page" href="{{ url_for('explore_type', dump_name=dump_name, type_name=type_name, page=page-1, sort=sort) }}">◀️</a>
+            {% endif %}
+            Page <input type="number" name="page" value="{{ page }}" min="1" max="{{ total_pages }}">
+                <input type="hidden" name="sort" value="{{ sort }}"> of {{ total_pages }}
+            {% if page < total_pages %}
+                <a title="next page" href="{{ url_for('explore_type', dump_name=dump_name, type_name=type_name, page=page+1, sort=sort) }}">▶️</a>
+            {% endif %}
+            <a title="last page" href="{{ url_for('explore_type', dump_name=dump_name, type_name=type_name, page=total_pages, sort=sort) }}">⏭️</a>
+        </form>
+    </p>
+    {{ components.object_list(objects, dump_name, sortable=True) }}
+</body>
+</html>

midden-0.1.0/src/midden/wsgi.py

@@ -0,0 +1,216 @@
+"""A simple Flask GUI, with templated HTML with very basic styling, that lets the user explore a heap dump created by dump_heap.py."""
+import webbrowser
+
+import shutil
+
+import argparse
+
+from werkzeug.exceptions import NotFound
+from ntpath import basename
+import os
+import pathlib
+
+from cheroot.wsgi import Server as WSGIServer
+from flask import Flask, request, redirect, url_for, render_template, session
+from midden_analysis import HeapDumpExplorer, TypeSummary, EstimatorPrecision
+
+DUMPS_DIR = os.getenv("DUMPS_DIR", "/tmp/dumps")
+
+PRECISION_MAP = {
+    "no_estimates": EstimatorPrecision.NoEstimates,
+    "low": EstimatorPrecision.Low,
+    "medium": EstimatorPrecision.Medium,
+    "high": EstimatorPrecision.High,
+    "exact": EstimatorPrecision.Exact,
+}
+
+
+def create_app():
+    """Create the Flask app and preload any dumps already stored on disk."""
+    os.makedirs(DUMPS_DIR, exist_ok=True)
+    loaded_dumps: dict[str, HeapDumpExplorer] = {
+        basename(path)[:-5]: HeapDumpExplorer(f"{DUMPS_DIR}/{path}")
+        for path in os.listdir(DUMPS_DIR)
+        if path.endswith(".lmdb")
+    }
+
+    def get_dump(dump_name) -> HeapDumpExplorer:
+        """Return a loaded dump or raise a 404 for unknown names."""
+        explorer = loaded_dumps.get(dump_name)
+        if not explorer:
+            raise NotFound(f"Dump '{dump_name}' not found")
+        return explorer
+
+    app = Flask(__name__)
+
+    @app.route("/")
+    def index():
+        dump_names = list(loaded_dumps.keys())
+        return render_template("index.html", dump_names=dump_names)
+
+    @app.route("/upload_dump", methods=["POST"])
+    def upload_dump():
+        """Import an uploaded JSONL heap dump into a new LMDB-backed explorer."""
+        dump_name = request.form.get("dump_name")
+        dump_file = request.files["dump_file"]
+        if not dump_name:
+            if upload_filename := dump_file.filename:
+                dump_name = pathlib.Path(upload_filename).stem
+            else:
+                dump_name = f"heap_dump_{len(loaded_dumps) + 1}"
+        if dump_name in loaded_dumps:
+            return f"Heap dump with name '{dump_name}' already exists", 409
+        if "/" in dump_name or "\\" in dump_name or dump_name.startswith("."):
+            return "Invalid heap dump name", 400
+        dump_dir = f"{DUMPS_DIR}/{dump_name}.lmdb"
+        precision = request.form.get("estimator_precision", "medium")
+        estimator_precision = PRECISION_MAP.get(precision, EstimatorPrecision.Medium)
+        os.mkdir(dump_dir)
+        try:
+            explorer = HeapDumpExplorer(f"{DUMPS_DIR}/{dump_name}.lmdb")
+            explorer.import_lines(dump_file, estimator_precision)
+        except Exception:
+            shutil.rmtree(dump_dir)
+            raise
+        loaded_dumps[dump_name] = explorer
+        return redirect(url_for("explore_dump", dump_name=dump_name))
+
+    @app.route("/explore/<dump_name>")
+    def explore_dump(dump_name):
+        explorer = get_dump(dump_name)
+        # The landing page for a dump is a type summary table.
+        type_summaries: list[tuple[str, TypeSummary]] = explorer.get_type_summaries()
+        sort_by = request.args.get("sort_by", "count")
+        match sort_by:
+            case "size":
+                type_summaries.sort(key=lambda x: x[1].total_size, reverse=True)
+            case "type":
+                type_summaries.sort(key=lambda x: x[0])
+            case "count" | _:
+                type_summaries.sort(key=lambda x: x[1].count, reverse=True)
+        return render_template(
+            "explore.html", dump_name=dump_name, type_summaries=type_summaries
+        )
+
+    @app.route("/explore/<dump_name>/type/<type_name>")
+    def explore_type(dump_name, type_name):
+        """Show one page of objects for a type, with optional size-based sorting."""
+        page = request.args.get("page", 1, type=int)
+        page_zero_indexed = max(page - 1, 0)
+        explorer = get_dump(dump_name)
+        sort = request.args.get("sort", "id")
+        match sort:
+            case "size":
+                objects = explorer.get_objects_by_type_ordered_by_size(
+                    type_name, page=page_zero_indexed, subtree_size=False
+                )
+            case "subtree_size":
+                objects = explorer.get_objects_by_type_ordered_by_size(
+                    type_name, page=page_zero_indexed, subtree_size=True
+                )
+            case "id" | _:
+                objects = explorer.get_objects_by_type(
+                    type_name, page=page_zero_indexed
+                )
+        total_pages = explorer.get_page_count_for_type(type_name)
+        return render_template(
+            "type.html",
+            dump_name=dump_name,
+            type_name=type_name,
+            objects=objects,
+            page=page,
+            total_pages=total_pages,
+            sort=sort,
+        )
+
+    @app.route("/explore/<dump_name>/object/<int:obj_id>")
+    def explore_object(dump_name, obj_id):
+        """Show one object together with its references and referrers."""
+        explorer = get_dump(dump_name)
+        obj = explorer.get_object(obj_id)
+        current_from_id = session.get(f"path_finding_from_id:{dump_name}")
+        current_to_id = session.get(f"path_finding_to_id:{dump_name}")
+        if not obj:
+            raise NotFound(f"Object with ID {obj_id} not found in dump '{dump_name}'")
+        return render_template(
+            "object.html",
+            dump_name=dump_name,
+            obj=obj,
+            current_from_id=current_from_id,
+            current_to_id=current_to_id,
+        )
+
+    @app.route("/explore/<dump_name>/set_path_finding_endpoint", methods=["POST"])
+    def set_path_finding_endpoint(dump_name):
+        """Store or complete the pair of object IDs used for path finding."""
+        from_id = request.form.get(
+            "from_id", session.get(f"path_finding_from_id:{dump_name}"), type=int
+        )
+        to_id = request.form.get(
+            "to_id", session.get(f"path_finding_to_id:{dump_name}"), type=int
+        )
+        if from_id is not None and to_id is not None:
+            del session[f"path_finding_from_id:{dump_name}"]
+            del session[f"path_finding_to_id:{dump_name}"]
+            return redirect(
+                url_for("find_path", dump_name=dump_name, from_id=from_id, to_id=to_id)
+            )
+        else:
+            session[f"path_finding_from_id:{dump_name}"] = from_id
+            session[f"path_finding_to_id:{dump_name}"] = to_id
+
+            return redirect(
+                url_for("explore_object", dump_name=dump_name, obj_id=from_id or to_id)
+            )
+
+    @app.route("/explore/<dump_name>/find_path")
+    def find_path(dump_name):
+        """Find a reference path between two objects in the selected dump."""
+        explorer = get_dump(dump_name)
+        from_id = request.args.get("from_id", type=int)
+        to_id = request.args.get("to_id", type=int)
+        avoid_ids = set(request.args.getlist("avoid_id", type=int))
+        if from_id is None or to_id is None:
+            return "Missing from_id or to_id query parameters", 400
+        path = explorer.find_path_between_objects(
+            from_id, to_id, avoiding_ids=avoid_ids
+        )
+        return render_template(
+            "path.html",
+            dump_name=dump_name,
+            from_id=from_id,
+            to_id=to_id,
+            path=path,
+            avoid_ids=list(avoid_ids),
+        )
+
+    app.secret_key = os.urandom(
+        16
+    )  # This app keeps state in-process, so a per-process key is fine.
+    return app
+
+
+def main():
+    """Run the local Midden web server."""
+    arg_parser = argparse.ArgumentParser(description="Run the Midden web server")
+    arg_parser.add_argument(
+        "--host", default="127.0.0.1"
+    )
+    arg_parser.add_argument(
+        "--port", default=5000, type=int
+    )
+    arg_parser.add_argument(
+        "--no-start-web-browser", action="store_false", dest="start_web_browser", help="Don't automatically open the web browser"
+    )
+    args = arg_parser.parse_args()
+    app = create_app()
+    url = f"http://{args.host}:{args.port}"
+    print(f"Starting Midden web server on {url}")
+    server = WSGIServer((args.host, args.port), app)
+    if args.start_web_browser:
+        webbrowser.open(url)
+    server.start()
+
+
+if __name__ == "__main__":
+    main()