python-leak-hunter 0.1.1__py3-none-any.whl

leak_hunter/__init__.py

@@ -0,0 +1,39 @@
+"""Find memory leaks in long-running Python services.
+
+- `measure_growth` / `assert_no_leak`: which object types accumulate per unit of work.
+- `why_alive`: what keeps them alive, down to "held by native code".
+- `MemorySampler` / `diagnose`: is it really a leak, or fragmentation, or native memory.
+"""
+
+from .growth import (
+    GrowthReport,
+    LeakError,
+    TypeGrowth,
+    assert_no_leak,
+    assert_no_leak_async,
+    count_objects,
+    measure_growth,
+    measure_growth_async,
+)
+from .memory import MemorySample, MemorySampler, diagnose, sample_memory
+from .retention import RetentionPath, RetentionReport, why_alive
+
+__all__ = [
+    "GrowthReport",
+    "LeakError",
+    "MemorySample",
+    "MemorySampler",
+    "RetentionPath",
+    "RetentionReport",
+    "TypeGrowth",
+    "assert_no_leak",
+    "assert_no_leak_async",
+    "count_objects",
+    "diagnose",
+    "measure_growth",
+    "measure_growth_async",
+    "sample_memory",
+    "why_alive",
+]
+
+__version__ = "0.1.1"

leak_hunter/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

leak_hunter/cli.py

@@ -0,0 +1,86 @@
+"""Command line: `leak-hunter check` and `leak-hunter why`, aimed at a small harness function.
+
+    leak-hunter check harness:handle_one_request --iterations 30 --trace
+    leak-hunter why harness:handle_many_requests myapp.graph.Graph
+
+`check` runs a zero-argument function repeatedly and exits 1 if objects accumulate.
+`why` runs a setup function once, then explains what keeps instances of a type alive.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import gc
+import importlib
+import inspect
+import os
+import sys
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from .growth import awaited, measure_growth, type_name
+from .retention import why_alive
+
+
+def _load(spec: str) -> Callable[[], Any]:
+    module_name, _, attribute = spec.partition(":")
+    if not module_name or not attribute:
+        raise SystemExit(f"expected module:function, got {spec!r}")
+    sys.path.insert(0, os.getcwd())
+    target: Any = importlib.import_module(module_name)
+    for part in attribute.split("."):
+        target = getattr(target, part)
+    if not callable(target):
+        raise SystemExit(f"{spec} is not callable")
+    return target  # type: ignore[no-any-return]  # verified callable above
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="leak-hunter", description="Find memory leaks in long-running Python services."
+    )
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    check = commands.add_parser("check", help="find object types that accumulate per call")
+    check.add_argument("work", help="module:function taking no arguments (sync or async)")
+    check.add_argument("--iterations", type=int, default=20)
+    check.add_argument("--warmup", type=int, default=3)
+    check.add_argument("--min-per-call", type=float, default=0.5)
+    check.add_argument("--include", action="append", help="only report types with this name prefix")
+    check.add_argument("--ignore", action="append", default=[], help="skip types with this name prefix")
+    check.add_argument("--trace", action="store_true", help="also report allocation sites (tracemalloc)")
+
+    why = commands.add_parser("why", help="explain what keeps objects of a type alive")
+    why.add_argument("setup", help="module:function to run first, e.g. one that handles 50 requests")
+    why.add_argument("type_name", help="type name as printed by `check`, e.g. myapp.Graph")
+    why.add_argument("--samples", type=int, default=5)
+
+    args = parser.parse_args(argv)
+    if args.command == "check":
+        report = measure_growth(
+            _load(args.work),
+            iterations=args.iterations,
+            warmup=args.warmup,
+            min_per_call=args.min_per_call,
+            include=args.include,
+            ignore=args.ignore,
+            trace_allocations=args.trace,
+        )
+        print(report.format())
+        return 1 if report.leaked else 0
+
+    setup = _load(args.setup)
+    # Only trace instances the setup created, so `why ... list` follows the leaked lists.
+    gc.collect()
+    existing = {id(obj) for obj in gc.get_objects() if type_name(obj) == args.type_name}
+    result = setup()
+    if inspect.isawaitable(result):
+        asyncio.run(awaited(result))
+    del result
+    print(why_alive(args.type_name, samples=args.samples, exclude=existing).format())
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

leak_hunter/growth.py

@@ -0,0 +1,305 @@
+"""Find object types that keep accumulating when the same unit of work runs repeatedly.
+
+The measurement runs the work in two equal windows after a warm-up. A type only counts as
+leaking when its live count grows in *both* windows: one-off growth such as a cache filling
+up or a lazy import shows up in one window, not both.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import gc
+import inspect
+import os
+import tracemalloc
+from collections import Counter
+from collections.abc import Awaitable, Callable, Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+Work = Callable[[], Any]
+AsyncWork = Callable[[], Awaitable[Any]]
+
+_PACKAGE_DIR = str(Path(__file__).parent)
+
+
+def type_name(obj: object) -> str:
+    """Fully qualified type name, without the noise of `builtins.`."""
+    cls = type(obj)
+    module = cls.__module__
+    return cls.__qualname__ if module == "builtins" else f"{module}.{cls.__qualname__}"
+
+
+def count_objects() -> Counter[str]:
+    """Count live objects tracked by the garbage collector, by type name.
+
+    Only container-like objects are tracked (class instances, dicts, lists, closures...).
+    Plain strings, ints and bytes are not, so they never appear here; `trace_allocations`
+    covers them by allocation site instead.
+    """
+    gc.collect()
+    counts: Counter[str] = Counter()
+    for obj in gc.get_objects():
+        counts[type_name(obj)] += 1
+    return counts
+
+
+@dataclass(frozen=True)
+class TypeGrowth:
+    type_name: str
+    first_window: int
+    second_window: int
+    iterations: int
+
+    @property
+    def per_call(self) -> float:
+        return self.second_window / self.iterations
+
+
+@dataclass(frozen=True)
+class AllocationGrowth:
+    location: str
+    size_bytes: int
+    blocks: int
+
+
+@dataclass(frozen=True)
+class GrowthReport:
+    iterations: int
+    growing: tuple[TypeGrowth, ...]
+    allocations: tuple[AllocationGrowth, ...] = ()
+
+    @property
+    def leaked(self) -> bool:
+        return bool(self.growing)
+
+    @property
+    def suspect(self) -> str | None:
+        """The type to trace next: the first one defined in a module, not a builtin like `list`."""
+        for growth in self.growing:
+            if "." in growth.type_name:
+                return growth.type_name
+        return self.growing[0].type_name if self.growing else None
+
+    def format(self, limit: int = 10) -> str:
+        lines = []
+        if not self.leaked:
+            lines.append(f"No object type grew steadily over 2 x {self.iterations} calls.")
+        else:
+            shown = self.growing[:limit]
+            width = max(len(g.type_name) for g in shown)
+            lines.append(f"Objects accumulating across calls (2 windows x {self.iterations} calls):")
+            for g in shown:
+                lines.append(
+                    f"  {g.type_name:<{width}}  +{_number(g.per_call)} per call"
+                    f"  (window 1: {g.first_window:+d}, window 2: {g.second_window:+d})"
+                )
+            if len(self.growing) > limit:
+                lines.append(f"  ... and {len(self.growing) - limit} more types")
+        if self.allocations:
+            # Covers what the object count cannot: bytes, str and other untracked objects.
+            lines.append(f"Allocation sites still holding memory from the last {self.iterations} calls:")
+            for a in self.allocations:
+                per_call = a.size_bytes / self.iterations / 1024
+                lines.append(f"  {a.location}  +{_number(per_call)} KiB per call  ({a.blocks:+d} blocks)")
+        if self.suspect is not None:
+            lines.append("Next, see what keeps them alive:")
+            lines.append(f"  leak-hunter why <module:function that runs many calls> {self.suspect}")
+            lines.append(f"  or in Python: why_alive({self.suspect!r})")
+        return "\n".join(lines)
+
+
+def _number(value: float) -> str:
+    return f"{value:.2f}".rstrip("0").rstrip(".")
+
+
+def _short_path(path: str) -> str:
+    """Relative to the working directory when inside it, so report lines stay readable."""
+    relative = os.path.relpath(path)
+    return path if relative.startswith("..") else relative
+
+
+class LeakError(AssertionError):
+    """Raised by `assert_no_leak` when objects keep accumulating."""
+
+    def __init__(self, report: GrowthReport) -> None:
+        super().__init__(report.format())
+        self.report = report
+
+
+class _Windows:
+    """Takes the three object counts around the two measured windows."""
+
+    def __init__(
+        self,
+        *,
+        iterations: int,
+        min_per_call: float,
+        include: Iterable[str] | None,
+        ignore: Iterable[str],
+        trace_allocations: bool,
+    ) -> None:
+        if iterations < 1:
+            raise ValueError("iterations must be at least 1")
+        self._iterations = iterations
+        self._threshold = min_per_call * iterations
+        self._include = tuple(include) if include is not None else None
+        self._ignore = tuple(ignore)
+        self._trace = trace_allocations
+        self._started_tracing = False
+        self._before: Counter[str] = Counter()
+        self._middle: Counter[str] = Counter()
+        self._middle_snapshot: tracemalloc.Snapshot | None = None
+
+    def start(self) -> None:
+        if self._trace and not tracemalloc.is_tracing():
+            tracemalloc.start()
+            self._started_tracing = True
+        self._before = count_objects()
+
+    def middle(self) -> None:
+        self._middle = count_objects()
+        if self._trace:
+            self._middle_snapshot = tracemalloc.take_snapshot()
+
+    def finish(self) -> GrowthReport:
+        after = count_objects()
+        allocations = self._allocations() if self._trace else ()
+        growing = []
+        for name, count in after.items():
+            first = self._middle[name] - self._before[name]
+            second = count - self._middle[name]
+            if first >= self._threshold and second >= self._threshold and self._wanted(name):
+                growing.append(TypeGrowth(name, first, second, self._iterations))
+        growing.sort(key=lambda g: g.second_window, reverse=True)
+        return GrowthReport(self._iterations, tuple(growing), allocations)
+
+    def close(self) -> None:
+        if self._started_tracing:
+            tracemalloc.stop()
+            self._started_tracing = False
+
+    def _wanted(self, name: str) -> bool:
+        if any(name.startswith(prefix) for prefix in self._ignore):
+            return False
+        return self._include is None or any(name.startswith(prefix) for prefix in self._include)
+
+    def _allocations(self, limit: int = 5) -> tuple[AllocationGrowth, ...]:
+        if self._middle_snapshot is None:
+            return ()
+        exclude = [
+            tracemalloc.Filter(False, tracemalloc.__file__),
+            tracemalloc.Filter(False, f"{_PACKAGE_DIR}/*"),
+        ]
+        after = tracemalloc.take_snapshot().filter_traces(exclude)
+        before = self._middle_snapshot.filter_traces(exclude)
+        found = []
+        for stat in after.compare_to(before, "lineno"):
+            if stat.size_diff > 0 and stat.count_diff > 0:
+                frame = stat.traceback[0]
+                location = f"{_short_path(frame.filename)}:{frame.lineno}"
+                found.append(AllocationGrowth(location, stat.size_diff, stat.count_diff))
+            if len(found) == limit:
+                break
+        return tuple(found)
+
+
+def measure_growth(
+    work: Work,
+    *,
+    iterations: int = 20,
+    warmup: int = 3,
+    min_per_call: float = 0.5,
+    include: Iterable[str] | None = None,
+    ignore: Iterable[str] = (),
+    trace_allocations: bool = False,
+) -> GrowthReport:
+    """Run `work` repeatedly and report object types that keep accumulating.
+
+    `work` takes no arguments and may be a plain function or return an awaitable (an
+    `async def` works); awaitables run on a private event loop. Inside an already running
+    loop, use `measure_growth_async` instead.
+
+    A type is reported when it gains at least `min_per_call` objects per call in both
+    windows. `include` / `ignore` filter by type-name prefix, e.g. `include=["myapp."]`.
+    """
+    windows = _Windows(
+        iterations=iterations,
+        min_per_call=min_per_call,
+        include=include,
+        ignore=ignore,
+        trace_allocations=trace_allocations,
+    )
+    loop = asyncio.new_event_loop()
+
+    def run(times: int) -> None:
+        for _ in range(times):
+            result = work()
+            if inspect.isawaitable(result):
+                loop.run_until_complete(awaited(result))
+
+    try:
+        run(warmup)
+        windows.start()
+        run(iterations)
+        windows.middle()
+        run(iterations)
+        return windows.finish()
+    finally:
+        windows.close()
+        loop.close()
+
+
+async def measure_growth_async(
+    work: AsyncWork,
+    *,
+    iterations: int = 20,
+    warmup: int = 3,
+    min_per_call: float = 0.5,
+    include: Iterable[str] | None = None,
+    ignore: Iterable[str] = (),
+    trace_allocations: bool = False,
+) -> GrowthReport:
+    """`measure_growth` for code that already runs inside an event loop."""
+    windows = _Windows(
+        iterations=iterations,
+        min_per_call=min_per_call,
+        include=include,
+        ignore=ignore,
+        trace_allocations=trace_allocations,
+    )
+
+    async def run(times: int) -> None:
+        for _ in range(times):
+            await work()
+
+    try:
+        await run(warmup)
+        windows.start()
+        await run(iterations)
+        windows.middle()
+        await run(iterations)
+        return windows.finish()
+    finally:
+        windows.close()
+
+
+def assert_no_leak(work: Work, **options: Any) -> GrowthReport:
+    """Fail with a readable report if `work` leaks objects. Takes `measure_growth`'s options."""
+    report = measure_growth(work, **options)
+    if report.leaked:
+        raise LeakError(report)
+    return report
+
+
+async def assert_no_leak_async(work: AsyncWork, **options: Any) -> GrowthReport:
+    """`assert_no_leak` for code that already runs inside an event loop."""
+    report = await measure_growth_async(work, **options)
+    if report.leaked:
+        raise LeakError(report)
+    return report
+
+
+async def awaited(awaitable: Awaitable[Any]) -> Any:
+    return await awaitable

leak_hunter/memory.py

@@ -0,0 +1,230 @@
+"""Sample process memory over time and tell real retention apart from allocator effects.
+
+RSS alone cannot distinguish "objects are being kept" from "freed memory is not returned to
+the OS" (fragmentation) or "a native library is allocating outside malloc". On glibc,
+`mallinfo2()` splits malloc's heap into bytes in use and bytes free, which separates them.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import gc
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+from collections.abc import Callable, Sequence
+from dataclasses import asdict, dataclass
+from types import TracebackType
+
+
+@dataclass(frozen=True)
+class MemorySample:
+    time: float
+    rss_bytes: int | None
+    heap_in_use_bytes: int | None
+    heap_free_bytes: int | None
+    gc_objects: int
+    open_fds: int | None
+
+    def to_json(self) -> str:
+        return json.dumps(asdict(self))
+
+
+class _MallInfo2(ctypes.Structure):
+    _fields_ = [
+        (field, ctypes.c_size_t)
+        for field in (
+            "arena",
+            "ordblks",
+            "smblks",
+            "hblks",
+            "hblkhd",
+            "usmblks",
+            "fsmblks",
+            "uordblks",
+            "fordblks",
+            "keepcost",
+        )
+    ]
+
+
+def _load_mallinfo2() -> Callable[[], _MallInfo2] | None:
+    if not sys.platform.startswith("linux"):
+        return None
+    try:
+        fn = ctypes.CDLL(None).mallinfo2
+    except (OSError, AttributeError):  # musl, or glibc older than 2.33
+        return None
+    fn.restype = _MallInfo2
+    fn.argtypes = []
+    return fn
+
+
+_mallinfo2 = _load_mallinfo2()
+
+
+def _rss_bytes() -> int | None:
+    try:
+        import psutil  # type: ignore[import-untyped, unused-ignore]  # optional dependency
+
+        return int(psutil.Process().memory_info().rss)
+    except ImportError:
+        pass
+    try:
+        with open("/proc/self/statm") as statm:
+            return int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE")
+    except OSError:
+        pass
+    try:  # macOS and other BSDs without psutil
+        out = subprocess.run(
+            ["ps", "-o", "rss=", "-p", str(os.getpid())], capture_output=True, text=True, check=True
+        )
+        return int(out.stdout.strip()) * 1024
+    except (OSError, ValueError, subprocess.CalledProcessError):
+        return None
+
+
+def _open_fds() -> int | None:
+    for path in ("/proc/self/fd", "/dev/fd"):
+        try:
+            return len(os.listdir(path))
+        except OSError:
+            continue
+    return None
+
+
+def sample_memory() -> MemorySample:
+    """One reading of this process's memory. Heap fields are None off glibc."""
+    heap_in_use = heap_free = None
+    if _mallinfo2 is not None:
+        info = _mallinfo2()
+        heap_in_use = info.uordblks + info.hblkhd
+        heap_free = info.fordblks
+    return MemorySample(
+        time=time.time(),
+        rss_bytes=_rss_bytes(),
+        heap_in_use_bytes=heap_in_use,
+        heap_free_bytes=heap_free,
+        gc_objects=len(gc.get_objects()),
+        open_fds=_open_fds(),
+    )
+
+
+class MemorySampler:
+    """Samples memory on a background thread, e.g. once a minute in a running service.
+
+    Each sample goes to `on_sample` (log it as one JSON line with `sample.to_json()`) and is
+    kept in `samples` for `diagnose()`. Use as a context manager or call start()/stop().
+    """
+
+    def __init__(
+        self,
+        interval: float = 60.0,
+        on_sample: Callable[[MemorySample], None] | None = None,
+        keep: int = 1_440,
+    ) -> None:
+        self.interval = interval
+        self.samples: list[MemorySample] = []
+        self._on_sample = on_sample
+        self._keep = keep
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def start(self) -> MemorySampler:
+        if self._thread is None:
+            self._stop.clear()
+            self._thread = threading.Thread(target=self._run, name="leak-hunter-sampler", daemon=True)
+            self._thread.start()
+        return self
+
+    def stop(self) -> None:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join()
+            self._thread = None
+
+    def __enter__(self) -> MemorySampler:
+        return self.start()
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.stop()
+
+    def _run(self) -> None:
+        while True:
+            sample = sample_memory()
+            self.samples.append(sample)
+            del self.samples[: -self._keep]
+            if self._on_sample is not None:
+                self._on_sample(sample)
+            if self._stop.wait(self.interval):
+                return
+
+
+def slope_per_hour(samples: Sequence[MemorySample], field: str) -> float | None:
+    """Least-squares growth of a numeric sample field, in units per hour."""
+    points = [(s.time, getattr(s, field)) for s in samples if getattr(s, field) is not None]
+    if len(points) < 3:
+        return None
+    mean_t = sum(t for t, _ in points) / len(points)
+    mean_v = sum(v for _, v in points) / len(points)
+    spread = sum((t - mean_t) ** 2 for t, _ in points)
+    if spread == 0:
+        return None
+    return float(sum((t - mean_t) * (v - mean_v) for t, v in points) / spread * 3600)
+
+
+def diagnose(samples: Sequence[MemorySample], min_mb_per_hour: float = 10.0) -> str:
+    """Classify memory growth from samples taken over a meaningful stretch of real traffic."""
+    mb = 1024 * 1024
+    rss = slope_per_hour(samples, "rss_bytes")
+    if rss is None:
+        return "Not enough samples: take at least 3, spread over time, with RSS available."
+    if rss < min_mb_per_hour * mb:
+        return f"RSS is flat ({rss / mb:+.1f} MB/h). No leak visible in this window."
+
+    lines = [f"RSS grows {rss / mb:+.1f} MB/h."]
+    in_use = slope_per_hour(samples, "heap_in_use_bytes")
+    free = slope_per_hour(samples, "heap_free_bytes")
+    objects = slope_per_hour(samples, "gc_objects")
+    fds = slope_per_hour(samples, "open_fds")
+
+    if in_use is not None and free is not None:
+        if in_use >= 0.5 * rss:
+            lines.append(
+                f"malloc heap in use grows {in_use / mb:+.1f} MB/h: memory is being retained, not "
+                "fragmented. Find what accumulates (measure_growth), then what holds it (why_alive)."
+            )
+        elif free >= 0.5 * rss:
+            lines.append(
+                f"malloc free space grows {free / mb:+.1f} MB/h while in-use is flat: fragmentation. "
+                "Try MALLOC_ARENA_MAX=2 or jemalloc before hunting objects."
+            )
+        else:
+            lines.append(
+                "malloc's heap explains little of it: memory is allocated outside malloc (mmap by a "
+                "native library, thread stacks). Profile natively with memray --native or jemalloc."
+            )
+    elif objects is not None and objects > 0:
+        lines.append(
+            f"The live Python object count grows {objects:+.0f}/h: objects are accumulating. "
+            "Run measure_growth on one unit of work."
+        )
+    else:
+        lines.append(
+            "The Python object count is flat, so the growth is native memory or fragmentation. "
+            "No mallinfo2 here (not glibc): use memray --native, or test under Linux."
+        )
+    if fds is not None and fds > 1:
+        lines.append(
+            f"Open file descriptors grow {fds:+.0f}/h: connections or files are not being closed "
+            "(clients created per request?)."
+        )
+    return "\n".join(lines)

leak_hunter/pytest_plugin.py

@@ -0,0 +1,20 @@
+"""pytest plugin: the `leak_check` fixture, registered automatically when the package is installed."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+import pytest
+
+from .growth import GrowthReport, Work, assert_no_leak
+
+
+@pytest.fixture
+def leak_check() -> Callable[..., GrowthReport]:
+    """`leak_check(work, **options)` fails the test if `work` leaks. Same options as `measure_growth`."""
+
+    def check(work: Work, **options: Any) -> GrowthReport:
+        return assert_no_leak(work, **options)
+
+    return check

leak_hunter/retention.py

@@ -0,0 +1,284 @@
+"""Explain what keeps an object alive by walking its referrers back to a root.
+
+A root is a module global or a variable in a running frame. When the walk instead reaches an
+object that nothing in Python refers to, yet it is still alive, the reference comes from
+somewhere the garbage collector cannot see: usually a C extension (gRPC, a database driver,
+an HTTP client) that stored the object without exposing it.
+"""
+
+from __future__ import annotations
+
+import gc
+import re
+import sys
+import types
+from collections import Counter, deque
+from collections.abc import Collection
+from dataclasses import dataclass
+
+from .growth import type_name
+
+_THIS_FILE = __file__
+
+
+@dataclass(frozen=True)
+class RetentionPath:
+    """One chain of references, root first and the object being explained last."""
+
+    steps: tuple[str, ...]
+    root: str
+    outside_python: bool
+
+    def format(self) -> str:
+        lines = [f"  {self.root}"]
+        lines += [f"    -> {step}" for step in self.steps]
+        return "\n".join(lines)
+
+
+@dataclass(frozen=True)
+class RetentionReport:
+    type_name: str
+    instances: int
+    paths: tuple[tuple[RetentionPath, int], ...]
+
+    @property
+    def outside_python(self) -> bool:
+        return any(path.outside_python for path, _ in self.paths)
+
+    def format(self) -> str:
+        if not self.paths:
+            return f"No live {self.type_name} objects found."
+        sampled = sum(count for _, count in self.paths)
+        lines = [f"{self.instances} live {self.type_name}; traced {sampled}:"]
+        for path, count in self.paths:
+            lines.append(f"{count} of {sampled} held by:")
+            lines.append(path.format())
+        return "\n".join(lines)
+
+
+def why_alive(
+    target: object,
+    *,
+    samples: int = 5,
+    max_nodes: int = 5_000,
+    exclude: Collection[int] = (),
+) -> RetentionReport:
+    """Explain what keeps objects alive.
+
+    `target` is a type, a type name as printed by `measure_growth` (e.g. "myapp.Graph"), or
+    a single object. For a type, up to `samples` instances spread across the live ones are
+    traced, and identical chains are grouped, so one report covers the common case and the
+    exceptions. Only objects tracked by the garbage collector can be found by type.
+
+    `exclude` is a set of object ids to skip, e.g. the instances that existed before the
+    code under test ran. That matters for builtin types: tracing "list" otherwise samples
+    arbitrary lists from all over the process.
+    """
+    gc.collect()  # unreachable cycles are garbage, not leaks: free them before looking
+    caller = sys._getframe(1)
+    skip_frames = {id(caller)} if not isinstance(target, (type, str)) else set()
+
+    if isinstance(target, (type, str)):
+        name = target if isinstance(target, str) else _qualified(target)
+        picked, total = _pick_instances(name, samples, exclude)
+    else:
+        name = type_name(target)
+        picked = [target]
+        total = 1
+
+    module_dicts = {id(vars(m)): m_name for m_name, m in list(sys.modules.items()) if m is not None}
+    frame_locals = _running_frame_locals(skip_frames)
+    del caller
+    # Group by the shape of the chain, so list[3] and list[7] count as the same cause.
+    first_of_shape: dict[tuple[str, tuple[str, ...]], RetentionPath] = {}
+    counts: Counter[tuple[str, tuple[str, ...]]] = Counter()
+    for obj in picked:
+        path = _trace(obj, {id(picked)}, skip_frames, module_dicts, frame_locals, max_nodes)
+        shape = (path.root, tuple(_INDEX.sub("[..]", step) for step in path.steps))
+        first_of_shape.setdefault(shape, path)
+        counts[shape] += 1
+    del picked
+    return RetentionReport(name, total, tuple((first_of_shape[s], n) for s, n in counts.most_common()))
+
+
+_INDEX = re.compile(r"\[[^\]]*\]")
+
+
+def _running_frame_locals(skip_frames: set[int]) -> dict[int, str]:
+    """Map id(value) to a description, for locals of functions still executing on any thread.
+
+    Since Python 3.11 the garbage collector does not report executing frames as referrers, so
+    without this an object held only by a running function's variable would look native.
+    """
+    found: dict[int, str] = {}
+    for top in sys._current_frames().values():
+        frame: types.FrameType | None = top
+        while frame is not None:
+            code = frame.f_code
+            if id(frame) not in skip_frames and code.co_filename != _THIS_FILE:
+                where = f"in {code.co_name}() at {code.co_filename}:{frame.f_lineno}"
+                for var, value in frame.f_locals.items():
+                    found.setdefault(id(value), f"variable {var} {where}")
+            frame = frame.f_back
+    return found
+
+
+def _qualified(cls: type) -> str:
+    return cls.__qualname__ if cls.__module__ == "builtins" else f"{cls.__module__}.{cls.__qualname__}"
+
+
+def _pick_instances(name: str, samples: int, exclude: Collection[int]) -> tuple[list[object], int]:
+    """Up to `samples` evenly spaced instances of the named type, and how many are alive."""
+    found = [obj for obj in gc.get_objects() if type_name(obj) == name and id(obj) not in exclude]
+    total = len(found)
+    if total <= samples:
+        picked: list[object] = list(found)
+    else:
+        step = (total - 1) / (samples - 1) if samples > 1 else 0
+        picked = [found[round(i * step)] for i in range(samples)]
+    del found
+    return picked, total
+
+
+def _trace(
+    target: object,
+    ignored: set[int],
+    skip_frames: set[int],
+    module_dicts: dict[int, str],
+    frame_locals: dict[int, str],
+    max_nodes: int,
+) -> RetentionPath:
+    """Breadth-first search from `target` up its referrers to the nearest root."""
+    nodes: dict[int, object] = {id(target): target}
+    # link[holder] = (held, how the holder refers to it), pointing back towards the target.
+    link: dict[int, tuple[int, str]] = {}
+    queue: deque[int] = deque([id(target)])
+    ours = {id(nodes), id(link), id(queue), *ignored}
+    dead_end: int | None = None
+
+    while queue and len(nodes) < max_nodes:
+        node_id = queue.popleft()
+        if node_id in frame_locals:
+            return _path(frame_locals[node_id], node_id, nodes, link, outside_python=False)
+        holders = [r for r in gc.get_referrers(nodes[node_id]) if not _is_ours(r, ours, skip_frames)]
+        ours.add(id(holders))
+        try:
+            if not holders and dead_end is None:
+                dead_end = node_id
+            for holder in holders:
+                holder_id = id(holder)
+                if holder_id in nodes:
+                    continue
+                how = _how_held(holder, nodes[node_id])
+                if holder_id in module_dicts:
+                    root = f"module {module_dicts[holder_id]}, global {how.strip('[]')}"
+                    return _path(root, node_id, nodes, link, outside_python=False)
+                if isinstance(holder, types.FrameType):
+                    code = holder.f_code
+                    root = (
+                        f"variable {how or '?'} in {code.co_name}() at {code.co_filename}:{holder.f_lineno}"
+                    )
+                    return _path(root, node_id, nodes, link, outside_python=False)
+                nodes[holder_id] = holder
+                link[holder_id] = (node_id, how)
+                queue.append(holder_id)
+        finally:
+            ours.discard(id(holders))
+            del holders
+
+    if dead_end is not None:
+        kind = type_name(nodes[dead_end])
+        root = (
+            f"outside Python: nothing the garbage collector can see refers to this {kind}, "
+            "yet it is alive (typically a C extension holding it)"
+        )
+        return _path(root, dead_end, nodes, link, outside_python=True)
+    return RetentionPath((type_name(target),), f"no root found within {max_nodes} objects", False)
+
+
+def _path(
+    root: str,
+    start: int,
+    nodes: dict[int, object],
+    link: dict[int, tuple[int, str]],
+    *,
+    outside_python: bool,
+) -> RetentionPath:
+    """Render the chain from `start` down to the target, merging `obj.__dict__['x']` into `obj.x`."""
+    hops: list[tuple[str, str]] = []
+    current = start
+    while True:
+        obj = nodes[current]
+        nxt = link.get(current)
+        hops.append((_label(obj), nxt[1] if nxt else ""))
+        if nxt is None:
+            break
+        current = nxt[0]
+
+    steps: list[str] = []
+    i = 0
+    while i < len(hops):
+        label, how = hops[i]
+        if how == ".__dict__" and i + 1 < len(hops) and hops[i + 1][1].startswith("["):
+            key = hops[i + 1][1][1:-1].strip("'\"")
+            steps.append(f"{label}.{key}")
+            i += 2
+            continue
+        steps.append(f"{label}{how}")
+        i += 1
+    steps[-1] += "  <- the object being explained"
+    return RetentionPath(tuple(steps), root, outside_python)
+
+
+def _is_ours(obj: object, ours: set[int], skip_frames: set[int]) -> bool:
+    if id(obj) in ours or id(obj) in skip_frames:
+        return True
+    return isinstance(obj, types.FrameType) and obj.f_code.co_filename == _THIS_FILE
+
+
+def _label(obj: object) -> str:
+    if isinstance(obj, type):
+        return f"class {obj.__qualname__}"
+    if isinstance(obj, types.MethodType):
+        return f"bound method {obj.__qualname__}"
+    if isinstance(obj, types.FunctionType):
+        return f"function {obj.__qualname__}"
+    return type_name(obj)
+
+
+def _how_held(holder: object, held: object) -> str:
+    """How `holder` refers to `held`, e.g. "['key']", "[3]" or ".__dict__"."""
+    try:
+        if isinstance(holder, dict):
+            for key, value in holder.items():
+                if value is held:
+                    return f"[{key!r}]"
+                if key is held:
+                    return " (dict key)"
+            return ""
+        if isinstance(holder, (list, tuple)):
+            for index, value in enumerate(holder):
+                if value is held:
+                    return f"[{index}]"
+            return ""
+        if isinstance(holder, (set, frozenset)):
+            return " (set member)"
+        if isinstance(holder, types.FrameType):
+            for key, value in holder.f_locals.items():
+                if value is held:
+                    return key
+            return ""
+        if isinstance(holder, types.MethodType):
+            return ".__self__" if holder.__self__ is held else ".__func__"
+        if isinstance(holder, types.CellType):
+            return " (closure variable)"
+        if isinstance(held, dict) and (isinstance(holder, type) or getattr(holder, "__dict__", None) is held):
+            return ".__dict__"
+        attributes = getattr(holder, "__dict__", None)
+        if isinstance(attributes, dict):
+            for key, value in attributes.items():
+                if value is held:
+                    return f".{key}"
+    except Exception:  # an exotic holder (proxy, broken __getattr__) just gets no label
+        return ""
+    return ""

python_leak_hunter-0.1.1.dist-info/METADATA

@@ -0,0 +1,188 @@
+Metadata-Version: 2.5
+Name: python-leak-hunter
+Version: 0.1.1
+Summary: Find memory leaks in long-running Python services: what accumulates, what keeps it alive, and a test that keeps it fixed. Ships with an agent skill for Claude Code and Codex.
+Project-URL: Homepage, https://github.com/tariqamarneh/python-leak-hunter
+Project-URL: The leak behind it, https://www.tariqamarneh.com/writing/grpc-contextvars-memory-leak
+Author-email: Tariq Amarneh <tariqs.naser@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent skill,contextvars,debugging,gc,langgraph,memory leak,profiling,pytest
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Debuggers
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Provides-Extra: psutil
+Requires-Dist: psutil>=5.9; extra == 'psutil'
+Description-Content-Type: text/markdown
+
+# python-leak-hunter
+
+[![CI](https://github.com/tariqamarneh/python-leak-hunter/actions/workflows/ci.yml/badge.svg)](https://github.com/tariqamarneh/python-leak-hunter/actions/workflows/ci.yml)
+
+Find memory leaks in long-running Python services: **what accumulates**, **what keeps it
+alive** (down to "held by a C extension"), and **a test that keeps it fixed**. It ships with an
+[agent skill](https://github.com/tariqamarneh/python-leak-hunter/blob/main/skills/python-leak-hunter/SKILL.md) that teaches Claude Code, Codex and other
+coding agents the whole investigation, not just the tools.
+
+```text
+$ leak-hunter why harness:many_requests app.RunTrace
+
+43 live app.RunTrace; traced 5:
+5 of 5 held by:
+  outside Python: nothing the garbage collector can see refers to this grpc._plugin_wrapping._Plugin, yet it is alive (typically a C extension holding it)
+    -> grpc._plugin_wrapping._Plugin._stored_ctx
+    -> _contextvars.Context
+    -> hamt
+    -> hamt_bitmap_node
+    -> app.RunTrace  <- the object being explained
+```
+
+That output is from [a real grpcio reproduction](https://github.com/tariqamarneh/python-leak-hunter/blob/main/examples/grpc_cycle.py): a gRPC channel built
+inside a request captures the request's `contextvars` context. When that context can reach
+the channel back, the cycle runs through C code and is never freed. In production this pinned
+whole AI-agent graphs in memory, about 2 GB every 6 hours per worker.
+[The full story](https://www.tariqamarneh.com/writing/grpc-contextvars-memory-leak).
+
+## Use it as an agent skill
+
+The skill walks an agent through the method: confirm the growth is real, reproduce one unit
+of work, find what accumulates, trace what holds it, fix it by pattern, then prove it and add a
+regression test. It installs the package below as needed.
+
+```bash
+git clone https://github.com/tariqamarneh/python-leak-hunter
+
+# Claude Code: for you, or for one project
+cp -r python-leak-hunter/skills/python-leak-hunter ~/.claude/skills/
+cp -r python-leak-hunter/skills/python-leak-hunter your-project/.claude/skills/
+
+# Codex: per repository
+cp -r python-leak-hunter/skills/python-leak-hunter your-project/.agents/skills/
+```
+
+Then ask: *"Memory in our worker keeps growing. Find the leak."*
+
+## Use it directly
+
+```bash
+pip install python-leak-hunter
+```
+
+No dependencies. Python 3.10+.
+
+### 1. What accumulates?
+
+Write a function that runs one unit of work (a request, a message, an agent turn), then:
+
+```bash
+leak-hunter check harness:one_request --include myapp. --iterations 30
+```
+
+```text
+Objects accumulating across calls (2 windows x 30 calls):
+  myapp.RunTrace  +1 per call  (window 1: +30, window 2: +30)
+Next: why_alive('myapp.RunTrace') shows what keeps them alive.
+```
+
+The work runs in two windows after a warm-up, and only types that grow in **both** are
+reported, so a cache filling up once is not a leak. Async functions work too. Add `--trace` to
+also list allocation sites, which catches `bytes` and `str` that the object count cannot see.
+
+### 2. What keeps it alive?
+
+```bash
+leak-hunter why harness:many_requests myapp.RunTrace
+```
+
+It walks referrers back to a root and groups instances that share a chain:
+
+- `module X, global 'name'`: a module-level cache or registry.
+- `variable v in f() at file:line`: a long-running function's local.
+- `outside Python: ...`: nothing in Python refers to that object, yet it is alive, so native
+  code holds it.
+
+### 3. Keep it fixed
+
+The `leak_check` pytest fixture is registered when the package is installed:
+
+```python
+def test_request_does_not_leak(leak_check):
+    leak_check(handle_one_request, include=["myapp."], iterations=30)
+```
+
+```python
+from leak_hunter import assert_no_leak, assert_no_leak_async, measure_growth, why_alive
+
+report = measure_growth(handle_one_request, include=["myapp."])
+print(report.format())
+print(why_alive("myapp.RunTrace").format())
+```
+
+### Is it even a leak?
+
+Sample memory in the running service and let `diagnose` separate real retention from
+fragmentation and native allocations:
+
+```python
+import logging
+from leak_hunter import MemorySampler, diagnose
+
+sampler = MemorySampler(interval=60, on_sample=lambda s: logging.info("memory %s", s.to_json()))
+sampler.start()
+...
+print(diagnose(sampler.samples))
+```
+
+```text
+RSS grows +312.4 MB/h.
+malloc heap in use grows +305.1 MB/h: memory is being retained, not fragmented. Find what accumulates (measure_growth), then what holds it (why_alive).
+```
+
+On glibc it reads `mallinfo2()` to split malloc's heap into bytes in use and bytes free. If
+free space grows while in-use stays flat, the problem is fragmentation, and hunting objects
+would be a waste of time.
+
+## Try the demo
+
+```bash
+git clone https://github.com/tariqamarneh/python-leak-hunter && cd python-leak-hunter
+uv run python examples/contextvars_leak/demo.py          # no gRPC needed
+uv run --with grpcio python examples/grpc_cycle.py       # the real thing
+```
+
+## How it works
+
+- **Growth:** counts live objects tracked by the garbage collector, by type, after a full
+  collection, around two equal windows of calls.
+- **Retention:** breadth-first search over `gc.get_referrers` from an instance to the nearest
+  module global or running frame, skipping the tool's own references. Since Python 3.11,
+  executing frames are not reported as referrers, so their locals are checked separately; an
+  object held by a running function is not mistaken for native memory. A chain that ends at
+  an object with no visible referrers is reported as held outside Python.
+- **Memory:** RSS from `psutil` when installed, otherwise `/proc` or `ps`; heap stats from
+  glibc's `mallinfo2()`; open file descriptors.
+
+## Limits
+
+- Finding instances by type only works for objects the garbage collector tracks: class
+  instances and containers, not plain `str`, `bytes` or `int`. Use `--trace` for those.
+- `why_alive` scans the heap at each step. Use it in a test or a staging process, not on a
+  hot path in production.
+- The heap split needs glibc 2.33+ (most Linux images). Elsewhere, `diagnose` falls back to the
+  object count.
+
+## Development
+
+```bash
+uv sync
+uv run pytest
+uv run ruff check . && uv run mypy
+```
+
+MIT licensed. Built by [Tariq Amarneh](https://www.tariqamarneh.com).

python_leak_hunter-0.1.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+leak_hunter/__init__.py,sha256=oHpF7bEtwQ-sWZe8qx3OqpmLE0mE9BAlRaJORntbFQM,980
+leak_hunter/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
+leak_hunter/cli.py,sha256=ldQwpFw0bseo7x-98hwUn25Wi_AV83KbfFb3_4MQipc,3438
+leak_hunter/growth.py,sha256=r8CtNPyWj1lA9glUaFB0WJLNSrV4VEIvD4gYlGrCD5Y,10297
+leak_hunter/memory.py,sha256=2sefNlUei3BgUGtBjMyu8OIOD3gXoe79-6LcJBR5ciM,7604
+leak_hunter/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+leak_hunter/pytest_plugin.py,sha256=3R6sYS8b4v9y86oPJmmCGcPalVhZQjjrdlKdT9QoQ98,569
+leak_hunter/retention.py,sha256=bjIE2lE0CmgqSLQdFMHzVRdZx2Ye7-VpuSfzHdC17qQ,10893
+python_leak_hunter-0.1.1.dist-info/METADATA,sha256=bVFQulTBooaTL8ke4-MKi1WcUIGe6cb_6QUSccRPHkY,7323
+python_leak_hunter-0.1.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+python_leak_hunter-0.1.1.dist-info/entry_points.txt,sha256=y3fhMVxLtEeUQcSEqUKFQQILfall0Pdp7pVAbDRCXyI,105
+python_leak_hunter-0.1.1.dist-info/licenses/LICENSE,sha256=E3TQTPEnoy1kVP3olEtqKd2YRGIp7TWk7xHf2fqMhW8,1070
+python_leak_hunter-0.1.1.dist-info/RECORD,,

python_leak_hunter-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

python_leak_hunter-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+leak-hunter = leak_hunter.cli:main
+
+[pytest11]
+leak_hunter = leak_hunter.pytest_plugin

python_leak_hunter-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tariq Amarneh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.