python-leak-hunter 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- python_leak_hunter-0.1.1/.github/workflows/ci.yml +33 -0
- python_leak_hunter-0.1.1/.github/workflows/release.yml +33 -0
- python_leak_hunter-0.1.1/.gitignore +9 -0
- python_leak_hunter-0.1.1/LICENSE +21 -0
- python_leak_hunter-0.1.1/PKG-INFO +188 -0
- python_leak_hunter-0.1.1/README.md +165 -0
- python_leak_hunter-0.1.1/examples/contextvars_leak/app.py +79 -0
- python_leak_hunter-0.1.1/examples/contextvars_leak/demo.py +25 -0
- python_leak_hunter-0.1.1/examples/grpc_cycle.py +60 -0
- python_leak_hunter-0.1.1/pyproject.toml +63 -0
- python_leak_hunter-0.1.1/skills/python-leak-hunter/SKILL.md +176 -0
- python_leak_hunter-0.1.1/skills/python-leak-hunter/references/culprits.md +139 -0
- python_leak_hunter-0.1.1/skills/python-leak-hunter/references/native-memory.md +62 -0
- python_leak_hunter-0.1.1/src/leak_hunter/__init__.py +39 -0
- python_leak_hunter-0.1.1/src/leak_hunter/__main__.py +3 -0
- python_leak_hunter-0.1.1/src/leak_hunter/cli.py +86 -0
- python_leak_hunter-0.1.1/src/leak_hunter/growth.py +305 -0
- python_leak_hunter-0.1.1/src/leak_hunter/memory.py +230 -0
- python_leak_hunter-0.1.1/src/leak_hunter/py.typed +0 -0
- python_leak_hunter-0.1.1/src/leak_hunter/pytest_plugin.py +20 -0
- python_leak_hunter-0.1.1/src/leak_hunter/retention.py +284 -0
- python_leak_hunter-0.1.1/tests/test_growth.py +149 -0
- python_leak_hunter-0.1.1/tests/test_grpc_cycle.py +36 -0
- python_leak_hunter-0.1.1/tests/test_memory.py +87 -0
- python_leak_hunter-0.1.1/tests/test_plugin_and_cli.py +71 -0
- python_leak_hunter-0.1.1/tests/test_retention.py +144 -0
- python_leak_hunter-0.1.1/uv.lock +523 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
lint:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v4
|
|
13
|
+
- uses: astral-sh/setup-uv@v6
|
|
14
|
+
- run: uv sync --python 3.13
|
|
15
|
+
- run: uv run ruff check .
|
|
16
|
+
- run: uv run ruff format --check .
|
|
17
|
+
- run: uv run mypy
|
|
18
|
+
|
|
19
|
+
test:
|
|
20
|
+
strategy:
|
|
21
|
+
fail-fast: false
|
|
22
|
+
matrix:
|
|
23
|
+
os: [ubuntu-latest, macos-latest]
|
|
24
|
+
python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
25
|
+
runs-on: ${{ matrix.os }}
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@v4
|
|
28
|
+
- uses: astral-sh/setup-uv@v6
|
|
29
|
+
- run: uv sync --python ${{ matrix.python }}
|
|
30
|
+
- run: uv run pytest
|
|
31
|
+
# Once with real grpcio, so the gRPC reproduction runs instead of being skipped.
|
|
32
|
+
- if: runner.os == 'Linux' && matrix.python == '3.12'
|
|
33
|
+
run: uv run --with grpcio pytest tests/test_grpc_cycle.py
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Publishes to PyPI with trusted publishing: PyPI trusts this workflow directly,
|
|
4
|
+
# so no API token is stored anywhere.
|
|
5
|
+
on:
|
|
6
|
+
release:
|
|
7
|
+
types: [published]
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
inputs:
|
|
10
|
+
tag:
|
|
11
|
+
description: Existing tag to publish, e.g. v0.1.0
|
|
12
|
+
required: true
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
pypi:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
environment:
|
|
18
|
+
name: pypi
|
|
19
|
+
url: https://pypi.org/project/python-leak-hunter/
|
|
20
|
+
permissions:
|
|
21
|
+
id-token: write
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
with:
|
|
25
|
+
ref: ${{ github.event.release.tag_name || inputs.tag }}
|
|
26
|
+
- uses: astral-sh/setup-uv@v6
|
|
27
|
+
- name: Check the tag matches the package version
|
|
28
|
+
run: |
|
|
29
|
+
tag="${{ github.event.release.tag_name || inputs.tag }}"
|
|
30
|
+
version="$(uv version --short)"
|
|
31
|
+
test "$tag" = "v$version" || { echo "tag $tag does not match version $version"; exit 1; }
|
|
32
|
+
- run: uv build
|
|
33
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tariq Amarneh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: python-leak-hunter
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Find memory leaks in long-running Python services: what accumulates, what keeps it alive, and a test that keeps it fixed. Ships with an agent skill for Claude Code and Codex.
|
|
5
|
+
Project-URL: Homepage, https://github.com/tariqamarneh/python-leak-hunter
|
|
6
|
+
Project-URL: The leak behind it, https://www.tariqamarneh.com/writing/grpc-contextvars-memory-leak
|
|
7
|
+
Author-email: Tariq Amarneh <tariqs.naser@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: agent skill,contextvars,debugging,gc,langgraph,memory leak,profiling,pytest
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Framework :: Pytest
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
17
|
+
Classifier: Topic :: Software Development :: Testing
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Provides-Extra: psutil
|
|
21
|
+
Requires-Dist: psutil>=5.9; extra == 'psutil'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# python-leak-hunter
|
|
25
|
+
|
|
26
|
+
[](https://github.com/tariqamarneh/python-leak-hunter/actions/workflows/ci.yml)
|
|
27
|
+
|
|
28
|
+
Find memory leaks in long-running Python services: **what accumulates**, **what keeps it
|
|
29
|
+
alive** (down to "held by a C extension"), and **a test that keeps it fixed**. It ships with an
|
|
30
|
+
[agent skill](https://github.com/tariqamarneh/python-leak-hunter/blob/main/skills/python-leak-hunter/SKILL.md) that teaches Claude Code, Codex and other
|
|
31
|
+
coding agents the whole investigation, not just the tools.
|
|
32
|
+
|
|
33
|
+
```text
|
|
34
|
+
$ leak-hunter why harness:many_requests app.RunTrace
|
|
35
|
+
|
|
36
|
+
43 live app.RunTrace; traced 5:
|
|
37
|
+
5 of 5 held by:
|
|
38
|
+
outside Python: nothing the garbage collector can see refers to this grpc._plugin_wrapping._Plugin, yet it is alive (typically a C extension holding it)
|
|
39
|
+
-> grpc._plugin_wrapping._Plugin._stored_ctx
|
|
40
|
+
-> _contextvars.Context
|
|
41
|
+
-> hamt
|
|
42
|
+
-> hamt_bitmap_node
|
|
43
|
+
-> app.RunTrace <- the object being explained
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
That output is from [a real grpcio reproduction](https://github.com/tariqamarneh/python-leak-hunter/blob/main/examples/grpc_cycle.py): a gRPC channel built
|
|
47
|
+
inside a request captures the request's `contextvars` context. When that context can reach
|
|
48
|
+
the channel back, the cycle runs through C code and is never freed. In production this pinned
|
|
49
|
+
whole AI-agent graphs in memory, about 2 GB every 6 hours per worker.
|
|
50
|
+
[The full story](https://www.tariqamarneh.com/writing/grpc-contextvars-memory-leak).
|
|
51
|
+
|
|
52
|
+
## Use it as an agent skill
|
|
53
|
+
|
|
54
|
+
The skill walks an agent through the method: confirm the growth is real, reproduce one unit
|
|
55
|
+
of work, find what accumulates, trace what holds it, fix it by pattern, then prove it and add a
|
|
56
|
+
regression test. It installs the package below as needed.
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
git clone https://github.com/tariqamarneh/python-leak-hunter
|
|
60
|
+
|
|
61
|
+
# Claude Code: for you, or for one project
|
|
62
|
+
cp -r python-leak-hunter/skills/python-leak-hunter ~/.claude/skills/
|
|
63
|
+
cp -r python-leak-hunter/skills/python-leak-hunter your-project/.claude/skills/
|
|
64
|
+
|
|
65
|
+
# Codex: per repository
|
|
66
|
+
cp -r python-leak-hunter/skills/python-leak-hunter your-project/.agents/skills/
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Then ask: *"Memory in our worker keeps growing. Find the leak."*
|
|
70
|
+
|
|
71
|
+
## Use it directly
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install python-leak-hunter
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
No dependencies. Python 3.10+.
|
|
78
|
+
|
|
79
|
+
### 1. What accumulates?
|
|
80
|
+
|
|
81
|
+
Write a function that runs one unit of work (a request, a message, an agent turn), then:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
leak-hunter check harness:one_request --include myapp. --iterations 30
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
```text
|
|
88
|
+
Objects accumulating across calls (2 windows x 30 calls):
|
|
89
|
+
myapp.RunTrace +1 per call (window 1: +30, window 2: +30)
|
|
90
|
+
Next: why_alive('myapp.RunTrace') shows what keeps them alive.
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
The work runs in two windows after a warm-up, and only types that grow in **both** are
|
|
94
|
+
reported, so a cache filling up once is not a leak. Async functions work too. Add `--trace` to
|
|
95
|
+
also list allocation sites, which catches `bytes` and `str` that the object count cannot see.
|
|
96
|
+
|
|
97
|
+
### 2. What keeps it alive?
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
leak-hunter why harness:many_requests myapp.RunTrace
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
It walks referrers back to a root and groups instances that share a chain:
|
|
104
|
+
|
|
105
|
+
- `module X, global 'name'`: a module-level cache or registry.
|
|
106
|
+
- `variable v in f() at file:line`: a long-running function's local.
|
|
107
|
+
- `outside Python: ...`: nothing in Python refers to that object, yet it is alive, so native
|
|
108
|
+
code holds it.
|
|
109
|
+
|
|
110
|
+
### 3. Keep it fixed
|
|
111
|
+
|
|
112
|
+
The `leak_check` pytest fixture is registered when the package is installed:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
def test_request_does_not_leak(leak_check):
|
|
116
|
+
leak_check(handle_one_request, include=["myapp."], iterations=30)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from leak_hunter import assert_no_leak, assert_no_leak_async, measure_growth, why_alive
|
|
121
|
+
|
|
122
|
+
report = measure_growth(handle_one_request, include=["myapp."])
|
|
123
|
+
print(report.format())
|
|
124
|
+
print(why_alive("myapp.RunTrace").format())
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Is it even a leak?
|
|
128
|
+
|
|
129
|
+
Sample memory in the running service and let `diagnose` separate real retention from
|
|
130
|
+
fragmentation and native allocations:
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
import logging
|
|
134
|
+
from leak_hunter import MemorySampler, diagnose
|
|
135
|
+
|
|
136
|
+
sampler = MemorySampler(interval=60, on_sample=lambda s: logging.info("memory %s", s.to_json()))
|
|
137
|
+
sampler.start()
|
|
138
|
+
...
|
|
139
|
+
print(diagnose(sampler.samples))
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
```text
|
|
143
|
+
RSS grows +312.4 MB/h.
|
|
144
|
+
malloc heap in use grows +305.1 MB/h: memory is being retained, not fragmented. Find what accumulates (measure_growth), then what holds it (why_alive).
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
On glibc it reads `mallinfo2()` to split malloc's heap into bytes in use and bytes free. If
|
|
148
|
+
free space grows while in-use stays flat, the problem is fragmentation, and hunting objects
|
|
149
|
+
would be a waste of time.
|
|
150
|
+
|
|
151
|
+
## Try the demo
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
git clone https://github.com/tariqamarneh/python-leak-hunter && cd python-leak-hunter
|
|
155
|
+
uv run python examples/contextvars_leak/demo.py # no gRPC needed
|
|
156
|
+
uv run --with grpcio python examples/grpc_cycle.py # the real thing
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## How it works
|
|
160
|
+
|
|
161
|
+
- **Growth:** counts live objects tracked by the garbage collector, by type, after a full
|
|
162
|
+
collection, around two equal windows of calls.
|
|
163
|
+
- **Retention:** breadth-first search over `gc.get_referrers` from an instance to the nearest
|
|
164
|
+
module global or running frame, skipping the tool's own references. Since Python 3.11,
|
|
165
|
+
executing frames are not reported as referrers, so their locals are checked separately; an
|
|
166
|
+
object held by a running function is not mistaken for native memory. A chain that ends at
|
|
167
|
+
an object with no visible referrers is reported as held outside Python.
|
|
168
|
+
- **Memory:** RSS from `psutil` when installed, otherwise `/proc` or `ps`; heap stats from
|
|
169
|
+
glibc's `mallinfo2()`; open file descriptors.
|
|
170
|
+
|
|
171
|
+
## Limits
|
|
172
|
+
|
|
173
|
+
- Finding instances by type only works for objects the garbage collector tracks: class
|
|
174
|
+
instances and containers, not plain `str`, `bytes` or `int`. Use `--trace` for those.
|
|
175
|
+
- `why_alive` scans the heap at each step. Use it in a test or a staging process, not on a
|
|
176
|
+
hot path in production.
|
|
177
|
+
- The heap split needs glibc 2.33+ (most Linux images). Elsewhere, `diagnose` falls back to the
|
|
178
|
+
object count.
|
|
179
|
+
|
|
180
|
+
## Development
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
uv sync
|
|
184
|
+
uv run pytest
|
|
185
|
+
uv run ruff check . && uv run mypy
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
MIT licensed. Built by [Tariq Amarneh](https://www.tariqamarneh.com).
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# python-leak-hunter
|
|
2
|
+
|
|
3
|
+
[](https://github.com/tariqamarneh/python-leak-hunter/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
Find memory leaks in long-running Python services: **what accumulates**, **what keeps it
|
|
6
|
+
alive** (down to "held by a C extension"), and **a test that keeps it fixed**. It ships with an
|
|
7
|
+
[agent skill](https://github.com/tariqamarneh/python-leak-hunter/blob/main/skills/python-leak-hunter/SKILL.md) that teaches Claude Code, Codex and other
|
|
8
|
+
coding agents the whole investigation, not just the tools.
|
|
9
|
+
|
|
10
|
+
```text
|
|
11
|
+
$ leak-hunter why harness:many_requests app.RunTrace
|
|
12
|
+
|
|
13
|
+
43 live app.RunTrace; traced 5:
|
|
14
|
+
5 of 5 held by:
|
|
15
|
+
outside Python: nothing the garbage collector can see refers to this grpc._plugin_wrapping._Plugin, yet it is alive (typically a C extension holding it)
|
|
16
|
+
-> grpc._plugin_wrapping._Plugin._stored_ctx
|
|
17
|
+
-> _contextvars.Context
|
|
18
|
+
-> hamt
|
|
19
|
+
-> hamt_bitmap_node
|
|
20
|
+
-> app.RunTrace <- the object being explained
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
That output is from [a real grpcio reproduction](https://github.com/tariqamarneh/python-leak-hunter/blob/main/examples/grpc_cycle.py): a gRPC channel built
|
|
24
|
+
inside a request captures the request's `contextvars` context. When that context can reach
|
|
25
|
+
the channel back, the cycle runs through C code and is never freed. In production this pinned
|
|
26
|
+
whole AI-agent graphs in memory, about 2 GB every 6 hours per worker.
|
|
27
|
+
[The full story](https://www.tariqamarneh.com/writing/grpc-contextvars-memory-leak).
|
|
28
|
+
|
|
29
|
+
## Use it as an agent skill
|
|
30
|
+
|
|
31
|
+
The skill walks an agent through the method: confirm the growth is real, reproduce one unit
|
|
32
|
+
of work, find what accumulates, trace what holds it, fix it by pattern, then prove it and add a
|
|
33
|
+
regression test. It installs the package below as needed.
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
git clone https://github.com/tariqamarneh/python-leak-hunter
|
|
37
|
+
|
|
38
|
+
# Claude Code: for you, or for one project
|
|
39
|
+
cp -r python-leak-hunter/skills/python-leak-hunter ~/.claude/skills/
|
|
40
|
+
cp -r python-leak-hunter/skills/python-leak-hunter your-project/.claude/skills/
|
|
41
|
+
|
|
42
|
+
# Codex: per repository
|
|
43
|
+
cp -r python-leak-hunter/skills/python-leak-hunter your-project/.agents/skills/
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Then ask: *"Memory in our worker keeps growing. Find the leak."*
|
|
47
|
+
|
|
48
|
+
## Use it directly
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install python-leak-hunter
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
No dependencies. Python 3.10+.
|
|
55
|
+
|
|
56
|
+
### 1. What accumulates?
|
|
57
|
+
|
|
58
|
+
Write a function that runs one unit of work (a request, a message, an agent turn), then:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
leak-hunter check harness:one_request --include myapp. --iterations 30
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
```text
|
|
65
|
+
Objects accumulating across calls (2 windows x 30 calls):
|
|
66
|
+
myapp.RunTrace +1 per call (window 1: +30, window 2: +30)
|
|
67
|
+
Next: why_alive('myapp.RunTrace') shows what keeps them alive.
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The work runs in two windows after a warm-up, and only types that grow in **both** are
|
|
71
|
+
reported, so a cache filling up once is not a leak. Async functions work too. Add `--trace` to
|
|
72
|
+
also list allocation sites, which catches `bytes` and `str` that the object count cannot see.
|
|
73
|
+
|
|
74
|
+
### 2. What keeps it alive?
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
leak-hunter why harness:many_requests myapp.RunTrace
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
It walks referrers back to a root and groups instances that share a chain:
|
|
81
|
+
|
|
82
|
+
- `module X, global 'name'`: a module-level cache or registry.
|
|
83
|
+
- `variable v in f() at file:line`: a long-running function's local.
|
|
84
|
+
- `outside Python: ...`: nothing in Python refers to that object, yet it is alive, so native
|
|
85
|
+
code holds it.
|
|
86
|
+
|
|
87
|
+
### 3. Keep it fixed
|
|
88
|
+
|
|
89
|
+
The `leak_check` pytest fixture is registered when the package is installed:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
def test_request_does_not_leak(leak_check):
|
|
93
|
+
leak_check(handle_one_request, include=["myapp."], iterations=30)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from leak_hunter import assert_no_leak, assert_no_leak_async, measure_growth, why_alive
|
|
98
|
+
|
|
99
|
+
report = measure_growth(handle_one_request, include=["myapp."])
|
|
100
|
+
print(report.format())
|
|
101
|
+
print(why_alive("myapp.RunTrace").format())
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Is it even a leak?
|
|
105
|
+
|
|
106
|
+
Sample memory in the running service and let `diagnose` separate real retention from
|
|
107
|
+
fragmentation and native allocations:
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
import logging
|
|
111
|
+
from leak_hunter import MemorySampler, diagnose
|
|
112
|
+
|
|
113
|
+
sampler = MemorySampler(interval=60, on_sample=lambda s: logging.info("memory %s", s.to_json()))
|
|
114
|
+
sampler.start()
|
|
115
|
+
...
|
|
116
|
+
print(diagnose(sampler.samples))
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
```text
|
|
120
|
+
RSS grows +312.4 MB/h.
|
|
121
|
+
malloc heap in use grows +305.1 MB/h: memory is being retained, not fragmented. Find what accumulates (measure_growth), then what holds it (why_alive).
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
On glibc it reads `mallinfo2()` to split malloc's heap into bytes in use and bytes free. If
|
|
125
|
+
free space grows while in-use stays flat, the problem is fragmentation, and hunting objects
|
|
126
|
+
would be a waste of time.
|
|
127
|
+
|
|
128
|
+
## Try the demo
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
git clone https://github.com/tariqamarneh/python-leak-hunter && cd python-leak-hunter
|
|
132
|
+
uv run python examples/contextvars_leak/demo.py # no gRPC needed
|
|
133
|
+
uv run --with grpcio python examples/grpc_cycle.py # the real thing
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## How it works
|
|
137
|
+
|
|
138
|
+
- **Growth:** counts live objects tracked by the garbage collector, by type, after a full
|
|
139
|
+
collection, around two equal windows of calls.
|
|
140
|
+
- **Retention:** breadth-first search over `gc.get_referrers` from an instance to the nearest
|
|
141
|
+
module global or running frame, skipping the tool's own references. Since Python 3.11,
|
|
142
|
+
executing frames are not reported as referrers, so their locals are checked separately; an
|
|
143
|
+
object held by a running function is not mistaken for native memory. A chain that ends at
|
|
144
|
+
an object with no visible referrers is reported as held outside Python.
|
|
145
|
+
- **Memory:** RSS from `psutil` when installed, otherwise `/proc` or `ps`; heap stats from
|
|
146
|
+
glibc's `mallinfo2()`; open file descriptors.
|
|
147
|
+
|
|
148
|
+
## Limits
|
|
149
|
+
|
|
150
|
+
- Finding instances by type only works for objects the garbage collector tracks: class
|
|
151
|
+
instances and containers, not plain `str`, `bytes` or `int`. Use `--trace` for those.
|
|
152
|
+
- `why_alive` scans the heap at each step. Use it in a test or a staging process, not on a
|
|
153
|
+
hot path in production.
|
|
154
|
+
- The heap split needs glibc 2.33+ (most Linux images). Elsewhere, `diagnose` falls back to the
|
|
155
|
+
object count.
|
|
156
|
+
|
|
157
|
+
## Development
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
uv sync
|
|
161
|
+
uv run pytest
|
|
162
|
+
uv run ruff check . && uv run mypy
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
MIT licensed. Built by [Tariq Amarneh](https://www.tariqamarneh.com).
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""A request handler that leaks the way a gRPC client built per request does.
|
|
2
|
+
|
|
3
|
+
gRPC's Python auth plugin stores `contextvars.copy_context()` when call credentials are
|
|
4
|
+
created, and the native channel keeps that plugin. Build a client inside a request and the
|
|
5
|
+
request's context, with everything in its context variables (tracing run trees, callbacks,
|
|
6
|
+
the agent graph), is now reachable from C code. If anything in that context reaches the
|
|
7
|
+
channel back, the cycle passes through C, the garbage collector cannot see it, and none of
|
|
8
|
+
it is ever freed. examples/grpc_cycle.py shows the same leak with real grpcio.
|
|
9
|
+
|
|
10
|
+
`_hold_in_native_code` stands in for the C extension so this demo needs no gRPC install.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import contextvars
|
|
16
|
+
import ctypes
|
|
17
|
+
import threading
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
current_run: contextvars.ContextVar[RunTrace] = contextvars.ContextVar("current_run")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class RunTrace:
|
|
24
|
+
"""Per-request tracing state, like a LangSmith run tree. Big on purpose."""
|
|
25
|
+
|
|
26
|
+
def __init__(self) -> None:
|
|
27
|
+
self.payload = bytearray(200_000)
|
|
28
|
+
self.children: list[Any] = []
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _hold_in_native_code(obj: object) -> None:
|
|
32
|
+
"""Take a reference the way a C extension does, invisible to the garbage collector."""
|
|
33
|
+
ctypes.pythonapi.Py_IncRef(ctypes.py_object(obj))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class SearchClient:
|
|
37
|
+
"""A client whose constructor captures the caller's context, like grpc credentials."""
|
|
38
|
+
|
|
39
|
+
def __init__(self) -> None:
|
|
40
|
+
_hold_in_native_code(contextvars.copy_context())
|
|
41
|
+
|
|
42
|
+
def search(self, query: str) -> list[str]:
|
|
43
|
+
return [query.upper()]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def handle_request_leaky(query: str = "shoes") -> list[str]:
|
|
47
|
+
token = current_run.set(RunTrace())
|
|
48
|
+
try:
|
|
49
|
+
client = SearchClient() # built inside the request: its context is captured
|
|
50
|
+
return client.search(query)
|
|
51
|
+
finally:
|
|
52
|
+
current_run.reset(token)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
_client: SearchClient | None = None
|
|
56
|
+
_client_lock = threading.Lock()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def get_client() -> SearchClient:
|
|
60
|
+
"""One client per process, constructed in an empty context so it captures nothing."""
|
|
61
|
+
global _client
|
|
62
|
+
if _client is None:
|
|
63
|
+
with _client_lock:
|
|
64
|
+
if _client is None:
|
|
65
|
+
_client = contextvars.Context().run(SearchClient)
|
|
66
|
+
return _client
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def handle_request_fixed(query: str = "shoes") -> list[str]:
|
|
70
|
+
token = current_run.set(RunTrace())
|
|
71
|
+
try:
|
|
72
|
+
return get_client().search(query)
|
|
73
|
+
finally:
|
|
74
|
+
current_run.reset(token)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def handle_many_leaky() -> None:
|
|
78
|
+
for _ in range(50):
|
|
79
|
+
handle_request_leaky()
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Find, explain and confirm the fix for the per-request client leak in app.py.
|
|
2
|
+
|
|
3
|
+
python examples/contextvars_leak/demo.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import app
|
|
9
|
+
|
|
10
|
+
from leak_hunter import measure_growth, why_alive
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main() -> None:
|
|
14
|
+
print("1. What accumulates per request?\n")
|
|
15
|
+
print(measure_growth(app.handle_request_leaky, include=["app."]).format())
|
|
16
|
+
|
|
17
|
+
print("\n2. What keeps it alive?\n")
|
|
18
|
+
print(why_alive("app.RunTrace").format())
|
|
19
|
+
|
|
20
|
+
print("\n3. After the fix (one client per process, built in an empty context):\n")
|
|
21
|
+
print(measure_growth(app.handle_request_fixed, include=["app."]).format())
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
if __name__ == "__main__":
|
|
25
|
+
main()
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""The real thing: a gRPC channel built per request, with the request's context reaching it.
|
|
2
|
+
|
|
3
|
+
pip install grpcio python-leak-hunter
|
|
4
|
+
python examples/grpc_cycle.py
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextvars
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import grpc # type: ignore[import-untyped, unused-ignore]
|
|
13
|
+
|
|
14
|
+
from leak_hunter import measure_growth, why_alive
|
|
15
|
+
|
|
16
|
+
current_run: contextvars.ContextVar[RunTrace] = contextvars.ContextVar("current_run")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class RunTrace:
|
|
20
|
+
"""Per-request state. Like a run tree whose inputs include the client, it reaches the channel."""
|
|
21
|
+
|
|
22
|
+
channel: Any = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _auth(context: Any, callback: Any) -> None:
|
|
26
|
+
callback((("authorization", "Bearer token"),), None)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _channel() -> Any:
|
|
30
|
+
credentials = grpc.composite_channel_credentials(
|
|
31
|
+
grpc.ssl_channel_credentials(), grpc.metadata_call_credentials(_auth)
|
|
32
|
+
)
|
|
33
|
+
return grpc.secure_channel("localhost:50051", credentials)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def handle_request_leaky() -> None:
|
|
37
|
+
run = RunTrace()
|
|
38
|
+
token = current_run.set(run)
|
|
39
|
+
try:
|
|
40
|
+
run.channel = _channel() # credentials capture this request's context
|
|
41
|
+
finally:
|
|
42
|
+
current_run.reset(token)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
_shared = contextvars.Context().run(_channel) # once per process, in an empty context
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def handle_request_fixed() -> None:
|
|
49
|
+
run = RunTrace()
|
|
50
|
+
token = current_run.set(run)
|
|
51
|
+
try:
|
|
52
|
+
run.channel = _shared
|
|
53
|
+
finally:
|
|
54
|
+
current_run.reset(token)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
if __name__ == "__main__":
|
|
58
|
+
print(measure_growth(handle_request_leaky, include=[f"{__name__}."]).format(), end="\n\n")
|
|
59
|
+
print(why_alive(f"{__name__}.RunTrace").format(), end="\n\n")
|
|
60
|
+
print(measure_growth(handle_request_fixed, include=[f"{__name__}."]).format())
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.26"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "python-leak-hunter"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "Find memory leaks in long-running Python services: what accumulates, what keeps it alive, and a test that keeps it fixed. Ships with an agent skill for Claude Code and Codex."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Tariq Amarneh", email = "tariqs.naser@gmail.com" }]
|
|
14
|
+
keywords = ["memory leak", "profiling", "debugging", "pytest", "gc", "contextvars", "langgraph", "agent skill"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Framework :: Pytest",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Topic :: Software Development :: Debuggers",
|
|
22
|
+
"Topic :: Software Development :: Testing",
|
|
23
|
+
"Typing :: Typed",
|
|
24
|
+
]
|
|
25
|
+
dependencies = []
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
psutil = ["psutil>=5.9"]
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://github.com/tariqamarneh/python-leak-hunter"
|
|
32
|
+
"The leak behind it" = "https://www.tariqamarneh.com/writing/grpc-contextvars-memory-leak"
|
|
33
|
+
|
|
34
|
+
[project.scripts]
|
|
35
|
+
leak-hunter = "leak_hunter.cli:main"
|
|
36
|
+
|
|
37
|
+
[project.entry-points.pytest11]
|
|
38
|
+
leak_hunter = "leak_hunter.pytest_plugin"
|
|
39
|
+
|
|
40
|
+
[dependency-groups]
|
|
41
|
+
dev = ["pytest>=8", "mypy>=1.11", "ruff>=0.6"]
|
|
42
|
+
|
|
43
|
+
[tool.hatch.build.targets.wheel]
|
|
44
|
+
packages = ["src/leak_hunter"]
|
|
45
|
+
|
|
46
|
+
[tool.ruff]
|
|
47
|
+
line-length = 110
|
|
48
|
+
target-version = "py310"
|
|
49
|
+
extend-exclude = ["*.md"]
|
|
50
|
+
|
|
51
|
+
[tool.ruff.lint]
|
|
52
|
+
select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
|
|
53
|
+
|
|
54
|
+
[tool.mypy]
|
|
55
|
+
strict = true
|
|
56
|
+
python_version = "3.10"
|
|
57
|
+
files = ["src", "tests", "examples"]
|
|
58
|
+
mypy_path = "examples/contextvars_leak"
|
|
59
|
+
|
|
60
|
+
[tool.pytest.ini_options]
|
|
61
|
+
testpaths = ["tests"]
|
|
62
|
+
addopts = "-ra"
|
|
63
|
+
pythonpath = ["examples/contextvars_leak"]
|