wherewent 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wherewent-0.1.0/LICENSE +21 -0
- wherewent-0.1.0/PKG-INFO +204 -0
- wherewent-0.1.0/README.md +171 -0
- wherewent-0.1.0/pyproject.toml +58 -0
- wherewent-0.1.0/setup.cfg +4 -0
- wherewent-0.1.0/src/wherewent/__init__.py +3 -0
- wherewent-0.1.0/src/wherewent/_shim/__init__.py +2 -0
- wherewent-0.1.0/src/wherewent/_shim/sitecustomize.py +18 -0
- wherewent-0.1.0/src/wherewent/callsite.py +99 -0
- wherewent-0.1.0/src/wherewent/cli.py +90 -0
- wherewent-0.1.0/src/wherewent/normalize.py +55 -0
- wherewent-0.1.0/src/wherewent/recorder.py +383 -0
- wherewent-0.1.0/src/wherewent/report.py +81 -0
- wherewent-0.1.0/src/wherewent/rules.py +119 -0
- wherewent-0.1.0/src/wherewent/stats.py +42 -0
- wherewent-0.1.0/src/wherewent.egg-info/PKG-INFO +204 -0
- wherewent-0.1.0/src/wherewent.egg-info/SOURCES.txt +22 -0
- wherewent-0.1.0/src/wherewent.egg-info/dependency_links.txt +1 -0
- wherewent-0.1.0/src/wherewent.egg-info/entry_points.txt +2 -0
- wherewent-0.1.0/src/wherewent.egg-info/requires.txt +7 -0
- wherewent-0.1.0/src/wherewent.egg-info/top_level.txt +1 -0
- wherewent-0.1.0/tests/test_callsite.py +56 -0
- wherewent-0.1.0/tests/test_normalize.py +114 -0
- wherewent-0.1.0/tests/test_rules.py +323 -0
wherewent-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Habiba Faisal
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
wherewent-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wherewent
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Zero-config SQL flight recorder that answers 'why did this Python batch job take so long?'
|
|
5
|
+
Author-email: Habiba Faisal <habibafaisal8@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/habibafaisal/wherewent
|
|
8
|
+
Project-URL: Repository, https://github.com/habibafaisal/wherewent
|
|
9
|
+
Project-URL: Issues, https://github.com/habibafaisal/wherewent/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/habibafaisal/wherewent/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: profiling,performance,sqlalchemy,postgresql,sql,observability,batch,n+1,orm
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Database
|
|
20
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
21
|
+
Classifier: Topic :: System :: Monitoring
|
|
22
|
+
Classifier: Environment :: Console
|
|
23
|
+
Classifier: Operating System :: OS Independent
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Provides-Extra: psutil
|
|
28
|
+
Requires-Dist: psutil; extra == "psutil"
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest; extra == "dev"
|
|
31
|
+
Requires-Dist: sqlalchemy>=2.0; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
<div align="center">
|
|
35
|
+
|
|
36
|
+
# wherewent
|
|
37
|
+
|
|
38
|
+
### Where did the time go? Find out in one command.
|
|
39
|
+
|
|
40
|
+
**A zero-config recorder that answers "why did this Python batch job take so long?"**
|
|
41
|
+
|
|
42
|
+
[](https://pypi.org/project/wherewent/)
|
|
43
|
+
[](https://pypi.org/project/wherewent/)
|
|
44
|
+
[](LICENSE)
|
|
45
|
+
[](https://github.com/habibafaisal/wherewent/actions/workflows/ci.yml)
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
wherewent run python your_job.py
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
</div>
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## The 0.4ms query that costs you 5 minutes
|
|
56
|
+
|
|
57
|
+
A query can be individually fast — `0.4ms` — and still sink your job, because it's
|
|
58
|
+
called **500,000 times from a single line of code**. Your app burns 300 seconds on
|
|
59
|
+
round-trips while Postgres itself only worked for 80. Every profiler you've tried shows
|
|
60
|
+
you *"time spent in psycopg"* and stops there.
|
|
61
|
+
|
|
62
|
+
**wherewent shows you the calling pattern.** It groups queries by shape, counts how
|
|
63
|
+
often each shape ran, sums the wall time, and points at the exact `file:line` in *your*
|
|
64
|
+
code that fired it — then tells you, in plain English with the arithmetic shown, what to
|
|
65
|
+
do about it.
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
====================================================================================================
|
|
69
|
+
wherewent — SQL flight recorder
|
|
70
|
+
----------------------------------------------------------------------------------------------------
|
|
71
|
+
wall: 26.15s cpu: 25.41s (97% CPU busy) queries: 20,004 commits: 20,001 rollbacks: 1
|
|
72
|
+
in-DB time: 5.46s (20.9% of wall; app-observed: includes network+driver+server)
|
|
73
|
+
commit time: 9.06s total rows: 20,000
|
|
74
|
+
recording added ~1.81s (~6.9% of wall)
|
|
75
|
+
====================================================================================================
|
|
76
|
+
QUERY GROUP CALLS TOTAL MEDIAN CALL SITE
|
|
77
|
+
----------------------------------------------------------------------------------------------------
|
|
78
|
+
INSERT INTO events (name, value) VALUES (?, ?) 20,000 5.46s 0.24ms demo/naive_job.py:65 in main
|
|
79
|
+
SELECT count(*) AS count_1 FROM events 1 0.00s 0.16ms demo/naive_job.py:71 in main
|
|
80
|
+
====================================================================================================
|
|
81
|
+
FINDINGS
|
|
82
|
+
----------------------------------------------------------------------------------------------------
|
|
83
|
+
1. [R1+R2] commit-per-row loop
|
|
84
|
+
20,000 calls x 0.24ms median ~= 5.5s = 21% of 26.1s wall, at demo/naive_job.py:65. Batch it.
|
|
85
|
+
20,001 commits for 20,000 rows (1.0 rows/commit), 9.1s in commit = 35% of wall. Batch to 1,000+ rows/txn.
|
|
86
|
+
~= 14.5s attributable
|
|
87
|
+
====================================================================================================
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Why it's different
|
|
91
|
+
|
|
92
|
+
| | Sampling profilers | APM / tracing | **wherewent** |
|
|
93
|
+
|---|:---:|:---:|:---:|
|
|
94
|
+
| Zero code changes | ✅ | ❌ | ✅ |
|
|
95
|
+
| Groups queries by shape | ❌ | ⚠️ | ✅ |
|
|
96
|
+
| Blames *your* call site | ⚠️ | ⚠️ | ✅ |
|
|
97
|
+
| Tells you the fix | ❌ | ❌ | ✅ |
|
|
98
|
+
| Runs anywhere, no server | ✅ | ❌ | ✅ |
|
|
99
|
+
| Works on a Ctrl-C'd partial run | ❌ | ⚠️ | ✅ |
|
|
100
|
+
|
|
101
|
+
## Install
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
pip install wherewent
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
That's it — the recorder is **pure standard library**. You only need SQLAlchemy
|
|
108
|
+
because *your job* already uses it.
|
|
109
|
+
|
|
110
|
+
## Use it
|
|
111
|
+
|
|
112
|
+
Wrap any command. Your script runs **completely unmodified** — no imports, no decorators,
|
|
113
|
+
no config:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
wherewent run python your_job.py --some arg
|
|
117
|
+
wherewent run python -m your_package
|
|
118
|
+
wherewent run --save run.json python your_job.py # also dump machine-readable JSON
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
- The report prints to **stderr** at exit; your job's own stdout/stderr pass through untouched.
|
|
122
|
+
- **Ctrl-C still produces a report.** Sampling the first 5 minutes of a 14-hour job is the
|
|
123
|
+
main use case — partial data is the point.
|
|
124
|
+
- **It can never crash or corrupt your job.** Every hook body is wrapped so the recorder
|
|
125
|
+
fails silent rather than taking your run down with it.
|
|
126
|
+
- **It never records your data.** Only query *shapes* and *counts* are kept — literal
|
|
127
|
+
values and bind parameters are stripped before anything is stored.
|
|
128
|
+
|
|
129
|
+
### Try the built-in demo
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
git clone https://github.com/habibafaisal/wherewent && cd wherewent
|
|
133
|
+
pip install -e ".[dev]"
|
|
134
|
+
wherewent run python demo/naive_job.py # watch the R1+R2 finding fire
|
|
135
|
+
python demo/benchmark.py # naive vs fixed, with the overhead gate
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## How it works
|
|
139
|
+
|
|
140
|
+
1. **Injects itself** into the target process via a `PYTHONPATH` sitecustomize shim — no
|
|
141
|
+
changes to your code, no wrapper imports.
|
|
142
|
+
2. **Listens at the class level** — `event.listen(sqlalchemy.engine.Engine, ...)` — so
|
|
143
|
+
*every* engine your app creates is captured automatically, config-free.
|
|
144
|
+
3. **Normalizes each statement** into a query *group*: literals, bind params, `IN`-lists
|
|
145
|
+
and multi-row `VALUES` collapse, so a million distinct inserts become one honest row.
|
|
146
|
+
4. **Resolves the call site** by walking the stack past library frames to the first line
|
|
147
|
+
of *your* code — cheaply: cached by filename, full stacks only for the first 5 samples
|
|
148
|
+
per group, so the hot path stays cheap enough to hit its overhead budget.
|
|
149
|
+
5. **Fires deterministic findings** from three rules, each showing its arithmetic.
|
|
150
|
+
|
|
151
|
+
### The findings engine
|
|
152
|
+
|
|
153
|
+
| Rule | Fires when | Tells you |
|
|
154
|
+
|---|---|---|
|
|
155
|
+
| **R1 — chatty group** | > 1,000 calls, > 10% of wall, median < 5ms | A fast query is called too many times — batch it (`executemany` / `IN`-list / `JOIN`). |
|
|
156
|
+
| **R2 — commit-per-row** | > 100 commits, < 10 rows/commit, > 5% of wall in commit | You're committing per row — batch to 1,000+ rows per transaction. |
|
|
157
|
+
| **R3 — DB-wait bound** | in-DB time > 60% of wall, CPU busy < 30% | The job is round-trip bound, not compute bound. |
|
|
158
|
+
|
|
159
|
+
Findings that share a root cause **merge** (e.g. `R1+R2`), everything under 5% of wall is
|
|
160
|
+
suppressed, and at most the top 3 are shown — ranked by seconds attributable.
|
|
161
|
+
|
|
162
|
+
**Every number is honest.** Query times are labelled *app-observed* (they include network,
|
|
163
|
+
driver, and server time — not just Postgres). Anything that can't be measured prints `—`,
|
|
164
|
+
never a guess. wherewent even times *its own hooks* and reports the overhead it added.
|
|
165
|
+
|
|
166
|
+
## Roadmap — help wanted 🙌
|
|
167
|
+
|
|
168
|
+
wherewent is built to grow **beyond SQLAlchemy**. Seven of its eight modules —
|
|
169
|
+
normalization, call-site resolution, the stats model, the rules engine, the report, the
|
|
170
|
+
CLI, and the injection shim — are already **framework-agnostic**. They operate on a plain
|
|
171
|
+
`RunSnapshot` of query events. Only `recorder.py`, which binds SQLAlchemy's event system,
|
|
172
|
+
is framework-specific.
|
|
173
|
+
|
|
174
|
+
**That means a new backend is a well-contained contribution:** capture query
|
|
175
|
+
start/end/rowcount/txn events from another driver, feed the same `RunSnapshot`, and the
|
|
176
|
+
entire findings-and-report pipeline works for free. Good first backends:
|
|
177
|
+
|
|
178
|
+
- [ ] **Raw `psycopg` / `psycopg2`** — cursor subclass or connection factory hook
|
|
179
|
+
- [ ] **`asyncpg` / async SQLAlchemy** — the async execution path
|
|
180
|
+
- [ ] **Django ORM** — via `connection.execute_wrapper`
|
|
181
|
+
- [ ] **Generic DB-API 2.0** — a monkeypatch-free `Cursor` proxy
|
|
182
|
+
- [ ] New findings rules (N+1 `SELECT` detection, lock-wait, seq-scan heuristics)
|
|
183
|
+
|
|
184
|
+
See [`CONTRIBUTING.md`](CONTRIBUTING.md) for the backend contract and the **< 15% overhead
|
|
185
|
+
gate** that every capture path must pass.
|
|
186
|
+
|
|
187
|
+
## Limitations (today)
|
|
188
|
+
|
|
189
|
+
- SQLAlchemy **2.x, synchronous** only (1.4 may work; async does not yet).
|
|
190
|
+
- Single process — no multiprocessing or async fan-out.
|
|
191
|
+
- Query times are app-observed (network + driver + server), by design.
|
|
192
|
+
- Commit timing is obtained by wrapping the dialect's commit; if that wrap fails it prints `—`.
|
|
193
|
+
|
|
194
|
+
These are the honest edges of a validation prototype, not permanent walls — see the roadmap.
|
|
195
|
+
|
|
196
|
+
## Contributing
|
|
197
|
+
|
|
198
|
+
Contributions are very welcome — new backends, new rules, docs, bug reports. Start with
|
|
199
|
+
[`CONTRIBUTING.md`](CONTRIBUTING.md), open an issue to discuss anything substantial, and
|
|
200
|
+
run `pytest && python demo/benchmark.py` before you push.
|
|
201
|
+
|
|
202
|
+
## License
|
|
203
|
+
|
|
204
|
+
[MIT](LICENSE) © 2026 Habiba Faisal
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# wherewent
|
|
4
|
+
|
|
5
|
+
### Where did the time go? Find out in one command.
|
|
6
|
+
|
|
7
|
+
**A zero-config recorder that answers "why did this Python batch job take so long?"**
|
|
8
|
+
|
|
9
|
+
[](https://pypi.org/project/wherewent/)
|
|
10
|
+
[](https://pypi.org/project/wherewent/)
|
|
11
|
+
[](LICENSE)
|
|
12
|
+
[](https://github.com/habibafaisal/wherewent/actions/workflows/ci.yml)
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
wherewent run python your_job.py
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
</div>
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## The 0.4ms query that costs you 5 minutes
|
|
23
|
+
|
|
24
|
+
A query can be individually fast — `0.4ms` — and still sink your job, because it's
|
|
25
|
+
called **500,000 times from a single line of code**. Your app burns 300 seconds on
|
|
26
|
+
round-trips while Postgres itself only worked for 80. Every profiler you've tried shows
|
|
27
|
+
you *"time spent in psycopg"* and stops there.
|
|
28
|
+
|
|
29
|
+
**wherewent shows you the calling pattern.** It groups queries by shape, counts how
|
|
30
|
+
often each shape ran, sums the wall time, and points at the exact `file:line` in *your*
|
|
31
|
+
code that fired it — then tells you, in plain English with the arithmetic shown, what to
|
|
32
|
+
do about it.
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
====================================================================================================
|
|
36
|
+
wherewent — SQL flight recorder
|
|
37
|
+
----------------------------------------------------------------------------------------------------
|
|
38
|
+
wall: 26.15s cpu: 25.41s (97% CPU busy) queries: 20,004 commits: 20,001 rollbacks: 1
|
|
39
|
+
in-DB time: 5.46s (20.9% of wall; app-observed: includes network+driver+server)
|
|
40
|
+
commit time: 9.06s total rows: 20,000
|
|
41
|
+
recording added ~1.81s (~6.9% of wall)
|
|
42
|
+
====================================================================================================
|
|
43
|
+
QUERY GROUP CALLS TOTAL MEDIAN CALL SITE
|
|
44
|
+
----------------------------------------------------------------------------------------------------
|
|
45
|
+
INSERT INTO events (name, value) VALUES (?, ?) 20,000 5.46s 0.24ms demo/naive_job.py:65 in main
|
|
46
|
+
SELECT count(*) AS count_1 FROM events 1 0.00s 0.16ms demo/naive_job.py:71 in main
|
|
47
|
+
====================================================================================================
|
|
48
|
+
FINDINGS
|
|
49
|
+
----------------------------------------------------------------------------------------------------
|
|
50
|
+
1. [R1+R2] commit-per-row loop
|
|
51
|
+
20,000 calls x 0.24ms median ~= 5.5s = 21% of 26.1s wall, at demo/naive_job.py:65. Batch it.
|
|
52
|
+
20,001 commits for 20,000 rows (1.0 rows/commit), 9.1s in commit = 35% of wall. Batch to 1,000+ rows/txn.
|
|
53
|
+
~= 14.5s attributable
|
|
54
|
+
====================================================================================================
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Why it's different
|
|
58
|
+
|
|
59
|
+
| | Sampling profilers | APM / tracing | **wherewent** |
|
|
60
|
+
|---|:---:|:---:|:---:|
|
|
61
|
+
| Zero code changes | ✅ | ❌ | ✅ |
|
|
62
|
+
| Groups queries by shape | ❌ | ⚠️ | ✅ |
|
|
63
|
+
| Blames *your* call site | ⚠️ | ⚠️ | ✅ |
|
|
64
|
+
| Tells you the fix | ❌ | ❌ | ✅ |
|
|
65
|
+
| Runs anywhere, no server | ✅ | ❌ | ✅ |
|
|
66
|
+
| Works on a Ctrl-C'd partial run | ❌ | ⚠️ | ✅ |
|
|
67
|
+
|
|
68
|
+
## Install
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip install wherewent
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
That's it — the recorder is **pure standard library**. You only need SQLAlchemy
|
|
75
|
+
because *your job* already uses it.
|
|
76
|
+
|
|
77
|
+
## Use it
|
|
78
|
+
|
|
79
|
+
Wrap any command. Your script runs **completely unmodified** — no imports, no decorators,
|
|
80
|
+
no config:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
wherewent run python your_job.py --some arg
|
|
84
|
+
wherewent run python -m your_package
|
|
85
|
+
wherewent run --save run.json python your_job.py # also dump machine-readable JSON
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
- The report prints to **stderr** at exit; your job's own stdout/stderr pass through untouched.
|
|
89
|
+
- **Ctrl-C still produces a report.** Sampling the first 5 minutes of a 14-hour job is the
|
|
90
|
+
main use case — partial data is the point.
|
|
91
|
+
- **It can never crash or corrupt your job.** Every hook body is wrapped so the recorder
|
|
92
|
+
fails silent rather than taking your run down with it.
|
|
93
|
+
- **It never records your data.** Only query *shapes* and *counts* are kept — literal
|
|
94
|
+
values and bind parameters are stripped before anything is stored.
|
|
95
|
+
|
|
96
|
+
### Try the built-in demo
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
git clone https://github.com/habibafaisal/wherewent && cd wherewent
|
|
100
|
+
pip install -e ".[dev]"
|
|
101
|
+
wherewent run python demo/naive_job.py # watch the R1+R2 finding fire
|
|
102
|
+
python demo/benchmark.py # naive vs fixed, with the overhead gate
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## How it works
|
|
106
|
+
|
|
107
|
+
1. **Injects itself** into the target process via a `PYTHONPATH` sitecustomize shim — no
|
|
108
|
+
changes to your code, no wrapper imports.
|
|
109
|
+
2. **Listens at the class level** — `event.listen(sqlalchemy.engine.Engine, ...)` — so
|
|
110
|
+
*every* engine your app creates is captured automatically, config-free.
|
|
111
|
+
3. **Normalizes each statement** into a query *group*: literals, bind params, `IN`-lists
|
|
112
|
+
and multi-row `VALUES` collapse, so a million distinct inserts become one honest row.
|
|
113
|
+
4. **Resolves the call site** by walking the stack past library frames to the first line
|
|
114
|
+
of *your* code — cheaply: cached by filename, full stacks only for the first 5 samples
|
|
115
|
+
per group, so the hot path stays cheap enough to hit its overhead budget.
|
|
116
|
+
5. **Fires deterministic findings** from three rules, each showing its arithmetic.
|
|
117
|
+
|
|
118
|
+
### The findings engine
|
|
119
|
+
|
|
120
|
+
| Rule | Fires when | Tells you |
|
|
121
|
+
|---|---|---|
|
|
122
|
+
| **R1 — chatty group** | > 1,000 calls, > 10% of wall, median < 5ms | A fast query is called too many times — batch it (`executemany` / `IN`-list / `JOIN`). |
|
|
123
|
+
| **R2 — commit-per-row** | > 100 commits, < 10 rows/commit, > 5% of wall in commit | You're committing per row — batch to 1,000+ rows per transaction. |
|
|
124
|
+
| **R3 — DB-wait bound** | in-DB time > 60% of wall, CPU busy < 30% | The job is round-trip bound, not compute bound. |
|
|
125
|
+
|
|
126
|
+
Findings that share a root cause **merge** (e.g. `R1+R2`), everything under 5% of wall is
|
|
127
|
+
suppressed, and at most the top 3 are shown — ranked by seconds attributable.
|
|
128
|
+
|
|
129
|
+
**Every number is honest.** Query times are labelled *app-observed* (they include network,
|
|
130
|
+
driver, and server time — not just Postgres). Anything that can't be measured prints `—`,
|
|
131
|
+
never a guess. wherewent even times *its own hooks* and reports the overhead it added.
|
|
132
|
+
|
|
133
|
+
## Roadmap — help wanted 🙌
|
|
134
|
+
|
|
135
|
+
wherewent is built to grow **beyond SQLAlchemy**. Seven of its eight modules —
|
|
136
|
+
normalization, call-site resolution, the stats model, the rules engine, the report, the
|
|
137
|
+
CLI, and the injection shim — are already **framework-agnostic**. They operate on a plain
|
|
138
|
+
`RunSnapshot` of query events. Only `recorder.py`, which binds SQLAlchemy's event system,
|
|
139
|
+
is framework-specific.
|
|
140
|
+
|
|
141
|
+
**That means a new backend is a well-contained contribution:** capture query
|
|
142
|
+
start/end/rowcount/txn events from another driver, feed the same `RunSnapshot`, and the
|
|
143
|
+
entire findings-and-report pipeline works for free. Good first backends:
|
|
144
|
+
|
|
145
|
+
- [ ] **Raw `psycopg` / `psycopg2`** — cursor subclass or connection factory hook
|
|
146
|
+
- [ ] **`asyncpg` / async SQLAlchemy** — the async execution path
|
|
147
|
+
- [ ] **Django ORM** — via `connection.execute_wrapper`
|
|
148
|
+
- [ ] **Generic DB-API 2.0** — a monkeypatch-free `Cursor` proxy
|
|
149
|
+
- [ ] New findings rules (N+1 `SELECT` detection, lock-wait, seq-scan heuristics)
|
|
150
|
+
|
|
151
|
+
See [`CONTRIBUTING.md`](CONTRIBUTING.md) for the backend contract and the **< 15% overhead
|
|
152
|
+
gate** that every capture path must pass.
|
|
153
|
+
|
|
154
|
+
## Limitations (today)
|
|
155
|
+
|
|
156
|
+
- SQLAlchemy **2.x, synchronous** only (1.4 may work; async does not yet).
|
|
157
|
+
- Single process — no multiprocessing or async fan-out.
|
|
158
|
+
- Query times are app-observed (network + driver + server), by design.
|
|
159
|
+
- Commit timing is obtained by wrapping the dialect's commit; if that wrap fails it prints `—`.
|
|
160
|
+
|
|
161
|
+
These are the honest edges of a validation prototype, not permanent walls — see the roadmap.
|
|
162
|
+
|
|
163
|
+
## Contributing
|
|
164
|
+
|
|
165
|
+
Contributions are very welcome — new backends, new rules, docs, bug reports. Start with
|
|
166
|
+
[`CONTRIBUTING.md`](CONTRIBUTING.md), open an issue to discuss anything substantial, and
|
|
167
|
+
run `pytest && python demo/benchmark.py` before you push.
|
|
168
|
+
|
|
169
|
+
## License
|
|
170
|
+
|
|
171
|
+
[MIT](LICENSE) © 2026 Habiba Faisal
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "wherewent"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Zero-config SQL flight recorder that answers 'why did this Python batch job take so long?'"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Habiba Faisal", email = "habibafaisal8@gmail.com" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"profiling",
|
|
16
|
+
"performance",
|
|
17
|
+
"sqlalchemy",
|
|
18
|
+
"postgresql",
|
|
19
|
+
"sql",
|
|
20
|
+
"observability",
|
|
21
|
+
"batch",
|
|
22
|
+
"n+1",
|
|
23
|
+
"orm",
|
|
24
|
+
]
|
|
25
|
+
classifiers = [
|
|
26
|
+
"Development Status :: 3 - Alpha",
|
|
27
|
+
"Intended Audience :: Developers",
|
|
28
|
+
"Programming Language :: Python :: 3",
|
|
29
|
+
"Programming Language :: Python :: 3.10",
|
|
30
|
+
"Programming Language :: Python :: 3.11",
|
|
31
|
+
"Programming Language :: Python :: 3.12",
|
|
32
|
+
"Programming Language :: Python :: 3.13",
|
|
33
|
+
"Topic :: Database",
|
|
34
|
+
"Topic :: Software Development :: Debuggers",
|
|
35
|
+
"Topic :: System :: Monitoring",
|
|
36
|
+
"Environment :: Console",
|
|
37
|
+
"Operating System :: OS Independent",
|
|
38
|
+
]
|
|
39
|
+
# The recorder itself is pure stdlib; SQLAlchemy is only needed by the job you record.
|
|
40
|
+
dependencies = []
|
|
41
|
+
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://github.com/habibafaisal/wherewent"
|
|
44
|
+
Repository = "https://github.com/habibafaisal/wherewent"
|
|
45
|
+
Issues = "https://github.com/habibafaisal/wherewent/issues"
|
|
46
|
+
Changelog = "https://github.com/habibafaisal/wherewent/blob/main/CHANGELOG.md"
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
wherewent = "wherewent.cli:main"
|
|
50
|
+
|
|
51
|
+
[project.optional-dependencies]
|
|
52
|
+
# Optional CPU-accounting backend; the recorder falls back to os.times() without it.
|
|
53
|
+
psutil = ["psutil"]
|
|
54
|
+
# Everything needed to run the test suite and the demo benchmark.
|
|
55
|
+
dev = ["pytest", "sqlalchemy>=2.0"]
|
|
56
|
+
|
|
57
|
+
[tool.setuptools.packages.find]
|
|
58
|
+
where = ["src"]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Auto-imported by Python at startup (this dir is on the child's PYTHONPATH).
|
|
2
|
+
|
|
3
|
+
Only acts when the CLI marked the environment. Any failure is swallowed so the
|
|
4
|
+
job runs exactly as it would without wherewent.
|
|
5
|
+
|
|
6
|
+
TODO: chaining the site's original sitecustomize is out of scope for this
|
|
7
|
+
prototype (if the environment already shipped one, it is shadowed here).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
|
|
12
|
+
if os.environ.get("WHEREWENT_ACTIVE") == "1":
|
|
13
|
+
try:
|
|
14
|
+
from wherewent.recorder import install_from_env
|
|
15
|
+
|
|
16
|
+
install_from_env()
|
|
17
|
+
except Exception:
|
|
18
|
+
pass
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Call-site resolution — find the user-code frame that issued a query.
|
|
2
|
+
|
|
3
|
+
This runs on the hot path (once per execution), so is_library_file is memoized
|
|
4
|
+
and the full stack capture is reserved for the first few samples of each group.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
import sysconfig
|
|
10
|
+
|
|
11
|
+
# memoization cache for is_library_file (keyed by raw filename string)
|
|
12
|
+
_LIB_CACHE: "dict[str, bool]" = {}
|
|
13
|
+
|
|
14
|
+
# directory of the installed wherewent package itself (its frames are "library")
|
|
15
|
+
_WHEREWENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
16
|
+
|
|
17
|
+
# stdlib / install-tree prefixes computed once
|
|
18
|
+
_STDLIB_PREFIXES = set()
|
|
19
|
+
for _key in ("stdlib", "platstdlib", "purelib", "platlib"):
|
|
20
|
+
try:
|
|
21
|
+
_p = sysconfig.get_paths().get(_key)
|
|
22
|
+
if _p:
|
|
23
|
+
_STDLIB_PREFIXES.add(os.path.abspath(_p))
|
|
24
|
+
except Exception:
|
|
25
|
+
pass
|
|
26
|
+
for _p in (getattr(sys, "prefix", None), getattr(sys, "base_prefix", None)):
|
|
27
|
+
if _p:
|
|
28
|
+
_STDLIB_PREFIXES.add(os.path.abspath(_p))
|
|
29
|
+
_STDLIB_PREFIXES = tuple(_STDLIB_PREFIXES)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _compute_is_library(filename: str) -> bool:
|
|
33
|
+
# frozen / built-in / <string> / <stdin> pseudo-files are never user code
|
|
34
|
+
if not filename or filename.startswith("<"):
|
|
35
|
+
return True
|
|
36
|
+
path = os.path.abspath(filename)
|
|
37
|
+
sep = os.sep
|
|
38
|
+
if "site-packages" in path or "dist-packages" in path:
|
|
39
|
+
return True
|
|
40
|
+
if sep + "sqlalchemy" + sep in path:
|
|
41
|
+
return True
|
|
42
|
+
if path.startswith(_WHEREWENT_DIR):
|
|
43
|
+
return True
|
|
44
|
+
for prefix in _STDLIB_PREFIXES:
|
|
45
|
+
if path.startswith(prefix):
|
|
46
|
+
return True
|
|
47
|
+
return False
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def is_library_file(filename: str) -> bool:
|
|
51
|
+
"""True if *filename* belongs to a library/stdlib/wherewent (memoized)."""
|
|
52
|
+
cached = _LIB_CACHE.get(filename)
|
|
53
|
+
if cached is not None:
|
|
54
|
+
return cached
|
|
55
|
+
result = _compute_is_library(filename)
|
|
56
|
+
_LIB_CACHE[filename] = result
|
|
57
|
+
return result
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _short(filename: str) -> str:
|
|
61
|
+
"""A compact display name: path relative to cwd, else basename."""
|
|
62
|
+
try:
|
|
63
|
+
rel = os.path.relpath(filename, os.getcwd())
|
|
64
|
+
if not rel.startswith(".."):
|
|
65
|
+
return rel
|
|
66
|
+
except Exception:
|
|
67
|
+
pass
|
|
68
|
+
return os.path.basename(filename)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def resolve_call_site(skip: int = 1) -> "tuple[str, int, str] | None":
|
|
72
|
+
"""Walk up the stack and return the first user-code frame.
|
|
73
|
+
|
|
74
|
+
Returns (short_file, lineno, function) or None if only library frames exist.
|
|
75
|
+
"""
|
|
76
|
+
try:
|
|
77
|
+
frame = sys._getframe(skip)
|
|
78
|
+
except ValueError:
|
|
79
|
+
return None
|
|
80
|
+
while frame is not None:
|
|
81
|
+
filename = frame.f_code.co_filename
|
|
82
|
+
if not is_library_file(filename):
|
|
83
|
+
return (_short(filename), frame.f_lineno, frame.f_code.co_name)
|
|
84
|
+
frame = frame.f_back
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def capture_stack(limit: int = 30) -> "list[str]":
|
|
89
|
+
"""Return up to *limit* frames (user + library) as 'file:line in func'."""
|
|
90
|
+
frames = []
|
|
91
|
+
try:
|
|
92
|
+
frame = sys._getframe(1)
|
|
93
|
+
except ValueError:
|
|
94
|
+
return frames
|
|
95
|
+
while frame is not None and len(frames) < limit:
|
|
96
|
+
code = frame.f_code
|
|
97
|
+
frames.append(f"{_short(code.co_filename)}:{frame.f_lineno} in {code.co_name}")
|
|
98
|
+
frame = frame.f_back
|
|
99
|
+
return frames
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Console entry point: `wherewent run [--save PATH] <command...>`.
|
|
2
|
+
|
|
3
|
+
Launches the target command in a subprocess whose PYTHONPATH is arranged so
|
|
4
|
+
Python auto-imports our sitecustomize shim, which installs the recorder before
|
|
5
|
+
the job's own code runs. Zero changes to the job itself.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import signal
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
|
|
13
|
+
USAGE = (
|
|
14
|
+
"usage: wherewent run [--save PATH] <command...>\n"
|
|
15
|
+
" e.g. wherewent run python job.py\n"
|
|
16
|
+
" wherewent run --save out.json python -m mypkg\n"
|
|
17
|
+
" wherewent run job.py (bare script uses this interpreter)\n"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _usage():
|
|
22
|
+
sys.stderr.write(USAGE)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _child_env(save):
|
|
26
|
+
import wherewent
|
|
27
|
+
import wherewent._shim as shim
|
|
28
|
+
|
|
29
|
+
# Directory that literally contains sitecustomize.py (so Python auto-imports it).
|
|
30
|
+
shim_dir = os.path.dirname(os.path.abspath(shim.__file__))
|
|
31
|
+
# Directory that contains the `wherewent` package, so the child can import it.
|
|
32
|
+
pkg_parent = os.path.dirname(os.path.dirname(os.path.abspath(wherewent.__file__)))
|
|
33
|
+
|
|
34
|
+
env = os.environ.copy()
|
|
35
|
+
existing = env.get("PYTHONPATH", "")
|
|
36
|
+
parts = [shim_dir, pkg_parent]
|
|
37
|
+
if existing:
|
|
38
|
+
parts.append(existing)
|
|
39
|
+
env["PYTHONPATH"] = os.pathsep.join(parts)
|
|
40
|
+
env["WHEREWENT_ACTIVE"] = "1"
|
|
41
|
+
env["WHEREWENT_SAVE"] = save or ""
|
|
42
|
+
return env
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def main(argv=None):
|
|
46
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
47
|
+
if not argv or argv[0] != "run":
|
|
48
|
+
_usage()
|
|
49
|
+
return 2
|
|
50
|
+
|
|
51
|
+
args = argv[1:]
|
|
52
|
+
save = None
|
|
53
|
+
i = 0
|
|
54
|
+
while i < len(args):
|
|
55
|
+
a = args[i]
|
|
56
|
+
if a == "--save":
|
|
57
|
+
if i + 1 >= len(args):
|
|
58
|
+
_usage()
|
|
59
|
+
return 2
|
|
60
|
+
save = args[i + 1]
|
|
61
|
+
i += 2
|
|
62
|
+
continue
|
|
63
|
+
break # first non-option token starts the command
|
|
64
|
+
command = args[i:]
|
|
65
|
+
|
|
66
|
+
if not command:
|
|
67
|
+
_usage()
|
|
68
|
+
return 2
|
|
69
|
+
|
|
70
|
+
# A bare `script.py` runs under this interpreter.
|
|
71
|
+
if command[0].endswith(".py"):
|
|
72
|
+
command = [sys.executable] + command
|
|
73
|
+
|
|
74
|
+
env = _child_env(save)
|
|
75
|
+
|
|
76
|
+
# While the child runs, the parent ignores Ctrl-C so the SIGINT goes to the
|
|
77
|
+
# child (which finalizes and prints the report); we exit with its code.
|
|
78
|
+
prev = signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
79
|
+
try:
|
|
80
|
+
proc = subprocess.run(command, env=env)
|
|
81
|
+
except FileNotFoundError:
|
|
82
|
+
sys.stderr.write(f"wherewent: command not found: {command[0]}\n")
|
|
83
|
+
return 127
|
|
84
|
+
finally:
|
|
85
|
+
signal.signal(signal.SIGINT, prev)
|
|
86
|
+
return proc.returncode
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == "__main__":
|
|
90
|
+
sys.exit(main())
|