fastapi-loopguard 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastapi_loopguard-0.5.0/CHANGELOG.md +39 -0
- fastapi_loopguard-0.5.0/CLAUDE.md +109 -0
- fastapi_loopguard-0.5.0/FINDINGS.md +84 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/PKG-INFO +13 -6
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/README.md +11 -4
- fastapi_loopguard-0.5.0/docs/CONFIGURATION.md +134 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/pyproject.toml +1 -1
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/src/fastapi_loopguard/__init__.py +8 -2
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/src/fastapi_loopguard/config.py +1 -1
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/src/fastapi_loopguard/context.py +21 -8
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/src/fastapi_loopguard/middleware.py +72 -46
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/src/fastapi_loopguard/monitor.py +43 -12
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/test_cumulative_blocking.py +71 -3
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/test_enforcement_mode.py +55 -60
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/test_middleware.py +243 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/test_monitor.py +126 -15
- fastapi_loopguard-0.5.0/tests/test_packaging.py +42 -0
- fastapi_loopguard-0.4.0/CLAUDE.md +0 -79
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/.github/dependabot.yml +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/.github/workflows/ci.yml +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/.github/workflows/publish.yml +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/.gitignore +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/LICENSE +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/assets/error-page-screenshot-console.png +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/assets/error-page-screenshot-endpoint.png +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/assets/error-page-screenshot.png +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/assets/loopguard-logo.webp +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/examples/demo_app.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/examples/locustfile.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/examples/run_stress_test.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/examples/stress_app.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/src/fastapi_loopguard/logging.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/src/fastapi_loopguard/metrics.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/src/fastapi_loopguard/py.typed +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/src/fastapi_loopguard/pytest_plugin.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/__init__.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/test_config.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/test_context.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/test_logging.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/test_metrics.py +0 -0
- {fastapi_loopguard-0.4.0 → fastapi_loopguard-0.5.0}/tests/test_pytest_plugin.py +0 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.5.0 (2026-08-11)
|
|
4
|
+
|
|
5
|
+
Correctness release. Every fix below changes externally observable behavior.
|
|
6
|
+
|
|
7
|
+
- `dev_mode` no longer turns blocking into 503s. Previously, one endpoint's
|
|
8
|
+
sync block made every concurrent innocent request fail with 503 while the
|
|
9
|
+
actual culprit returned 200. `dev_mode` now only adds diagnostic headers;
|
|
10
|
+
a 503 requires explicitly setting `enforcement_mode="strict"`. Docs no
|
|
11
|
+
longer claim the library identifies which endpoint caused blocking — it
|
|
12
|
+
reports the requests that were in flight during the stall.
|
|
13
|
+
- A single blocking event is reported once, not twice. A 300ms block used to
|
|
14
|
+
produce two events summing to ~600ms in headers and logs (single-shot plus
|
|
15
|
+
a cumulative re-count of the same sample). Cumulative window sums are now
|
|
16
|
+
recorded separately from individual lag samples.
|
|
17
|
+
- Calibration can no longer be poisoned by the app's own blocking. Background
|
|
18
|
+
calibration during a busy blocking startup used to raise the detection
|
|
19
|
+
threshold to ~8x the fallback, after which real blocking went undetected.
|
|
20
|
+
A calibrated threshold now only ever tightens the fallback.
|
|
21
|
+
- The adaptive threshold no longer chases sustained blocking. ~120 blocks of
|
|
22
|
+
~70ms used to yield ~9 detections and a final threshold near 380ms;
|
|
23
|
+
detection now keeps firing for the whole run.
|
|
24
|
+
- Adaptive mode no longer discards a calibration-tightened threshold. Its
|
|
25
|
+
floor used to be pinned at the fallback and its first update fired
|
|
26
|
+
immediately, snapping a calibrated 10ms threshold back to 50ms on the
|
|
27
|
+
first tick. The floor now follows the calibrated threshold.
|
|
28
|
+
- Responses using ASGI extension messages no longer hang in strict mode:
|
|
29
|
+
`http.response.pathsend` (Starlette `FileResponse` on Hypercorn/Granian),
|
|
30
|
+
`http.response.trailers`, and unknown message types now pass through.
|
|
31
|
+
- `"trailers": True` and any other key on `http.response.start` survive
|
|
32
|
+
header injection instead of being silently dropped.
|
|
33
|
+
- The monitor no longer leaks background tasks when startup fails
|
|
34
|
+
(`lifespan.startup.failed` / `lifespan.shutdown.failed` now stop it), and a
|
|
35
|
+
lazily started monitor (apps without lifespan, e.g. tests using
|
|
36
|
+
`httpx.ASGITransport`) stops when the last in-flight request finishes.
|
|
37
|
+
- `__version__` now matches the installed package metadata (it was hard-coded
|
|
38
|
+
to 0.3.0 while the package shipped as 0.4.1), and the 503 error page links
|
|
39
|
+
to the real repository.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
+
|
|
5
|
+
## Project Overview
|
|
6
|
+
|
|
7
|
+
fastapi-loopguard — a middleware library that detects event-loop blocking in FastAPI/Starlette apps with per-request attribution. When synchronous code (`time.sleep`, blocking I/O, CPU-bound work) freezes the async event loop, LoopGuard reports which requests were in flight and, depending on enforcement mode, warns, logs, or fails the response with an educational 503.
|
|
8
|
+
|
|
9
|
+
**Division of docs — do not duplicate them here.** `README.md` owns the pitch, install, and quick-start snippet. `docs/CONFIGURATION.md` is the authoritative reference for every `LoopGuardConfig` option and the recommended recipes; add new options there. This file owns architecture, invariants, and workflow.
|
|
10
|
+
|
|
11
|
+
There is no CONTRIBUTING.md. Release history lives in `CHANGELOG.md` (since 0.5.0) and in git tags (`v0.3.0` …).
|
|
12
|
+
|
|
13
|
+
## Quick Start
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install -e ".[dev]" # dev deps: pytest, mypy, ruff, coverage, httpx, fastapi
|
|
17
|
+
pytest # full suite (asyncio_mode=auto, testpaths=tests)
|
|
18
|
+
pytest tests/test_middleware.py::TestLoopGuardMiddleware::test_dev_mode_headers # one test
|
|
19
|
+
mypy src/ # strict mode, src/ only
|
|
20
|
+
ruff check src/ tests/ # lint — CI runs this exact scope
|
|
21
|
+
ruff format --check src/ tests/ # CI verifies formatting; drop --check to rewrite
|
|
22
|
+
coverage run -m pytest tests/ && coverage report --fail-under=80 # the CI gate
|
|
23
|
+
|
|
24
|
+
pip install -e ".[stress]" # everything under examples/ needs uvicorn + locust
|
|
25
|
+
python examples/demo_app.py # demo on :8765 — /api/users returns 503 (dev_mode)
|
|
26
|
+
python examples/stress_app.py # stress target on :8000
|
|
27
|
+
python examples/run_stress_test.py --skip-locust # validation suite against a running :8000
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
CI lints and type-checks `src/` and `tests/` only — `examples/` is unchecked and does not satisfy the `ANN` rules the rest of the tree does. `tests/test_metrics.py` skips entirely unless `.[prometheus]` is installed, which is why `metrics.py` reports 0% coverage on a plain `.[dev]` run; the 80% gate still passes at ~88% overall.
|
|
31
|
+
|
|
32
|
+
## Tech Stack (versions verified 2026-08-11)
|
|
33
|
+
|
|
34
|
+
- Python `>=3.12`; CI matrix is 3.12 and 3.13. mypy is pinned to `python_version = "3.12"` with `strict = true`.
|
|
35
|
+
- **One runtime dependency: `starlette>=0.37.0,<1.0`.** FastAPI is a *dev* dependency — the middleware is pure ASGI and must never import `fastapi` from `src/`. Adding any runtime dependency requires asking first.
|
|
36
|
+
- Extras: `prometheus` (prometheus-client), `structlog`, `all`, `dev`, `stress` (locust + uvicorn). The `structlog` extra is currently declared but unused — nothing imports structlog.
|
|
37
|
+
- ruff selects `["E","F","I","N","W","UP","B","C4","SIM","ANN"]`, ignoring only `ANN401`. **`ANN` means every function needs full annotations**, tests and fixtures included. Line length 88, double quotes.
|
|
38
|
+
- Build backend is hatchling; wheel packages `src/fastapi_loopguard`. Publishing triggers on a `v*` tag via PyPI trusted publishing (OIDC) — there is **no version-bump automation**, so `pyproject.toml` must be bumped by hand. `__init__.__version__` is derived from installed package metadata; after bumping, re-run `pip install -e .` or the version tests fail against stale metadata.
|
|
39
|
+
- `.claude/` is git-ignored and excluded from the sdist. Committing anything there needs an explicit `.gitignore` negation.
|
|
40
|
+
|
|
41
|
+
## Architecture
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
src/fastapi_loopguard/
|
|
45
|
+
config.py LoopGuardConfig — frozen slotted dataclass + __post_init__ validation
|
|
46
|
+
context.py RequestContext, RequestRegistry, module-global _registry, free functions
|
|
47
|
+
monitor.py SentinelMonitor sleep-measure loop, AdaptiveThreshold
|
|
48
|
+
middleware.py pure-ASGI LoopGuardMiddleware, enforcement modes, HTML/JSON error pages
|
|
49
|
+
logging.py StructuredFormatter (JSON), configure_logging, log_blocking_event
|
|
50
|
+
metrics.py optional Prometheus LoopGuardMetrics (see Known Gaps — not wired in)
|
|
51
|
+
pytest_plugin.py pytest11 entry point, @pytest.mark.no_blocking, BlockingDetector
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
**Layering (strict — a module imports only lower layers):** `config`, `context` → `monitor` → `middleware`. `logging`, `metrics`, and `pytest_plugin` are leaves; nothing on the detection path imports them, and they must not import `middleware`. `middleware.py` imports `LoopGuardConfig` inside `__init__` with an "avoid circular imports" comment — the cycle no longer exists, but the deferred import is harmless and not worth churning.
|
|
55
|
+
|
|
56
|
+
`pytest_plugin.py` is registered as a `pytest11` entry point, so it auto-loads for **every** project that installs this package. Treat its hooks as public API and keep them cheap and side-effect-free for unmarked tests.
|
|
57
|
+
|
|
58
|
+
**Detection flow:** ASGI `lifespan.startup` starts the monitor → `_handle_http` skips `exclude_paths`, then registers a `RequestContext` and writes the id to `scope["state"]["loopguard_request_id"]` → `_monitor_loop` sleeps `monitor_interval_ms` and computes `lag_ms = (elapsed - interval) * 1000` → lag over threshold calls `_handle_blocking`, which calls `record_blocking` on every active context → the send wrapper reads `ctx.blocking_count` / `ctx.total_blocking_ms` and emits headers or a 503 → `finally` unregisters (and stops a lazily started monitor once the registry empties) → any terminal lifespan message (`shutdown.complete`, `shutdown.failed`, `startup.failed`) stops the monitor.
|
|
59
|
+
|
|
60
|
+
## Detection Invariants (non-negotiable)
|
|
61
|
+
|
|
62
|
+
1. **Pure ASGI — never `BaseHTTPMiddleware`.** It is deprecated, breaks contextvars, and leaks memory. `__call__` dispatches on `scope["type"]`; WebSocket and every other type pass through untouched, with no context registered.
|
|
63
|
+
2. **Blocking attributes to ALL active requests.** The sentinel measures loop lag, not call stacks, so it cannot know which request blocked. `_handle_blocking` iterates every context from `get_active_requests()` and records the same lag on each. This over-reports under concurrency **by design** — narrowing it is a redesign, not a bug fix.
|
|
64
|
+
3. **The registry needs no locks.** asyncio is single-threaded, so `RequestRegistry` is a plain dict keyed by `request_id`. Never add a lock, a `threading` primitive, or thread-safety without first changing that premise and saying so.
|
|
65
|
+
4. **Calibration never blocks the first request, and can only tighten.** `start_with_background_calibration()` starts `_monitor_loop` immediately on `fallback_threshold_ms` and calibrates in a named background task. Baseline is the **minimum** of `calibration_iterations` samples (the idle floor — robust to contamination from live traffic), and the calibrated threshold is clamped to `[monitor_interval_ms, fallback_threshold_ms]`: it may lower the fallback, never raise it. The adaptive window is censored (only sub-threshold samples admitted) and its floor follows the calibrated threshold, so neither calibration nor adaptation can be poisoned upward by the app's own blocking. A failed or cancelled calibration keeps the fallback threshold and must never raise into startup.
|
|
66
|
+
5. **Unregister always runs.** `_handle_http` wraps dispatch in `try/finally: unregister_request(request_id)`. An exception from the wrapped app must never leak a context into the registry — a leak makes every later blocking event attribute to a dead request forever.
|
|
67
|
+
6. **`dev_mode` is headers-only; a 503 requires explicit strict mode.** `_get_effective_enforcement_mode()` always returns `enforcement_mode` — `dev_mode` never changes it. Because blocking is attributed to ALL in-flight requests (invariant 2), a status change punishes innocent bystanders, so it must stay opt-in via `enforcement_mode="strict"`. No text anywhere may claim the library identifies WHICH endpoint blocked — it narrows it to the requests in flight during the stall.
|
|
68
|
+
7. **One blocking event is reported exactly once.** A sample that fires the single-shot detection is never appended to `_lag_history`, within one `_monitor_loop` iteration a single-lag trigger suppresses the cumulative one, and `_lag_history` is cleared after a cumulative fire so a window reports at most once. Cumulative window sums are recorded in `RequestContext.cumulative_events`, separate from the individual-lag `blocking_events` list; `blocking_count` / `total_blocking_ms` sum both.
|
|
69
|
+
8. **Lifecycle calls are idempotent, and lazy monitors stop when idle.** `start()`, `start_with_background_calibration()`, and `stop()` all return early when already in the target state. `_handle_http` lazily starts the monitor for apps that run without ASGI lifespan and stops it when the last in-flight request unregisters (so `httpx.ASGITransport` tests leak no tasks); lifespan-managed monitors persist between requests. Double-start must stay a no-op. Consequence: in lazy mode background calibration rarely completes and the fallback threshold governs.
|
|
70
|
+
|
|
71
|
+
## Conventions
|
|
72
|
+
|
|
73
|
+
- **`LoopGuardConfig` is frozen — never mutate it, construct a new one.** All validation lives in `__post_init__` and raises `ValueError` with a message naming the field.
|
|
74
|
+
- Every new config option needs three things: a validation rule in `__post_init__`, a case in `TestLoopGuardConfig`, and a row in `docs/CONFIGURATION.md`. Missing any one of them is an incomplete change.
|
|
75
|
+
- **Response headers are lowercase bytes.** Pass-through responses carry `x-request-id`, `x-blocking-count`, `x-blocking-total-ms`, and `x-blocking-detected`, plus `x-loopguard-warning: blocking-detected` in warn mode. The strict 503 built by `_send_strict_error` carries `x-loopguard-enforcement: strict` and **omits `x-blocking-detected`** — a separate header set, not an extension of the first.
|
|
76
|
+
- `request_id` is the first 8 characters of a `uuid4` — short enough to read in a terminal, not a security token.
|
|
77
|
+
- **All internal timing uses `loop.time()` or `time.monotonic()`** — never `datetime` or wall-clock, which jump under NTP.
|
|
78
|
+
- Hot-path classes use `__slots__`: `LoopGuardConfig`, `RequestContext`, `RequestRegistry`, `SentinelMonitor`, `AdaptiveThreshold`, `LoopGuardMiddleware`. Adding an attribute means adding it to `__slots__`, and `test_context.py` asserts the absence of `__dict__`.
|
|
79
|
+
- Logger name is `"fastapi_loopguard"`, shared by `monitor.py` and `logging.py`. `monitor.py` logs inline with `%`-style lazy formatting; `logging.log_blocking_event()` is a helper for library *users* and is intentionally not called internally.
|
|
80
|
+
- `exclude_paths` is checked before anything else in `_handle_http`, so health checks cost nothing.
|
|
81
|
+
- Backward-compat shims are public API and stay: `get_current_request`, `set_current_request`, `reset_current_request`, `init_metrics`. `get_current_request` returns an arbitrary active context and is only correct for single-request cases — new code uses `get_active_requests()`.
|
|
82
|
+
- `logging`, `metrics`, and `pytest_plugin` are **not** re-exported from `__init__.py`; import them by module path.
|
|
83
|
+
|
|
84
|
+
## Testing
|
|
85
|
+
|
|
86
|
+
- Definition of done: `pytest` green, `mypy src/` clean, `ruff check src/ tests/` and `ruff format --check src/ tests/` clean, and coverage at or above **80** (`--fail-under=80` is the CI gate).
|
|
87
|
+
- **There is no `conftest.py`.** The `clear_registry` fixture is duplicated per file, roughly 15 times. Match the local pattern in the file you are editing; introducing a shared conftest is its own change, not a side effect of an unrelated one.
|
|
88
|
+
- Blocking is simulated with `time.sleep(...)` followed by a short `await asyncio.sleep(...)` — the second call is what lets the sentinel observe the lag while the context is still registered. Omitting it makes the test pass for the wrong reason.
|
|
89
|
+
- HTTP tests use `httpx.AsyncClient(transport=ASGITransport(app=app))`. Plugin tests use the `pytester` fixture to run generated test files in-process.
|
|
90
|
+
- Each invariant has dedicated coverage: `test_enforcement_mode.py` for modes and dev-mode escalation, `test_monitor.py` for calibration, idempotency, and task cancellation, `test_cumulative_blocking.py` for the window, `test_context.py` for registry lifecycle and `__slots__`, `test_pytest_plugin.py` for the marker.
|
|
91
|
+
- Timing tests are inherently flaky under load. Prefer driving `SentinelMonitor` directly with an `on_blocking` callback (as `test_cumulative_blocking.py` does) over asserting on wall-clock durations.
|
|
92
|
+
|
|
93
|
+
## Known Gaps (accurate as of 2026-08-11)
|
|
94
|
+
|
|
95
|
+
Recorded so they are not rediscovered. None are fixed yet; fixing any of them is its own task. The fuller list, including design tensions deferred from the 0.5 correctness pass, is `FINDINGS.md`.
|
|
96
|
+
|
|
97
|
+
1. **`prometheus_enabled` is inert.** Neither `middleware.py` nor `monitor.py` imports `metrics.py`, so enabling the flag exposes nothing. Wiring it up means calling `record_blocking` / `record_request` / `set_threshold` from the monitor.
|
|
98
|
+
2. **`docs/CONFIGURATION.md` names metrics that do not exist.** It lists `loopguard_blocking_events_total` and `loopguard_blocking_duration_ms`; the real names are `loopguard_blocking_total`, `loopguard_lag_seconds`, `loopguard_requests_monitored_total`, and `loopguard_threshold_seconds`.
|
|
99
|
+
3. **`get_metrics()` can never find an instance.** It reads `_instances[prefix]` while `create_metrics` writes `f"{prefix}:{id(registry)}"`. `tests/test_metrics.py:188` documents the mismatch in a comment instead of failing on it.
|
|
100
|
+
4. **`_generate_warning_banner()` is dead code** — defined in `middleware.py`, never called.
|
|
101
|
+
|
|
102
|
+
## Fundamental Guidelines
|
|
103
|
+
|
|
104
|
+
*(Distilled from [andrej-karpathy-skills](https://github.com/forrestchang/andrej-karpathy-skills))*
|
|
105
|
+
|
|
106
|
+
- **Think first.** Surface assumptions and tradeoffs; if a request is ambiguous or a simpler approach exists, say so before coding — don't pick silently.
|
|
107
|
+
- **Simplicity first.** Minimum code that solves the problem. No speculative features, abstractions, config, or error handling for impossible cases.
|
|
108
|
+
- **Surgical changes.** Touch only what the task requires. Match existing style, don't refactor working code, remove only orphans your own change created.
|
|
109
|
+
- **Goal-driven.** Turn tasks into verifiable goals (e.g. "fix bug" → "write a failing test, make it pass"); state a brief plan for multi-step work.
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Findings not fixed in the 0.5 pass
|
|
2
|
+
|
|
3
|
+
Everything noticed while fixing the 0.5 correctness defects but deliberately
|
|
4
|
+
left alone. Each item is its own future task.
|
|
5
|
+
|
|
6
|
+
## Detection design
|
|
7
|
+
|
|
8
|
+
1. **Sub-threshold noise can still inflate the adaptive threshold.**
|
|
9
|
+
The 0.5 fix censors detected-blocking samples out of the window, but
|
|
10
|
+
samples just below the threshold are admitted by design, and
|
|
11
|
+
`P95 x threshold_multiplier` of ~40ms noise still produces a ~230ms
|
|
12
|
+
threshold that would mask a later 100ms block. Per the forensics guidance,
|
|
13
|
+
median+MAD over the censored window would be more robust than a percentile.
|
|
14
|
+
|
|
15
|
+
2. **Strict mode cannot fail a response after `http.response.start` has passed.**
|
|
16
|
+
If blocking is first detected mid-stream, the 200 and its headers are
|
|
17
|
+
already on the wire and the body keeps streaming; the response then claims
|
|
18
|
+
no blocking. Inherent to header-based reporting; worth documenting.
|
|
19
|
+
|
|
20
|
+
3. **Explicit strict mode still 503s all in-flight requests.**
|
|
21
|
+
The sentinel cannot name the culprit, so strict mode punishes bystanders
|
|
22
|
+
under concurrency. Now opt-in and documented, but the real per-callback
|
|
23
|
+
attribution rewrite (the 0.5+ plan) is what removes this.
|
|
24
|
+
|
|
25
|
+
4. **Lazy mode (no ASGI lifespan) rarely completes calibration.**
|
|
26
|
+
The stop-when-idle lifecycle cancels background calibration when the last
|
|
27
|
+
request finishes, so lifespan-less deployments run on the fallback
|
|
28
|
+
threshold and have no between-request monitoring. Accepted tradeoff for
|
|
29
|
+
not leaking tasks; worth a README note if such deployments matter.
|
|
30
|
+
|
|
31
|
+
## ASGI / architecture
|
|
32
|
+
|
|
33
|
+
5. **`exclude_paths` matches the raw `scope["path"]`.**
|
|
34
|
+
Breaks under `root_path` and `Mount`, and `/health/live` is not excluded by
|
|
35
|
+
`/health`. Prefix or route-template matching would fix it.
|
|
36
|
+
|
|
37
|
+
6. **Module-global `RequestRegistry` breaks with two apps in one process.**
|
|
38
|
+
A mounted sub-app or a second middleware-wrapped app cross-attributes
|
|
39
|
+
blocking between apps. Keying state per middleware instance (or per loop)
|
|
40
|
+
would fix it.
|
|
41
|
+
|
|
42
|
+
7. **The package could be zero-dependency.**
|
|
43
|
+
`middleware.py` imports only type aliases from `starlette.types`; moving
|
|
44
|
+
that import under `TYPE_CHECKING` removes the runtime dependency entirely
|
|
45
|
+
(works on Litestar, Quart, Django ASGI, ...). Also, the `<1.0` upper bound
|
|
46
|
+
on starlette will block installs the day Starlette 1.0 ships.
|
|
47
|
+
|
|
48
|
+
8. **`get_current_request()` returns an arbitrary active context.**
|
|
49
|
+
A backward-compat shim that silently gives wrong answers under concurrency.
|
|
50
|
+
Pre-1.0 with few users, removal is cheaper than the confusion.
|
|
51
|
+
|
|
52
|
+
## Dead / inert code and docs drift
|
|
53
|
+
|
|
54
|
+
9. **`prometheus_enabled` is inert** (known gap #1): nothing on the detection
|
|
55
|
+
path imports `metrics.py`. Out of scope here (`metrics.py` untouchable).
|
|
56
|
+
|
|
57
|
+
10. **`_generate_warning_banner()` is dead code** (known gap #4).
|
|
58
|
+
|
|
59
|
+
11. **`docs/CONFIGURATION.md` names metrics that do not exist** (known gap #2)
|
|
60
|
+
and `get_metrics()` can never find an instance (known gap #3).
|
|
61
|
+
|
|
62
|
+
12. **`docs/CONFIGURATION.md` describes `fallback_threshold_ms` as "Used if
|
|
63
|
+
calibration is unreliable"**; since 0.5 it is also the hard ceiling for
|
|
64
|
+
the calibrated threshold.
|
|
65
|
+
|
|
66
|
+
13. **`_handle_lifespan` locals `started` / `shutdown_complete` are write-only.**
|
|
67
|
+
|
|
68
|
+
14. **The `structlog` extra is declared but nothing imports structlog**
|
|
69
|
+
(already noted in CLAUDE.md).
|
|
70
|
+
|
|
71
|
+
## Repro caveat worth keeping
|
|
72
|
+
|
|
73
|
+
15. **Calibration poisoning (defect 2) required blocking to span the whole
|
|
74
|
+
calibration window.** With idle gaps, clean tail samples dominated and the
|
|
75
|
+
old `max(..., fallback)` floor masked the bug. The fix removes the
|
|
76
|
+
mechanism either way, but tests that "prove" poisoning need dense blocking.
|
|
77
|
+
|
|
78
|
+
## Fixed since the first draft
|
|
79
|
+
|
|
80
|
+
- Adaptive mode discarding the calibrated threshold (floor pinned at the
|
|
81
|
+
fallback, first update firing immediately) — fixed on this branch; the
|
|
82
|
+
adaptive floor now follows the calibrated threshold.
|
|
83
|
+
- CLAUDE.md invariants 4 and 6 and known gaps 1-2 describing pre-0.5
|
|
84
|
+
behavior — CLAUDE.md updated on this branch.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: fastapi-loopguard
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Detect event-loop blocking in FastAPI/Starlette with per-request attribution
|
|
5
5
|
Project-URL: Homepage, https://github.com/parhamdavari/fastapi-loopguard
|
|
6
6
|
Project-URL: Documentation, https://github.com/parhamdavari/fastapi-loopguard#readme
|
|
@@ -48,7 +48,7 @@ Description-Content-Type: text/markdown
|
|
|
48
48
|
</p>
|
|
49
49
|
|
|
50
50
|
<p align="center">
|
|
51
|
-
<strong>Catch event-loop blocking in FastAPI
|
|
51
|
+
<strong>Catch event-loop blocking in FastAPI and see which requests were in flight.</strong>
|
|
52
52
|
</p>
|
|
53
53
|
|
|
54
54
|
<p align="center">
|
|
@@ -59,7 +59,7 @@ Description-Content-Type: text/markdown
|
|
|
59
59
|
|
|
60
60
|
---
|
|
61
61
|
|
|
62
|
-
When
|
|
62
|
+
When something blocks your event loop (via `time.sleep()`, blocking I/O, or CPU work), LoopGuard detects it **and narrows it down to the requests that were in flight when the loop stalled**. The sentinel measures loop lag, so it cannot name the single guilty handler — it reports every request that was active during the stall.
|
|
63
63
|
|
|
64
64
|
## Install
|
|
65
65
|
|
|
@@ -88,9 +88,12 @@ app.add_middleware(LoopGuardMiddleware)
|
|
|
88
88
|
```python
|
|
89
89
|
from fastapi_loopguard import LoopGuardConfig
|
|
90
90
|
|
|
91
|
-
# Development:
|
|
91
|
+
# Development: diagnostic headers on every response
|
|
92
92
|
config = LoopGuardConfig(dev_mode=True)
|
|
93
93
|
|
|
94
|
+
# Development / CI: fail loudly with an educational 503
|
|
95
|
+
config = LoopGuardConfig(enforcement_mode="strict")
|
|
96
|
+
|
|
94
97
|
# Production: silent logging
|
|
95
98
|
config = LoopGuardConfig(enforcement_mode="log")
|
|
96
99
|
|
|
@@ -118,10 +121,14 @@ Adds diagnostic headers to every response for debugging:
|
|
|
118
121
|
---
|
|
119
122
|
|
|
120
123
|
### Log Mode
|
|
121
|
-
Writes structured logs
|
|
124
|
+
Writes structured logs listing the requests that were in flight:
|
|
122
125
|
|
|
123
126
|
<p align="center">
|
|
124
127
|
<img src="assets/error-page-screenshot-console.png" alt="Console output" width="600" />
|
|
125
128
|
</p>
|
|
126
129
|
|
|
127
130
|
---
|
|
131
|
+
|
|
132
|
+
<p align="center">
|
|
133
|
+
<a href="docs/CONFIGURATION.md"><strong>Full Configuration Reference</strong></a>
|
|
134
|
+
</p>
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
</p>
|
|
4
4
|
|
|
5
5
|
<p align="center">
|
|
6
|
-
<strong>Catch event-loop blocking in FastAPI
|
|
6
|
+
<strong>Catch event-loop blocking in FastAPI and see which requests were in flight.</strong>
|
|
7
7
|
</p>
|
|
8
8
|
|
|
9
9
|
<p align="center">
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
---
|
|
16
16
|
|
|
17
|
-
When
|
|
17
|
+
When something blocks your event loop (via `time.sleep()`, blocking I/O, or CPU work), LoopGuard detects it **and narrows it down to the requests that were in flight when the loop stalled**. The sentinel measures loop lag, so it cannot name the single guilty handler — it reports every request that was active during the stall.
|
|
18
18
|
|
|
19
19
|
## Install
|
|
20
20
|
|
|
@@ -43,9 +43,12 @@ app.add_middleware(LoopGuardMiddleware)
|
|
|
43
43
|
```python
|
|
44
44
|
from fastapi_loopguard import LoopGuardConfig
|
|
45
45
|
|
|
46
|
-
# Development:
|
|
46
|
+
# Development: diagnostic headers on every response
|
|
47
47
|
config = LoopGuardConfig(dev_mode=True)
|
|
48
48
|
|
|
49
|
+
# Development / CI: fail loudly with an educational 503
|
|
50
|
+
config = LoopGuardConfig(enforcement_mode="strict")
|
|
51
|
+
|
|
49
52
|
# Production: silent logging
|
|
50
53
|
config = LoopGuardConfig(enforcement_mode="log")
|
|
51
54
|
|
|
@@ -73,10 +76,14 @@ Adds diagnostic headers to every response for debugging:
|
|
|
73
76
|
---
|
|
74
77
|
|
|
75
78
|
### Log Mode
|
|
76
|
-
Writes structured logs
|
|
79
|
+
Writes structured logs listing the requests that were in flight:
|
|
77
80
|
|
|
78
81
|
<p align="center">
|
|
79
82
|
<img src="assets/error-page-screenshot-console.png" alt="Console output" width="600" />
|
|
80
83
|
</p>
|
|
81
84
|
|
|
82
85
|
---
|
|
86
|
+
|
|
87
|
+
<p align="center">
|
|
88
|
+
<a href="docs/CONFIGURATION.md"><strong>Full Configuration Reference</strong></a>
|
|
89
|
+
</p>
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# Configuration Reference
|
|
2
|
+
|
|
3
|
+
All configuration options for `LoopGuardConfig`.
|
|
4
|
+
|
|
5
|
+
## Quick Reference
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from fastapi_loopguard import LoopGuardConfig
|
|
9
|
+
|
|
10
|
+
config = LoopGuardConfig(
|
|
11
|
+
# Enforcement
|
|
12
|
+
enforcement_mode="warn", # "log" | "warn" | "strict"
|
|
13
|
+
dev_mode=False, # Adds X-Blocking-* response headers when True
|
|
14
|
+
|
|
15
|
+
# Detection tuning
|
|
16
|
+
monitor_interval_ms=10.0, # How often to check (ms)
|
|
17
|
+
threshold_multiplier=5.0, # Blocking = lag > baseline × multiplier
|
|
18
|
+
fallback_threshold_ms=50.0, # Threshold if calibration fails
|
|
19
|
+
|
|
20
|
+
# Cumulative detection (enabled by default)
|
|
21
|
+
cumulative_blocking_enabled=True,
|
|
22
|
+
cumulative_blocking_threshold_ms=200.0,
|
|
23
|
+
cumulative_window_ms=1000.0,
|
|
24
|
+
|
|
25
|
+
# Adaptive threshold (disabled by default)
|
|
26
|
+
adaptive_threshold=False,
|
|
27
|
+
|
|
28
|
+
# Integrations
|
|
29
|
+
prometheus_enabled=False,
|
|
30
|
+
log_blocking_events=True,
|
|
31
|
+
)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Core Settings
|
|
37
|
+
|
|
38
|
+
| Option | Type | Default | Description |
|
|
39
|
+
|--------|------|---------|-------------|
|
|
40
|
+
| `enabled` | bool | `True` | Master switch. Set `False` to disable entirely. |
|
|
41
|
+
| `enforcement_mode` | str | `"warn"` | How to respond: `"log"`, `"warn"`, or `"strict"` |
|
|
42
|
+
| `dev_mode` | bool | `False` | Enables response headers. Never changes the enforcement mode. |
|
|
43
|
+
| `log_blocking_events` | bool | `True` | Log blocking events to console |
|
|
44
|
+
| `exclude_paths` | frozenset | `{"/health", ...}` | Paths to skip monitoring |
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## Detection Tuning
|
|
49
|
+
|
|
50
|
+
| Option | Type | Default | Description |
|
|
51
|
+
|--------|------|---------|-------------|
|
|
52
|
+
| `monitor_interval_ms` | float | `10.0` | Sentinel check frequency (ms) |
|
|
53
|
+
| `threshold_multiplier` | float | `5.0` | Blocking detected when lag > baseline × this |
|
|
54
|
+
| `calibration_iterations` | int | `100` | Samples during startup calibration |
|
|
55
|
+
| `fallback_threshold_ms` | float | `50.0` | Used if calibration is unreliable |
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## Cumulative Blocking Detection
|
|
60
|
+
|
|
61
|
+
Catches "death by a thousand cuts" - many small blocks that add up.
|
|
62
|
+
|
|
63
|
+
| Option | Type | Default | Description |
|
|
64
|
+
|--------|------|---------|-------------|
|
|
65
|
+
| `cumulative_blocking_enabled` | bool | `True` | Enable cumulative detection |
|
|
66
|
+
| `cumulative_blocking_threshold_ms` | float | `200.0` | Alert if total blocking exceeds this... |
|
|
67
|
+
| `cumulative_window_ms` | float | `1000.0` | ...within this time window (ms) |
|
|
68
|
+
|
|
69
|
+
**Example:** With defaults, alerts if blocking totals >200ms within any 1-second window.
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## Adaptive Threshold
|
|
74
|
+
|
|
75
|
+
Dynamically adjusts threshold based on observed latency. Useful for high-concurrency environments.
|
|
76
|
+
|
|
77
|
+
| Option | Type | Default | Description |
|
|
78
|
+
|--------|------|---------|-------------|
|
|
79
|
+
| `adaptive_threshold` | bool | `False` | Enable adaptive mode |
|
|
80
|
+
| `adaptive_window_size` | int | `1000` | Samples in sliding window |
|
|
81
|
+
| `adaptive_percentile` | float | `0.95` | Percentile for baseline (0.5-0.99) |
|
|
82
|
+
| `adaptive_min_samples` | int | `100` | Min samples before activation |
|
|
83
|
+
| `adaptive_update_interval_ms` | float | `1000.0` | Recalculation frequency (ms) |
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
87
|
+
## Integrations
|
|
88
|
+
|
|
89
|
+
| Option | Type | Default | Description |
|
|
90
|
+
|--------|------|---------|-------------|
|
|
91
|
+
| `prometheus_enabled` | bool | `False` | Expose Prometheus metrics |
|
|
92
|
+
|
|
93
|
+
When enabled, exposes:
|
|
94
|
+
- `loopguard_blocking_events_total` - Counter of blocking events
|
|
95
|
+
- `loopguard_blocking_duration_ms` - Histogram of blocking durations
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## Common Configurations
|
|
100
|
+
|
|
101
|
+
### Development (diagnostic headers)
|
|
102
|
+
```python
|
|
103
|
+
config = LoopGuardConfig(dev_mode=True)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Development / CI (strict enforcement, 503 on blocking)
|
|
107
|
+
```python
|
|
108
|
+
config = LoopGuardConfig(enforcement_mode="strict")
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Production (silent monitoring)
|
|
112
|
+
```python
|
|
113
|
+
config = LoopGuardConfig(
|
|
114
|
+
enforcement_mode="log",
|
|
115
|
+
prometheus_enabled=True,
|
|
116
|
+
)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
### High-concurrency (adaptive threshold)
|
|
120
|
+
```python
|
|
121
|
+
config = LoopGuardConfig(
|
|
122
|
+
adaptive_threshold=True,
|
|
123
|
+
adaptive_percentile=0.99,
|
|
124
|
+
)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Sensitive detection (lower threshold)
|
|
128
|
+
```python
|
|
129
|
+
config = LoopGuardConfig(
|
|
130
|
+
monitor_interval_ms=5.0,
|
|
131
|
+
threshold_multiplier=3.0,
|
|
132
|
+
fallback_threshold_ms=30.0,
|
|
133
|
+
)
|
|
134
|
+
```
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
"""FastAPI LoopGuard - Detect event-loop blocking
|
|
1
|
+
"""FastAPI LoopGuard - Detect event-loop blocking and report in-flight requests.
|
|
2
|
+
|
|
3
|
+
The sentinel measures event-loop lag, so it cannot identify the single
|
|
4
|
+
handler that blocked; it attributes each stall to every request that was
|
|
5
|
+
in flight at the time.
|
|
2
6
|
|
|
3
7
|
Usage:
|
|
4
8
|
from fastapi import FastAPI
|
|
@@ -29,6 +33,8 @@ v0.3.0 Changes:
|
|
|
29
33
|
- High-concurrency configuration documentation
|
|
30
34
|
"""
|
|
31
35
|
|
|
36
|
+
from importlib.metadata import version as _package_version
|
|
37
|
+
|
|
32
38
|
from .config import LoopGuardConfig
|
|
33
39
|
from .context import (
|
|
34
40
|
RequestContext,
|
|
@@ -42,7 +48,7 @@ from .context import (
|
|
|
42
48
|
from .middleware import LoopGuardMiddleware
|
|
43
49
|
from .monitor import SentinelMonitor
|
|
44
50
|
|
|
45
|
-
__version__ = "
|
|
51
|
+
__version__ = _package_version("fastapi-loopguard")
|
|
46
52
|
|
|
47
53
|
__all__ = [
|
|
48
54
|
# Core classes
|
|
@@ -46,7 +46,7 @@ class LoopGuardConfig:
|
|
|
46
46
|
adaptive_update_interval_ms: float = 1000.0
|
|
47
47
|
|
|
48
48
|
# Cumulative blocking detection
|
|
49
|
-
cumulative_blocking_enabled: bool =
|
|
49
|
+
cumulative_blocking_enabled: bool = True
|
|
50
50
|
cumulative_blocking_threshold_ms: float = 200.0
|
|
51
51
|
cumulative_window_ms: float = 1000.0
|
|
52
52
|
|
|
@@ -24,7 +24,10 @@ class RequestContext:
|
|
|
24
24
|
path: The request path (e.g., "/api/users").
|
|
25
25
|
method: HTTP method (GET, POST, etc.).
|
|
26
26
|
start_time: Monotonic timestamp when request started.
|
|
27
|
-
blocking_events: List of (lag_ms, timestamp) tuples
|
|
27
|
+
blocking_events: List of (lag_ms, timestamp) tuples, one per observed
|
|
28
|
+
lag sample that exceeded the threshold.
|
|
29
|
+
cumulative_events: List of (lag_ms, timestamp) tuples where lag_ms is
|
|
30
|
+
a window sum from cumulative detection, not an individual sample.
|
|
28
31
|
"""
|
|
29
32
|
|
|
30
33
|
request_id: str
|
|
@@ -32,20 +35,30 @@ class RequestContext:
|
|
|
32
35
|
method: str
|
|
33
36
|
start_time: float = field(default_factory=time.monotonic)
|
|
34
37
|
blocking_events: list[tuple[float, float]] = field(default_factory=list)
|
|
38
|
+
cumulative_events: list[tuple[float, float]] = field(default_factory=list)
|
|
35
39
|
|
|
36
|
-
def record_blocking(self, lag_ms: float) -> None:
|
|
37
|
-
"""Record a blocking event for this request.
|
|
38
|
-
|
|
40
|
+
def record_blocking(self, lag_ms: float, cumulative: bool = False) -> None:
|
|
41
|
+
"""Record a blocking event for this request.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
lag_ms: The observed lag in milliseconds.
|
|
45
|
+
cumulative: True when lag_ms is a window sum from cumulative
|
|
46
|
+
detection rather than a single observed lag sample.
|
|
47
|
+
"""
|
|
48
|
+
events = self.cumulative_events if cumulative else self.blocking_events
|
|
49
|
+
events.append((lag_ms, time.monotonic()))
|
|
39
50
|
|
|
40
51
|
@property
|
|
41
52
|
def total_blocking_ms(self) -> float:
|
|
42
|
-
"""Sum of all blocking event durations."""
|
|
43
|
-
return sum(lag for lag, _ in self.blocking_events)
|
|
53
|
+
"""Sum of all blocking event durations (individual and cumulative)."""
|
|
54
|
+
return sum(lag for lag, _ in self.blocking_events) + sum(
|
|
55
|
+
lag for lag, _ in self.cumulative_events
|
|
56
|
+
)
|
|
44
57
|
|
|
45
58
|
@property
|
|
46
59
|
def blocking_count(self) -> int:
|
|
47
|
-
"""Number of blocking events detected."""
|
|
48
|
-
return len(self.blocking_events)
|
|
60
|
+
"""Number of blocking events detected (individual and cumulative)."""
|
|
61
|
+
return len(self.blocking_events) + len(self.cumulative_events)
|
|
49
62
|
|
|
50
63
|
|
|
51
64
|
class RequestRegistry:
|