fastapi-perfstats 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastapi_perfstats/__init__.py +73 -0
- fastapi_perfstats/auth.py +60 -0
- fastapi_perfstats/collector.py +159 -0
- fastapi_perfstats/config.py +163 -0
- fastapi_perfstats/ext/__init__.py +9 -0
- fastapi_perfstats/ext/base.py +102 -0
- fastapi_perfstats/ext/recent.py +167 -0
- fastapi_perfstats/ext/sizes.py +109 -0
- fastapi_perfstats/ext/stddev.py +84 -0
- fastapi_perfstats/headers.py +36 -0
- fastapi_perfstats/keys.py +34 -0
- fastapi_perfstats/lifecycle.py +43 -0
- fastapi_perfstats/live.py +133 -0
- fastapi_perfstats/middleware.py +81 -0
- fastapi_perfstats/models.py +93 -0
- fastapi_perfstats/mounts.py +34 -0
- fastapi_perfstats/page.py +299 -0
- fastapi_perfstats/parts.py +9 -0
- fastapi_perfstats/prometheus.py +96 -0
- fastapi_perfstats/py.typed +0 -0
- fastapi_perfstats/reset.py +60 -0
- fastapi_perfstats/route.py +48 -0
- fastapi_perfstats/slow_context.py +118 -0
- fastapi_perfstats/sqlite_schema.py +53 -0
- fastapi_perfstats/stores.py +400 -0
- fastapi_perfstats-0.1.0.dist-info/METADATA +478 -0
- fastapi_perfstats-0.1.0.dist-info/RECORD +29 -0
- fastapi_perfstats-0.1.0.dist-info/WHEEL +4 -0
- fastapi_perfstats-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""``StatsRoute``: an ``APIRoute`` that records each call. Requires fastapi."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Callable, Coroutine
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from fastapi.routing import APIRoute
|
|
10
|
+
from starlette.requests import Request
|
|
11
|
+
from starlette.responses import Response
|
|
12
|
+
|
|
13
|
+
from fastapi_perfstats.collector import StatsCollector, get_default_collector
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class StatsRoute(APIRoute):
|
|
17
|
+
"""Use as ``APIRouter(route_class=StatsRoute)``; records into the default collector.
|
|
18
|
+
|
|
19
|
+
For your own collector use ``collector.route_class`` (a subclass bound to it).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
collector: StatsCollector | None = None # None -> process-wide default, resolved per request
|
|
23
|
+
|
|
24
|
+
def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]:
|
|
25
|
+
original = super().get_route_handler()
|
|
26
|
+
|
|
27
|
+
async def handler(request: Request) -> Response:
|
|
28
|
+
collector = self.collector or get_default_collector()
|
|
29
|
+
if collector.is_excluded(request.url.path):
|
|
30
|
+
return await original(request)
|
|
31
|
+
start = time.perf_counter()
|
|
32
|
+
status = 500
|
|
33
|
+
try:
|
|
34
|
+
response = await original(request)
|
|
35
|
+
status = response.status_code
|
|
36
|
+
return response
|
|
37
|
+
except Exception as exc:
|
|
38
|
+
status = getattr(exc, "status_code", 500)
|
|
39
|
+
raise
|
|
40
|
+
finally:
|
|
41
|
+
collector.finish(request.scope, (time.perf_counter() - start) * 1000, status)
|
|
42
|
+
|
|
43
|
+
return handler
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def make_route_class(collector: StatsCollector) -> type[StatsRoute]:
|
|
47
|
+
"""A ``StatsRoute`` subclass bound to ``collector``."""
|
|
48
|
+
return type("StatsRoute", (StatsRoute,), {"collector": collector})
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Extra detail captured with slow calls: redacted query string, request / trace id and a custom hook.
|
|
2
|
+
|
|
3
|
+
Everything here is opt-in or privacy-safe: the query is only captured when ``slow_capture_query`` is on and
|
|
4
|
+
sensitive parameter values are redacted; only the configured id headers are read; bodies and other headers never are.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import re
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
from typing import Any
|
|
13
|
+
from urllib.parse import parse_qsl, urlencode
|
|
14
|
+
|
|
15
|
+
from fastapi_perfstats.config import StatsConfig
|
|
16
|
+
from fastapi_perfstats.ext.base import Column, NoopExtension
|
|
17
|
+
from fastapi_perfstats.models import SlowCall, SlowExtra
|
|
18
|
+
|
|
19
|
+
log = logging.getLogger("fastapi_perfstats")
|
|
20
|
+
|
|
21
|
+
REDACTED = "[redacted]"
|
|
22
|
+
MAX_QUERY_LENGTH = 500
|
|
23
|
+
MAX_ID_LENGTH = 128
|
|
24
|
+
MAX_CONTEXT_ITEMS = 10
|
|
25
|
+
MAX_CONTEXT_TEXT = 200
|
|
26
|
+
|
|
27
|
+
_TRACE_ID = re.compile(r"[0-9a-fA-F]{32}")
|
|
28
|
+
_logged_hook_errors: set[str] = set()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def redact_query(raw: str, fragments: tuple[str, ...]) -> str | None:
|
|
32
|
+
"""Redact values whose parameter NAME contains any of ``fragments`` (case-insensitive); ``None`` if empty."""
|
|
33
|
+
pairs = parse_qsl(raw, keep_blank_values=True)
|
|
34
|
+
if not pairs:
|
|
35
|
+
return None
|
|
36
|
+
lowered = tuple(f.lower() for f in fragments)
|
|
37
|
+
cleaned = [(k, REDACTED if any(f in k.lower() for f in lowered) else v) for k, v in pairs]
|
|
38
|
+
return urlencode(cleaned)[:MAX_QUERY_LENGTH]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _header_value(scope: Mapping[str, Any], name: str) -> str | None:
|
|
42
|
+
for raw_name, raw_value in scope.get("headers") or ():
|
|
43
|
+
if raw_name.decode("latin-1").lower() == name:
|
|
44
|
+
return str(raw_value.decode("latin-1")).strip()
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _request_id(scope: Mapping[str, Any], names: tuple[str, ...]) -> str | None:
|
|
49
|
+
"""First usable header wins; ``traceparent`` contributes only its 32-hex trace id."""
|
|
50
|
+
for name in names:
|
|
51
|
+
value = _header_value(scope, name)
|
|
52
|
+
if not value:
|
|
53
|
+
continue
|
|
54
|
+
if name == "traceparent":
|
|
55
|
+
parts = value.split("-")
|
|
56
|
+
if len(parts) < 2 or not _TRACE_ID.fullmatch(parts[1]):
|
|
57
|
+
continue
|
|
58
|
+
value = parts[1]
|
|
59
|
+
return value[:MAX_ID_LENGTH]
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _context(scope: Mapping[str, Any], config: StatsConfig) -> dict[str, str] | None:
|
|
64
|
+
hook = config.slow_context
|
|
65
|
+
if hook is None:
|
|
66
|
+
return None
|
|
67
|
+
try:
|
|
68
|
+
result = hook(scope)
|
|
69
|
+
return {
|
|
70
|
+
str(k)[:MAX_CONTEXT_TEXT]: str(v)[:MAX_CONTEXT_TEXT] for k, v in list(result.items())[:MAX_CONTEXT_ITEMS]
|
|
71
|
+
}
|
|
72
|
+
except Exception as exc:
|
|
73
|
+
name = type(exc).__name__
|
|
74
|
+
if name not in _logged_hook_errors:
|
|
75
|
+
_logged_hook_errors.add(name)
|
|
76
|
+
log.warning("slow_context hook raised %s; further errors of this type are not logged", name, exc_info=True)
|
|
77
|
+
return {"context_error": name}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def capture(scope: Mapping[str, Any], config: StatsConfig, key: str, duration_ms: float) -> SlowExtra:
|
|
81
|
+
"""Query / request id / context for a slow call. Never raises (the hook's errors are contained)."""
|
|
82
|
+
query = None
|
|
83
|
+
if config.slow_capture_query:
|
|
84
|
+
raw = scope.get("query_string") or b""
|
|
85
|
+
query = redact_query(raw.decode("utf-8", errors="replace"), config.redact_query_params)
|
|
86
|
+
return SlowExtra(
|
|
87
|
+
query=query,
|
|
88
|
+
request_id=_request_id(scope, config.slow_request_id_headers),
|
|
89
|
+
context=_context(scope, config),
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _text_cell(attr: str) -> Any:
|
|
94
|
+
def cell(call: SlowCall, config: StatsConfig) -> tuple[str, float | None]:
|
|
95
|
+
return getattr(call, attr) or "", None
|
|
96
|
+
|
|
97
|
+
return cell
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _context_cell(call: SlowCall, config: StatsConfig) -> tuple[str, float | None]:
|
|
101
|
+
return "; ".join(f"{k}={v}" for k, v in call.context.items()), None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class SlowContextExtension(NoopExtension):
|
|
105
|
+
"""Only contributes the slow-table columns; persistence of the extras is done by the stores."""
|
|
106
|
+
|
|
107
|
+
def __init__(self) -> None:
|
|
108
|
+
super().__init__("slow_context")
|
|
109
|
+
|
|
110
|
+
def columns(self, config: StatsConfig) -> tuple[Column, ...]:
|
|
111
|
+
return (
|
|
112
|
+
Column("slow", "Request id", 60, _text_cell("request_id")),
|
|
113
|
+
Column("slow", "Query", 70, _text_cell("query")),
|
|
114
|
+
Column("slow", "Context", 80, _context_cell),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
EXT = SlowContextExtension()
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""SQLite schema and its version (owned by #2)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sqlite3
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
SCHEMA_VERSION = 1
|
|
9
|
+
|
|
10
|
+
_STATEMENTS = (
|
|
11
|
+
"CREATE TABLE routes ("
|
|
12
|
+
"key TEXT PRIMARY KEY, count INTEGER NOT NULL, e4 INTEGER NOT NULL, e5 INTEGER NOT NULL, "
|
|
13
|
+
"total_ms REAL NOT NULL, min_ms REAL NOT NULL, max_ms REAL NOT NULL, "
|
|
14
|
+
"first_ts REAL NOT NULL, last_ts REAL NOT NULL)",
|
|
15
|
+
"CREATE TABLE durations (id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT NOT NULL, ms REAL NOT NULL)",
|
|
16
|
+
"CREATE INDEX durations_key ON durations (key, id)",
|
|
17
|
+
"CREATE TABLE slow_calls ("
|
|
18
|
+
"id INTEGER PRIMARY KEY AUTOINCREMENT, ts REAL NOT NULL, key TEXT NOT NULL, "
|
|
19
|
+
"ms REAL NOT NULL, status INTEGER NOT NULL)",
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
_BUSY_TIMEOUT_S = 30.0
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def ensure_schema(path: str | Path) -> None:
|
|
26
|
+
"""Create the core tables in a new/empty file and stamp it; accept ``SCHEMA_VERSION``; refuse anything else.
|
|
27
|
+
|
|
28
|
+
Uses its own short-lived connection. ``BEGIN IMMEDIATE`` serialises concurrent workers. Raises ``RuntimeError``
|
|
29
|
+
for a file with another version or with tables but no version stamp; never migrates or deletes anything.
|
|
30
|
+
"""
|
|
31
|
+
db = sqlite3.connect(path, timeout=_BUSY_TIMEOUT_S, isolation_level=None)
|
|
32
|
+
try:
|
|
33
|
+
db.execute("BEGIN IMMEDIATE")
|
|
34
|
+
try:
|
|
35
|
+
found = int(db.execute("PRAGMA user_version").fetchone()[0])
|
|
36
|
+
has_tables = db.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' LIMIT 1").fetchone() is not None
|
|
37
|
+
if found == SCHEMA_VERSION:
|
|
38
|
+
pass
|
|
39
|
+
elif found == 0 and not has_tables:
|
|
40
|
+
for statement in _STATEMENTS:
|
|
41
|
+
db.execute(statement)
|
|
42
|
+
db.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
|
|
43
|
+
else:
|
|
44
|
+
raise RuntimeError(
|
|
45
|
+
f"stats database {str(path)!r} has schema version {found}, expected {SCHEMA_VERSION}; "
|
|
46
|
+
"delete the file or point sqlite_path elsewhere"
|
|
47
|
+
)
|
|
48
|
+
except BaseException:
|
|
49
|
+
db.execute("ROLLBACK")
|
|
50
|
+
raise
|
|
51
|
+
db.execute("COMMIT")
|
|
52
|
+
finally:
|
|
53
|
+
db.close()
|
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
"""Storage backends: in-memory (default) and SQLite (stdlib), behind one ABC."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import queue
|
|
8
|
+
import sqlite3
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from collections import deque
|
|
13
|
+
from collections.abc import Callable, Sequence
|
|
14
|
+
from dataclasses import dataclass, field, replace
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from fastapi_perfstats import sqlite_schema
|
|
18
|
+
from fastapi_perfstats.config import StatsConfig, quantile_label
|
|
19
|
+
from fastapi_perfstats.ext.base import Extension
|
|
20
|
+
from fastapi_perfstats.models import OTHER_KEY, CallRecord, RouteStats, SlowCall, nearest_rank
|
|
21
|
+
|
|
22
|
+
log = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _resolve_extensions(extensions: Sequence[Extension] | None) -> tuple[Extension, ...]:
|
|
26
|
+
"""``None`` means the registered extensions (imported lazily: the registry imports modules that use stores)."""
|
|
27
|
+
if extensions is None:
|
|
28
|
+
from fastapi_perfstats.ext import EXTENSIONS
|
|
29
|
+
|
|
30
|
+
return EXTENSIONS
|
|
31
|
+
return tuple(extensions)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class StatsStore(ABC):
|
|
35
|
+
"""Minimal contract. ``record`` must be cheap and thread-safe; reads may be slower."""
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
def record(self, call: CallRecord) -> None: ...
|
|
39
|
+
|
|
40
|
+
@abstractmethod
|
|
41
|
+
def routes(self) -> list[RouteStats]: ...
|
|
42
|
+
|
|
43
|
+
@abstractmethod
|
|
44
|
+
def slow_calls(self) -> list[SlowCall]: ...
|
|
45
|
+
|
|
46
|
+
def reset(self) -> None:
|
|
47
|
+
"""Forget all statistics. Optional; stores that support it override this."""
|
|
48
|
+
raise NotImplementedError(f"{type(self).__name__} does not support reset()")
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def dropped(self) -> int:
|
|
52
|
+
"""Records dropped because the store could not keep up (0 for stores that never drop)."""
|
|
53
|
+
return 0
|
|
54
|
+
|
|
55
|
+
def flush(self) -> None: # noqa: B027 - optional hook
|
|
56
|
+
"""Block until everything recorded so far is visible to readers."""
|
|
57
|
+
|
|
58
|
+
def close(self) -> None: # noqa: B027 - optional hook
|
|
59
|
+
"""Release resources."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _finish(rows: list[RouteStats], now: float) -> list[RouteStats]:
|
|
63
|
+
total = sum(r.count for r in rows) or 1
|
|
64
|
+
for r in rows:
|
|
65
|
+
r.share = r.count / total
|
|
66
|
+
r.rate_per_min = r.count / max(now - r.first_ts, 1.0) * 60
|
|
67
|
+
return sorted(rows, key=lambda r: r.count, reverse=True)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class _Agg:
|
|
72
|
+
window: deque[float]
|
|
73
|
+
count: int = 0
|
|
74
|
+
e4: int = 0
|
|
75
|
+
e5: int = 0
|
|
76
|
+
total: float = 0.0
|
|
77
|
+
lo: float = float("inf")
|
|
78
|
+
hi: float = 0.0
|
|
79
|
+
first: float = 0.0
|
|
80
|
+
last: float = 0.0
|
|
81
|
+
ext: dict[str, dict[str, Any]] = field(default_factory=dict) # per-extension state, keyed by extension name
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _make_stats(cfg: StatsConfig, key: str, agg: _Agg) -> RouteStats:
|
|
85
|
+
window = sorted(agg.window)
|
|
86
|
+
return RouteStats(
|
|
87
|
+
key=key,
|
|
88
|
+
count=agg.count,
|
|
89
|
+
errors_4xx=agg.e4,
|
|
90
|
+
errors_5xx=agg.e5,
|
|
91
|
+
min_ms=agg.lo,
|
|
92
|
+
mean_ms=agg.total / agg.count if agg.count else 0.0,
|
|
93
|
+
max_ms=agg.hi,
|
|
94
|
+
quantiles={quantile_label(q): nearest_rank(window, q) for q in cfg.quantiles},
|
|
95
|
+
first_ts=agg.first,
|
|
96
|
+
last_ts=agg.last,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class MemoryStore(StatsStore):
|
|
101
|
+
"""Bounded in-memory store: ring buffer of ``window_size`` durations per route, ``max_routes`` routes."""
|
|
102
|
+
|
|
103
|
+
def __init__(self, config: StatsConfig, extensions: Sequence[Extension] | None = None) -> None:
|
|
104
|
+
self.cfg = config
|
|
105
|
+
self._exts = _resolve_extensions(extensions)
|
|
106
|
+
self._lock = threading.Lock()
|
|
107
|
+
self._aggs: dict[str, _Agg] = {}
|
|
108
|
+
self._slow: deque[SlowCall] = deque(maxlen=config.max_slow_calls)
|
|
109
|
+
|
|
110
|
+
def record(self, call: CallRecord) -> None:
|
|
111
|
+
with self._lock:
|
|
112
|
+
if call.key not in self._aggs and len(self._aggs) >= self.cfg.max_routes and call.key != OTHER_KEY:
|
|
113
|
+
call = replace(call, key=OTHER_KEY)
|
|
114
|
+
key, duration_ms, status, ts = call.key, call.duration_ms, call.status, call.ts
|
|
115
|
+
agg = self._aggs.get(key)
|
|
116
|
+
if agg is None:
|
|
117
|
+
agg = self._aggs[key] = _Agg(
|
|
118
|
+
window=deque(maxlen=self.cfg.window_size), first=ts, ext={e.name: {} for e in self._exts}
|
|
119
|
+
)
|
|
120
|
+
agg.count += 1
|
|
121
|
+
agg.e5 += status >= 500
|
|
122
|
+
agg.e4 += 400 <= status < 500
|
|
123
|
+
agg.total += duration_ms
|
|
124
|
+
agg.lo = min(agg.lo, duration_ms)
|
|
125
|
+
agg.hi = max(agg.hi, duration_ms)
|
|
126
|
+
agg.last = ts
|
|
127
|
+
agg.window.append(duration_ms)
|
|
128
|
+
for ext in self._exts:
|
|
129
|
+
ext.memory_update(agg.ext[ext.name], call)
|
|
130
|
+
thr = self.cfg.slow_threshold_for(key)
|
|
131
|
+
if thr is not None and duration_ms >= thr:
|
|
132
|
+
self._slow.append(
|
|
133
|
+
SlowCall(
|
|
134
|
+
ts,
|
|
135
|
+
key,
|
|
136
|
+
duration_ms,
|
|
137
|
+
status,
|
|
138
|
+
query=call.query,
|
|
139
|
+
request_id=call.request_id,
|
|
140
|
+
context=dict(call.context or {}),
|
|
141
|
+
)
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
def routes(self) -> list[RouteStats]:
|
|
145
|
+
now = time.time()
|
|
146
|
+
with self._lock:
|
|
147
|
+
rows = []
|
|
148
|
+
for k, a in self._aggs.items():
|
|
149
|
+
row = _make_stats(self.cfg, k, a)
|
|
150
|
+
for ext in self._exts:
|
|
151
|
+
ext.memory_finish(row, a.ext[ext.name], self.cfg, now)
|
|
152
|
+
rows.append(row)
|
|
153
|
+
return _finish(rows, now)
|
|
154
|
+
|
|
155
|
+
def slow_calls(self) -> list[SlowCall]:
|
|
156
|
+
with self._lock:
|
|
157
|
+
return list(reversed(self._slow)) # newest first
|
|
158
|
+
|
|
159
|
+
def reset(self) -> None:
|
|
160
|
+
with self._lock:
|
|
161
|
+
self._aggs.clear()
|
|
162
|
+
self._slow.clear()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
_STOP = object()
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class _Reset:
|
|
169
|
+
"""Queue sentinel: the writer clears everything when it reaches it, in order with the records."""
|
|
170
|
+
|
|
171
|
+
def __init__(self) -> None:
|
|
172
|
+
self.done = threading.Event()
|
|
173
|
+
self.error: Exception | None = None
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
_UPSERT = """
|
|
177
|
+
INSERT INTO routes (key, count, e4, e5, total_ms, min_ms, max_ms, first_ts, last_ts)
|
|
178
|
+
VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?)
|
|
179
|
+
ON CONFLICT(key) DO UPDATE SET count = count + 1, e4 = e4 + excluded.e4, e5 = e5 + excluded.e5,
|
|
180
|
+
total_ms = total_ms + excluded.total_ms, min_ms = MIN(min_ms, excluded.min_ms),
|
|
181
|
+
max_ms = MAX(max_ms, excluded.max_ms), last_ts = excluded.last_ts
|
|
182
|
+
"""
|
|
183
|
+
_PRUNE = (
|
|
184
|
+
"DELETE FROM durations WHERE key = ? AND id NOT IN "
|
|
185
|
+
"(SELECT id FROM durations WHERE key = ? ORDER BY id DESC LIMIT ?)"
|
|
186
|
+
)
|
|
187
|
+
_SLOW_EXTRA_SCHEMA = (
|
|
188
|
+
"CREATE TABLE IF NOT EXISTS slow_call_extra "
|
|
189
|
+
"(slow_id INTEGER PRIMARY KEY, query TEXT, request_id TEXT, context TEXT)"
|
|
190
|
+
)
|
|
191
|
+
_RESET_TABLES = ("routes", "durations", "slow_calls", "slow_call_extra")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class SqliteStore(StatsStore):
|
|
195
|
+
"""Persistent store. ``record`` only enqueues; one writer thread applies batches in one transaction.
|
|
196
|
+
|
|
197
|
+
The queue is bounded (``QUEUE_SIZE``); if the writer cannot keep up, new records are dropped
|
|
198
|
+
(counted in ``dropped``) instead of blocking the event loop or growing memory without limit.
|
|
199
|
+
Size limits: ``window_size`` durations per route, ``max_routes`` routes (rest -> "other"),
|
|
200
|
+
``max_slow_calls`` slow-call rows. Reads call ``flush()`` first, so they see all prior records.
|
|
201
|
+
A batch whose SQL or extension hook fails is rolled back and logged; the writer keeps running.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
QUEUE_SIZE = 10_000
|
|
205
|
+
BATCH = 500
|
|
206
|
+
|
|
207
|
+
def __init__(self, config: StatsConfig, extensions: Sequence[Extension] | None = None) -> None:
|
|
208
|
+
self.cfg = config
|
|
209
|
+
self._exts = _resolve_extensions(extensions)
|
|
210
|
+
self._dropped = 0
|
|
211
|
+
self._dropped_lock = threading.Lock() # separate from ``_lock``: ``record`` must never wait for the writer
|
|
212
|
+
self._closed = False
|
|
213
|
+
self._close_lock = threading.Lock()
|
|
214
|
+
self._q: queue.Queue[object] = queue.Queue(self.QUEUE_SIZE)
|
|
215
|
+
self._lock = threading.Lock()
|
|
216
|
+
sqlite_schema.ensure_schema(config.sqlite_path)
|
|
217
|
+
# timeout + WAL: several processes (workers) may share one file; per-process stats are not merged in memory,
|
|
218
|
+
# but the SQL upserts are atomic so counts add up across processes.
|
|
219
|
+
self._db = sqlite3.connect(config.sqlite_path, check_same_thread=False, timeout=10)
|
|
220
|
+
self._db.execute("PRAGMA journal_mode=WAL")
|
|
221
|
+
self._db.execute(_SLOW_EXTRA_SCHEMA)
|
|
222
|
+
for ext in self._exts:
|
|
223
|
+
for statement in ext.sqlite_schema():
|
|
224
|
+
self._db.execute(statement)
|
|
225
|
+
self._db.commit()
|
|
226
|
+
self._known = {r[0] for r in self._db.execute("SELECT key FROM routes")}
|
|
227
|
+
self._thread = threading.Thread(target=self._run, name="fastapi-perfstats-writer", daemon=True)
|
|
228
|
+
self._thread.start()
|
|
229
|
+
|
|
230
|
+
@property
|
|
231
|
+
def dropped(self) -> int:
|
|
232
|
+
with self._dropped_lock:
|
|
233
|
+
return self._dropped
|
|
234
|
+
|
|
235
|
+
def record(self, call: CallRecord) -> None:
|
|
236
|
+
try:
|
|
237
|
+
self._q.put_nowait(call)
|
|
238
|
+
except queue.Full:
|
|
239
|
+
with self._dropped_lock:
|
|
240
|
+
self._dropped += 1
|
|
241
|
+
|
|
242
|
+
def flush(self) -> None:
|
|
243
|
+
if not self._thread.is_alive():
|
|
244
|
+
return
|
|
245
|
+
done = threading.Event()
|
|
246
|
+
self._q.put(done)
|
|
247
|
+
done.wait(timeout=10)
|
|
248
|
+
|
|
249
|
+
def reset(self) -> None:
|
|
250
|
+
"""Delete all statistics. Records enqueued before this call are discarded, later ones are kept."""
|
|
251
|
+
if self._closed:
|
|
252
|
+
raise RuntimeError("cannot reset(): SqliteStore is closed")
|
|
253
|
+
marker = _Reset()
|
|
254
|
+
self._q.put(marker)
|
|
255
|
+
marker.done.wait(timeout=10)
|
|
256
|
+
if marker.error is not None:
|
|
257
|
+
raise marker.error
|
|
258
|
+
|
|
259
|
+
def close(self) -> None:
|
|
260
|
+
with self._close_lock:
|
|
261
|
+
if self._closed:
|
|
262
|
+
return
|
|
263
|
+
self._closed = True
|
|
264
|
+
if self._thread.is_alive():
|
|
265
|
+
self._q.put(_STOP)
|
|
266
|
+
self._thread.join(timeout=10)
|
|
267
|
+
with self._lock:
|
|
268
|
+
self._db.close()
|
|
269
|
+
|
|
270
|
+
def _run(self) -> None:
|
|
271
|
+
stop = False
|
|
272
|
+
while not stop:
|
|
273
|
+
batch = [self._q.get()]
|
|
274
|
+
try:
|
|
275
|
+
while len(batch) < self.BATCH:
|
|
276
|
+
batch.append(self._q.get_nowait())
|
|
277
|
+
except queue.Empty:
|
|
278
|
+
pass
|
|
279
|
+
with self._lock:
|
|
280
|
+
pending: list[CallRecord] = []
|
|
281
|
+
for item in batch:
|
|
282
|
+
if isinstance(item, CallRecord):
|
|
283
|
+
pending.append(item)
|
|
284
|
+
elif isinstance(item, _Reset):
|
|
285
|
+
self._guarded(self._apply, pending)
|
|
286
|
+
pending = []
|
|
287
|
+
item.error = self._guarded(self._reset_db)
|
|
288
|
+
self._guarded(self._apply, pending)
|
|
289
|
+
for b in batch:
|
|
290
|
+
if isinstance(b, threading.Event):
|
|
291
|
+
b.set()
|
|
292
|
+
elif isinstance(b, _Reset):
|
|
293
|
+
b.done.set()
|
|
294
|
+
elif b is _STOP:
|
|
295
|
+
stop = True
|
|
296
|
+
|
|
297
|
+
def _guarded(self, fn: Callable[..., None], *args: Any) -> Exception | None:
|
|
298
|
+
"""Run a unit of writer work; on any failure roll back, restore ``_known``, log and return the error."""
|
|
299
|
+
known = set(self._known)
|
|
300
|
+
try:
|
|
301
|
+
fn(*args)
|
|
302
|
+
except Exception as exc:
|
|
303
|
+
self._db.rollback()
|
|
304
|
+
self._known = known
|
|
305
|
+
log.exception("fastapi-perfstats: store batch rolled back")
|
|
306
|
+
return exc
|
|
307
|
+
return None
|
|
308
|
+
|
|
309
|
+
def _reset_db(self) -> None:
|
|
310
|
+
for table in _RESET_TABLES:
|
|
311
|
+
self._db.execute(f"DELETE FROM {table}") # noqa: S608 - fixed table names
|
|
312
|
+
for ext in self._exts:
|
|
313
|
+
ext.sqlite_reset(self._db)
|
|
314
|
+
self._db.commit()
|
|
315
|
+
self._known.clear()
|
|
316
|
+
|
|
317
|
+
def _apply(self, records: list[CallRecord]) -> None:
|
|
318
|
+
if not records:
|
|
319
|
+
return
|
|
320
|
+
touched: set[str] = set()
|
|
321
|
+
for call in records:
|
|
322
|
+
if call.key not in self._known and len(self._known) >= self.cfg.max_routes:
|
|
323
|
+
call = replace(call, key=OTHER_KEY)
|
|
324
|
+
key, ms, status, ts = call.key, call.duration_ms, call.status, call.ts
|
|
325
|
+
self._known.add(key)
|
|
326
|
+
touched.add(key)
|
|
327
|
+
self._db.execute(_UPSERT, (key, 400 <= status < 500, status >= 500, ms, ms, ms, ts, ts))
|
|
328
|
+
self._db.execute("INSERT INTO durations (key, ms) VALUES (?, ?)", (key, ms))
|
|
329
|
+
thr = self.cfg.slow_threshold_for(key)
|
|
330
|
+
if thr is not None and ms >= thr:
|
|
331
|
+
cur = self._db.execute(
|
|
332
|
+
"INSERT INTO slow_calls (ts, key, ms, status) VALUES (?, ?, ?, ?)", (ts, key, ms, status)
|
|
333
|
+
)
|
|
334
|
+
context = json.dumps(dict(call.context)) if call.context else None
|
|
335
|
+
self._db.execute(
|
|
336
|
+
"INSERT INTO slow_call_extra (slow_id, query, request_id, context) VALUES (?, ?, ?, ?)",
|
|
337
|
+
(cur.lastrowid, call.query, call.request_id, context),
|
|
338
|
+
)
|
|
339
|
+
for ext in self._exts:
|
|
340
|
+
ext.sqlite_update(self._db, call)
|
|
341
|
+
for key in touched:
|
|
342
|
+
self._db.execute(_PRUNE, (key, key, self.cfg.window_size))
|
|
343
|
+
self._db.execute(
|
|
344
|
+
"DELETE FROM slow_calls WHERE id <= (SELECT MAX(id) FROM slow_calls) - ?", (self.cfg.max_slow_calls,)
|
|
345
|
+
)
|
|
346
|
+
self._db.execute("DELETE FROM slow_call_extra WHERE slow_id NOT IN (SELECT id FROM slow_calls)")
|
|
347
|
+
now = time.time()
|
|
348
|
+
for ext in self._exts:
|
|
349
|
+
ext.sqlite_prune(self._db, self.cfg, now)
|
|
350
|
+
self._db.commit()
|
|
351
|
+
|
|
352
|
+
def routes(self) -> list[RouteStats]:
|
|
353
|
+
self.flush()
|
|
354
|
+
now = time.time()
|
|
355
|
+
with self._lock:
|
|
356
|
+
windows: dict[str, list[float]] = {}
|
|
357
|
+
for key, ms in self._db.execute("SELECT key, ms FROM durations ORDER BY id"):
|
|
358
|
+
windows.setdefault(key, []).append(ms)
|
|
359
|
+
rows = []
|
|
360
|
+
for k, c, e4, e5, tot, lo, hi, first, last in self._db.execute(
|
|
361
|
+
"SELECT key, count, e4, e5, total_ms, min_ms, max_ms, first_ts, last_ts FROM routes"
|
|
362
|
+
):
|
|
363
|
+
agg = _Agg(deque(windows.get(k, [])), c, e4, e5, tot, lo, hi, first, last)
|
|
364
|
+
rows.append(_make_stats(self.cfg, k, agg))
|
|
365
|
+
finished = _finish(rows, now)
|
|
366
|
+
by_key = {r.key: r for r in finished}
|
|
367
|
+
for ext in self._exts:
|
|
368
|
+
ext.sqlite_finish(by_key, self._db, self.cfg, now)
|
|
369
|
+
return finished
|
|
370
|
+
|
|
371
|
+
def slow_calls(self) -> list[SlowCall]:
|
|
372
|
+
self.flush()
|
|
373
|
+
with self._lock:
|
|
374
|
+
cur = self._db.execute(
|
|
375
|
+
"SELECT s.ts, s.key, s.ms, s.status, e.query, e.request_id, e.context FROM slow_calls s "
|
|
376
|
+
"LEFT JOIN slow_call_extra e ON e.slow_id = s.id ORDER BY s.id DESC"
|
|
377
|
+
)
|
|
378
|
+
return [
|
|
379
|
+
SlowCall(ts, key, ms, status, query, rid, json.loads(ctx) if ctx else {})
|
|
380
|
+
for ts, key, ms, status, query, rid, ctx in cur
|
|
381
|
+
]
|
|
382
|
+
|
|
383
|
+
def row_counts(self) -> dict[str, int]:
|
|
384
|
+
"""Row counts per table (used to verify the size limits)."""
|
|
385
|
+
self.flush()
|
|
386
|
+
out = {}
|
|
387
|
+
with self._lock:
|
|
388
|
+
for table in ("routes", "durations", "slow_calls", "slow_call_extra"):
|
|
389
|
+
out[table] = self._db.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608
|
|
390
|
+
return out
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def build_store(config: StatsConfig) -> StatsStore:
|
|
394
|
+
if isinstance(config.storage, StatsStore):
|
|
395
|
+
return config.storage
|
|
396
|
+
if config.storage == "memory":
|
|
397
|
+
return MemoryStore(config)
|
|
398
|
+
if config.storage == "sqlite":
|
|
399
|
+
return SqliteStore(config)
|
|
400
|
+
raise ValueError(f"unknown storage {config.storage!r}")
|