devbench 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {devbench-0.1.0 → devbench-0.2.0}/PKG-INFO +34 -1
- {devbench-0.1.0 → devbench-0.2.0}/README.md +33 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/__init__.py +4 -2
- {devbench-0.1.0 → devbench-0.2.0}/devbench/_core.py +2 -0
- devbench-0.2.0/devbench/_requests.py +145 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/asgi.py +40 -2
- {devbench-0.1.0 → devbench-0.2.0}/devbench/config.py +1 -1
- {devbench-0.1.0 → devbench-0.2.0}/devbench/context.py +19 -2
- {devbench-0.1.0 → devbench-0.2.0}/devbench/direct.py +64 -4
- {devbench-0.1.0 → devbench-0.2.0}/devbench/django/middleware.py +25 -5
- {devbench-0.1.0 → devbench-0.2.0}/devbench/flask.py +17 -3
- {devbench-0.1.0 → devbench-0.2.0}/devbench/identity.py +1 -1
- {devbench-0.1.0 → devbench-0.2.0}/devbench/logs.py +1 -1
- {devbench-0.1.0 → devbench-0.2.0}/devbench/reporter.py +39 -1
- devbench-0.2.0/devbench/reqsignal.py +241 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/trace.py +1 -1
- {devbench-0.1.0 → devbench-0.2.0}/devbench/version.py +8 -2
- devbench-0.2.0/devbench/wsgi.py +126 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench.egg-info/PKG-INFO +34 -1
- {devbench-0.1.0 → devbench-0.2.0}/devbench.egg-info/SOURCES.txt +4 -0
- {devbench-0.1.0 → devbench-0.2.0}/tests/test_django.py +50 -0
- devbench-0.2.0/tests/test_fastapi.py +274 -0
- {devbench-0.1.0 → devbench-0.2.0}/tests/test_flask.py +83 -0
- devbench-0.2.0/tests/test_requests.py +348 -0
- devbench-0.2.0/tests/test_sdk_field.py +100 -0
- {devbench-0.1.0 → devbench-0.2.0}/tests/test_vectors.py +82 -1
- devbench-0.1.0/devbench/wsgi.py +0 -67
- devbench-0.1.0/tests/test_fastapi.py +0 -119
- {devbench-0.1.0 → devbench-0.2.0}/LICENSE +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/__main__.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/_integration.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/celery.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/django/__init__.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/django/apps.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/django/management/__init__.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/django/management/commands/__init__.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/django/management/commands/devbench_test.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/egress_policy.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/fastapi.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/fingerprint.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/py.typed +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench/scrub.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench.egg-info/dependency_links.txt +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/devbench.egg-info/top_level.txt +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/pyproject.toml +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/setup.cfg +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/tests/test_celery.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/tests/test_config.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/tests/test_direct.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/tests/test_logs.py +0 -0
- {devbench-0.1.0 → devbench-0.2.0}/tests/test_reporter.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: devbench
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Dev Bench server SDK for Python: errors, who was affected, and the server log lines of a failing request, for Django, FastAPI, Flask and Celery.
|
|
5
5
|
Author: Dev Bench
|
|
6
6
|
License-Expression: MIT
|
|
@@ -153,6 +153,39 @@ the user.
|
|
|
153
153
|
reporting is dropped, and the application's own exception always comes
|
|
154
154
|
out unchanged.
|
|
155
155
|
|
|
156
|
+
### Failed, slow and rate-limited requests
|
|
157
|
+
|
|
158
|
+
The same middleware also watches what each request does *without* raising
|
|
159
|
+
(no code needed; Django, Flask, FastAPI/Starlette and bare WSGI/ASGI):
|
|
160
|
+
|
|
161
|
+
| Signal | When | Reported as |
|
|
162
|
+
|---|---|---|
|
|
163
|
+
| Failed | the app answered 500–599 and nothing was reported as an exception during the request | kind `error`, type `http_<status>` |
|
|
164
|
+
| Slow | the app took **more than 3,000 ms** and answered 100–499 | kind `degradation`, type `slow_request` |
|
|
165
|
+
| 429 spike | in one unix minute, a route answered at least ten 429s and at least 10% of its requests were 429s | kind `error`, type `http_429_spike` |
|
|
166
|
+
|
|
167
|
+
- **Exceptions win.** An unhandled error, or a `capture_exception` during
|
|
168
|
+
the request, is the report on its own: its 500 is not also failed. A
|
|
169
|
+
`report_handled` changes nothing: a 5xx after it is still failed, and the
|
|
170
|
+
`x-adt-handled` header is unchanged. Each status is its own problem
|
|
171
|
+
(`http_500`, `http_503`). Raising an `HTTPException` with a 5xx status is
|
|
172
|
+
a response, so it is failed.
|
|
173
|
+
- **Slow** is measured on a monotonic clock from the middleware's entry
|
|
174
|
+
until the app hands back its response (the WSGI callable returns, the
|
|
175
|
+
ASGI app's coroutine completes, Django's `get_response` returns).
|
|
176
|
+
Exactly 3,000 ms is not slow. Event streams (`text/event-stream`) are
|
|
177
|
+
never slow; a streaming 5xx is still failed. The threshold is a
|
|
178
|
+
constant, not a setting, for now.
|
|
179
|
+
- **429 spikes** are counted per process and reported once per route per
|
|
180
|
+
minute, when the minute closes (the next request in a later minute, the
|
|
181
|
+
next flush, or process exit), with no trace or user.
|
|
182
|
+
- Other 4xx are not reported. Each signal is identified by the method and
|
|
183
|
+
the route **pattern**, never the raw path: Django's
|
|
184
|
+
`resolver_match.route` (`orders/<int:id>`), Flask's URL rule
|
|
185
|
+
(`/orders/<int:id>`), the FastAPI/Starlette route path
|
|
186
|
+
(`/orders/{id}`), or `*` when no route matched (and for a bare WSGI/ASGI
|
|
187
|
+
app). A failed or slow request carries its trace and user.
|
|
188
|
+
|
|
156
189
|
## What is sent, and when
|
|
157
190
|
|
|
158
191
|
- Each report is **fingerprinted and counted in the process** (class,
|
|
@@ -134,6 +134,39 @@ the user.
|
|
|
134
134
|
reporting is dropped, and the application's own exception always comes
|
|
135
135
|
out unchanged.
|
|
136
136
|
|
|
137
|
+
### Failed, slow and rate-limited requests
|
|
138
|
+
|
|
139
|
+
The same middleware also watches what each request does *without* raising
|
|
140
|
+
(no code needed; Django, Flask, FastAPI/Starlette and bare WSGI/ASGI):
|
|
141
|
+
|
|
142
|
+
| Signal | When | Reported as |
|
|
143
|
+
|---|---|---|
|
|
144
|
+
| Failed | the app answered 500–599 and nothing was reported as an exception during the request | kind `error`, type `http_<status>` |
|
|
145
|
+
| Slow | the app took **more than 3,000 ms** and answered 100–499 | kind `degradation`, type `slow_request` |
|
|
146
|
+
| 429 spike | in one unix minute, a route answered at least ten 429s and at least 10% of its requests were 429s | kind `error`, type `http_429_spike` |
|
|
147
|
+
|
|
148
|
+
- **Exceptions win.** An unhandled error, or a `capture_exception` during
|
|
149
|
+
the request, is the report on its own: its 500 is not also failed. A
|
|
150
|
+
`report_handled` changes nothing: a 5xx after it is still failed, and the
|
|
151
|
+
`x-adt-handled` header is unchanged. Each status is its own problem
|
|
152
|
+
(`http_500`, `http_503`). Raising an `HTTPException` with a 5xx status is
|
|
153
|
+
a response, so it is failed.
|
|
154
|
+
- **Slow** is measured on a monotonic clock from the middleware's entry
|
|
155
|
+
until the app hands back its response (the WSGI callable returns, the
|
|
156
|
+
ASGI app's coroutine completes, Django's `get_response` returns).
|
|
157
|
+
Exactly 3,000 ms is not slow. Event streams (`text/event-stream`) are
|
|
158
|
+
never slow; a streaming 5xx is still failed. The threshold is a
|
|
159
|
+
constant, not a setting, for now.
|
|
160
|
+
- **429 spikes** are counted per process and reported once per route per
|
|
161
|
+
minute, when the minute closes (the next request in a later minute, the
|
|
162
|
+
next flush, or process exit), with no trace or user.
|
|
163
|
+
- Other 4xx are not reported. Each signal is identified by the method and
|
|
164
|
+
the route **pattern**, never the raw path: Django's
|
|
165
|
+
`resolver_match.route` (`orders/<int:id>`), Flask's URL rule
|
|
166
|
+
(`/orders/<int:id>`), the FastAPI/Starlette route path
|
|
167
|
+
(`/orders/{id}`), or `*` when no route matched (and for a bare WSGI/ASGI
|
|
168
|
+
app). A failed or slow request carries its trace and user.
|
|
169
|
+
|
|
137
170
|
## What is sent, and when
|
|
138
171
|
|
|
139
172
|
- Each report is **fingerprinted and counted in the process** (class,
|
|
@@ -13,6 +13,8 @@ What it does:
|
|
|
13
13
|
* accepts and forwards the correlation id (the middlewares, trace_headers)
|
|
14
14
|
* reports unhandled exceptions (the middlewares, Celery hooks,
|
|
15
15
|
capture_exception)
|
|
16
|
+
* reports 5xx answered without an exception,
|
|
17
|
+
requests over 3 s and 429 spikes (the middlewares; reqsignal)
|
|
16
18
|
* reports handled failures (report_handled)
|
|
17
19
|
* records who was affected (set_user)
|
|
18
20
|
* keeps recent log lines per trace (capture_logs; a logging handler)
|
|
@@ -29,12 +31,12 @@ from .config import DSN, Configuration, InvalidDSN
|
|
|
29
31
|
from .identity import set_user
|
|
30
32
|
from .trace import HEADER as TRACE_HEADER
|
|
31
33
|
from .trace import Trace
|
|
32
|
-
from .version import PROTOCOL_VERSION, VERSION
|
|
34
|
+
from .version import PROTOCOL_VERSION, SDK_NAME, VERSION
|
|
33
35
|
|
|
34
36
|
__version__ = VERSION
|
|
35
37
|
|
|
36
38
|
__all__ = [
|
|
37
|
-
"PROTOCOL_VERSION", "VERSION", "DSN", "Configuration", "InvalidDSN", "Trace", "TRACE_HEADER",
|
|
39
|
+
"PROTOCOL_VERSION", "VERSION", "SDK_NAME", "DSN", "Configuration", "InvalidDSN", "Trace", "TRACE_HEADER",
|
|
38
40
|
"configure", "enabled", "set_user", "capture_exception", "report_handled", "ignore_exceptions",
|
|
39
41
|
"trace_headers", "current_trace", "capture_logs", "flush", "test",
|
|
40
42
|
]
|
|
@@ -247,6 +247,8 @@ def _exit_flush() -> None:
|
|
|
247
247
|
current = _transport
|
|
248
248
|
if isinstance(current, DirectTransport):
|
|
249
249
|
try:
|
|
250
|
+
# The open 429 spike window is evaluated as it stands at exit.
|
|
251
|
+
current.end_requests()
|
|
250
252
|
current.flush(timeout=EXIT_TIMEOUT)
|
|
251
253
|
except Exception:
|
|
252
254
|
pass
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""How the request middlewares observe Capability 2b (reqsignal has the
|
|
2
|
+
rules).
|
|
3
|
+
|
|
4
|
+
Once per request, after the application has handed back its response
|
|
5
|
+
without raising: classify it (failed / slow / nothing), report the signal
|
|
6
|
+
like any other report, and feed the process's 429 spike detector. On the
|
|
7
|
+
request's thread, none of it blocking: a classification, a count under a
|
|
8
|
+
lock, one mutex-guarded table update.
|
|
9
|
+
|
|
10
|
+
A request that raised out of the middleware is not evaluated: the exception
|
|
11
|
+
is its report. Only the outermost Dev Bench middleware observes (a nested one
|
|
12
|
+
passes straight through), so a request is never counted twice.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import threading
|
|
18
|
+
from typing import Any, Iterable, Optional, Tuple
|
|
19
|
+
|
|
20
|
+
from . import _core, context, reporter, reqsignal
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Requests evaluated in this process. A test seam: an ASGI request is
|
|
24
|
+
# evaluated after its response has reached the client (when the app's
|
|
25
|
+
# coroutine completes), so a test waits for this to move. Changes nothing.
|
|
26
|
+
_finished = 0
|
|
27
|
+
_finished_lock = threading.Lock()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def finished() -> int:
|
|
31
|
+
return _finished
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Request:
|
|
35
|
+
"""One request being observed, held on the request's Scope."""
|
|
36
|
+
|
|
37
|
+
__slots__ = ("method", "route", "started", "exception_reported")
|
|
38
|
+
|
|
39
|
+
def __init__(self, method: Any) -> None:
|
|
40
|
+
self.method = method
|
|
41
|
+
# The framework's matched route pattern, set when routing ran; None
|
|
42
|
+
# reports as "*".
|
|
43
|
+
self.route: Optional[str] = None
|
|
44
|
+
self.started = reqsignal.monotonic_ns()
|
|
45
|
+
# An `exception` was reported during this request (any context).
|
|
46
|
+
self.exception_reported = False
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def observing() -> bool:
|
|
50
|
+
"""True inside a request an outer Dev Bench middleware already
|
|
51
|
+
observes."""
|
|
52
|
+
scope = context.current()
|
|
53
|
+
return scope is not None and scope.request is not None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def begin(scope: context.Scope, method: Any) -> Optional[Request]:
|
|
57
|
+
"""Starts observing the request ``scope`` belongs to. None when this
|
|
58
|
+
process reports nothing (no DSN, or a DSN that does not parse): then
|
|
59
|
+
nothing is timed or counted. Never raises."""
|
|
60
|
+
try:
|
|
61
|
+
if not _core.direct():
|
|
62
|
+
return None
|
|
63
|
+
request = Request(method)
|
|
64
|
+
scope.request = request
|
|
65
|
+
return request
|
|
66
|
+
except Exception:
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def set_route(route: Any) -> None:
|
|
71
|
+
"""Records the matched route pattern on the request being observed."""
|
|
72
|
+
try:
|
|
73
|
+
scope = context.current()
|
|
74
|
+
request = scope.request if scope is not None else None
|
|
75
|
+
if request is not None and route is not None:
|
|
76
|
+
request.route = str(route)
|
|
77
|
+
except Exception:
|
|
78
|
+
pass
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def elapsed_ms(request: Request) -> int:
|
|
82
|
+
"""Whole milliseconds since the middleware's entry (truncated, so 3000.9
|
|
83
|
+
ms is 3000: not slow)."""
|
|
84
|
+
return (reqsignal.monotonic_ns() - request.started) // 1000000
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def finish(request: Optional[Request], status: Any, streaming: bool, duration_ms: Optional[int] = None) -> None:
|
|
88
|
+
"""The application handed back its response: classify, report, feed the
|
|
89
|
+
spike detector. ``duration_ms`` defaults to the time since the
|
|
90
|
+
middleware's entry. Never raises."""
|
|
91
|
+
if request is None:
|
|
92
|
+
return
|
|
93
|
+
try:
|
|
94
|
+
if duration_ms is None:
|
|
95
|
+
duration_ms = elapsed_ms(request)
|
|
96
|
+
code = status_code(status)
|
|
97
|
+
method = reqsignal.normalize_method(request.method)
|
|
98
|
+
route = reqsignal.normalize_route(request.route)
|
|
99
|
+
signal = reqsignal.classify(code, duration_ms, request.exception_reported, streaming)
|
|
100
|
+
if signal:
|
|
101
|
+
reporter.report_request(signal, method, route, code, duration_ms)
|
|
102
|
+
observe = getattr(_core.transport(), "observe_request", None)
|
|
103
|
+
if observe is not None:
|
|
104
|
+
observe(reqsignal.now_ms(), method, route, code)
|
|
105
|
+
except Exception:
|
|
106
|
+
pass
|
|
107
|
+
finally:
|
|
108
|
+
global _finished
|
|
109
|
+
with _finished_lock:
|
|
110
|
+
_finished += 1
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def status_code(status: Any) -> int:
|
|
114
|
+
"""An int from 503, "503 Service Unavailable" (WSGI) or b"503"; 0 when
|
|
115
|
+
there is none."""
|
|
116
|
+
try:
|
|
117
|
+
if isinstance(status, int):
|
|
118
|
+
return status
|
|
119
|
+
if isinstance(status, bytes):
|
|
120
|
+
status = status.decode("latin-1")
|
|
121
|
+
head = str(status).strip().split(" ", 1)[0]
|
|
122
|
+
return int(head) if head.isdigit() and len(head) == 3 else 0
|
|
123
|
+
except Exception:
|
|
124
|
+
return 0
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def event_stream(content_type: Any) -> bool:
|
|
128
|
+
"""A ``text/event-stream`` response (any case, parameters allowed): a
|
|
129
|
+
connection's lifetime, never slow."""
|
|
130
|
+
try:
|
|
131
|
+
if isinstance(content_type, bytes):
|
|
132
|
+
content_type = content_type.decode("latin-1")
|
|
133
|
+
return str(content_type or "").strip().lower().startswith("text/event-stream")
|
|
134
|
+
except Exception:
|
|
135
|
+
return False
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def header(headers: Optional[Iterable[Tuple[Any, Any]]], name: str) -> Any:
|
|
139
|
+
"""The first value of a header in a WSGI/ASGI header list."""
|
|
140
|
+
want = name.lower()
|
|
141
|
+
for key, value in headers or ():
|
|
142
|
+
k = key.decode("latin-1") if isinstance(key, bytes) else str(key)
|
|
143
|
+
if k.lower() == want:
|
|
144
|
+
return value
|
|
145
|
+
return None
|
|
@@ -18,7 +18,7 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
from typing import Any, Awaitable, Callable, MutableMapping, Optional
|
|
20
20
|
|
|
21
|
-
from . import _core, _integration, context
|
|
21
|
+
from . import _core, _integration, _requests, context
|
|
22
22
|
from .trace import Trace
|
|
23
23
|
|
|
24
24
|
Scope = MutableMapping[str, Any]
|
|
@@ -38,22 +38,41 @@ class DevbenchMiddleware:
|
|
|
38
38
|
if scope.get("type") != "http" or not _core.enabled():
|
|
39
39
|
await self.app(scope, receive, send)
|
|
40
40
|
return
|
|
41
|
+
# Inside another Dev Bench middleware: that one scopes, reports and
|
|
42
|
+
# observes this request.
|
|
43
|
+
if _requests.observing():
|
|
44
|
+
await self.app(scope, receive, send)
|
|
45
|
+
return
|
|
41
46
|
trace = _parse(scope)
|
|
47
|
+
answered: list = []
|
|
42
48
|
|
|
43
49
|
async def send_wrapper(message: Message) -> None:
|
|
44
50
|
if message.get("type") == "http.response.start":
|
|
51
|
+
try:
|
|
52
|
+
if not answered:
|
|
53
|
+
answered[:] = [message.get("status"), _requests.header(message.get("headers"), "content-type")]
|
|
54
|
+
except Exception:
|
|
55
|
+
pass
|
|
45
56
|
try:
|
|
46
57
|
message = _with_handled(message)
|
|
47
58
|
except Exception:
|
|
48
59
|
pass
|
|
49
60
|
await send(message)
|
|
50
61
|
|
|
51
|
-
with context.scoped(trace):
|
|
62
|
+
with context.scoped(trace) as request_scope:
|
|
63
|
+
request = _requests.begin(request_scope, scope.get("method"))
|
|
52
64
|
try:
|
|
53
65
|
await self.app(scope, receive, send_wrapper)
|
|
54
66
|
except Exception as error:
|
|
55
67
|
_integration.report_request_error(error, _symbol(scope))
|
|
56
68
|
raise
|
|
69
|
+
if request is not None:
|
|
70
|
+
# An app that returned without answering: the server sends
|
|
71
|
+
# 500 (uvicorn, hypercorn), and nothing was reported.
|
|
72
|
+
status = answered[0] if answered else 500
|
|
73
|
+
content_type = answered[1] if answered else None
|
|
74
|
+
request.route = _route(scope)
|
|
75
|
+
_requests.finish(request, status, _requests.event_stream(content_type))
|
|
57
76
|
|
|
58
77
|
|
|
59
78
|
def _parse(scope: Scope) -> Optional[Trace]:
|
|
@@ -66,6 +85,25 @@ def _parse(scope: Scope) -> Optional[Trace]:
|
|
|
66
85
|
return None
|
|
67
86
|
|
|
68
87
|
|
|
88
|
+
def _route(scope: Scope) -> Optional[str]:
|
|
89
|
+
"""The matched route's pattern: ``scope["route"].path`` (FastAPI, and
|
|
90
|
+
Starlette since it records the route), else the route whose endpoint the
|
|
91
|
+
router put in the scope. None (reported as "*") when nothing matched."""
|
|
92
|
+
try:
|
|
93
|
+
route = scope.get("route")
|
|
94
|
+
path = getattr(route, "path", None) if route is not None else None
|
|
95
|
+
if not isinstance(path, str):
|
|
96
|
+
endpoint, router = scope.get("endpoint"), scope.get("router")
|
|
97
|
+
if endpoint is not None and router is not None:
|
|
98
|
+
for candidate in getattr(router, "routes", None) or ():
|
|
99
|
+
if getattr(candidate, "endpoint", None) is endpoint:
|
|
100
|
+
path = getattr(candidate, "path", None)
|
|
101
|
+
break
|
|
102
|
+
return path if isinstance(path, str) else None
|
|
103
|
+
except Exception:
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
|
|
69
107
|
def _symbol(scope: Scope) -> Optional[str]:
|
|
70
108
|
# Starlette's router puts the matched endpoint in the scope.
|
|
71
109
|
endpoint = scope.get("endpoint")
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"""Where reports go, parsed from DEVBENCH_DSN, and everything else read at
|
|
2
|
-
start (docs/
|
|
2
|
+
start (docs/server-sdk/configuration.md, "Dev Bench naming and configuration";
|
|
3
3
|
DECISIONS #161)."""
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
@@ -18,13 +18,13 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
import contextlib
|
|
20
20
|
import contextvars
|
|
21
|
-
from typing import Iterator, Optional
|
|
21
|
+
from typing import Any, Iterator, Optional
|
|
22
22
|
|
|
23
23
|
from .trace import Trace
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
class Scope:
|
|
27
|
-
__slots__ = ("trace", "user", "handled", "__weakref__")
|
|
27
|
+
__slots__ = ("trace", "user", "handled", "request", "__weakref__")
|
|
28
28
|
|
|
29
29
|
def __init__(self, trace: Optional[Trace] = None) -> None:
|
|
30
30
|
self.trace = trace
|
|
@@ -34,6 +34,9 @@ class Scope:
|
|
|
34
34
|
# Failures handled during this request: the middleware turns a
|
|
35
35
|
# non-zero count into the x-adt-handled response header.
|
|
36
36
|
self.handled = 0
|
|
37
|
+
# The request middleware's observation (devbench._requests.Request)
|
|
38
|
+
# while a request is being observed; None in a job or a script.
|
|
39
|
+
self.request: Optional[Any] = None
|
|
37
40
|
|
|
38
41
|
|
|
39
42
|
_SCOPE: "contextvars.ContextVar[Optional[Scope]]" = contextvars.ContextVar("devbench_scope", default=None)
|
|
@@ -80,6 +83,20 @@ def scoped(trace: Optional[Trace]) -> Iterator[Scope]:
|
|
|
80
83
|
_SCOPE.set(None)
|
|
81
84
|
|
|
82
85
|
|
|
86
|
+
@contextlib.contextmanager
|
|
87
|
+
def within(scope: Scope) -> Iterator[Scope]:
|
|
88
|
+
"""Runs the block in an existing scope (a WSGI body closed after the
|
|
89
|
+
middleware returned), restoring whatever was there before."""
|
|
90
|
+
token = _SCOPE.set(scope)
|
|
91
|
+
try:
|
|
92
|
+
yield scope
|
|
93
|
+
finally:
|
|
94
|
+
try:
|
|
95
|
+
_SCOPE.reset(token)
|
|
96
|
+
except ValueError:
|
|
97
|
+
_SCOPE.set(None)
|
|
98
|
+
|
|
99
|
+
|
|
83
100
|
def enter(trace: Optional[Trace]) -> "tuple[Scope, contextvars.Token]":
|
|
84
101
|
"""For hooks whose start and end are separate calls (Celery's prerun and
|
|
85
102
|
postrun): returns the new scope and the token ``leave`` needs."""
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Direct mode (docs/
|
|
1
|
+
"""Direct mode (docs/server-sdk/transports.md, "Direct mode"): fingerprint and
|
|
2
2
|
count in-process, flush counts to ingest once a minute, upload one redacted
|
|
3
3
|
evidence bundle per fingerprint only when ingest asks, and answer ingest's
|
|
4
4
|
requests for the log lines of a trace (need_logs) from the in-process buffer,
|
|
@@ -34,10 +34,10 @@ import urllib.error
|
|
|
34
34
|
import urllib.request
|
|
35
35
|
from typing import IO, Any, Dict, List, Optional, Tuple
|
|
36
36
|
|
|
37
|
-
from . import fingerprint, logs, scrub
|
|
37
|
+
from . import fingerprint, logs, reqsignal, scrub
|
|
38
38
|
from .config import DSN
|
|
39
39
|
from .egress_policy import EgressPolicy
|
|
40
|
-
from .version import PROTOCOL_VERSION, VERSION
|
|
40
|
+
from .version import PROTOCOL_VERSION, SDK_NAME, VERSION
|
|
41
41
|
|
|
42
42
|
MAX_FINGERPRINTS = 512
|
|
43
43
|
MAX_USERS = 20
|
|
@@ -56,6 +56,8 @@ MAX_SLICE_BYTES = 64 * 1024
|
|
|
56
56
|
MAX_SLICES_PER_POST = 25
|
|
57
57
|
HTTP_TIMEOUT = 5.0
|
|
58
58
|
EXIT_TIMEOUT = 2.0
|
|
59
|
+
# Which SDK sent a flush, on every request of it (#310).
|
|
60
|
+
_SDK_FIELD = {"name": SDK_NAME, "version": VERSION}
|
|
59
61
|
|
|
60
62
|
|
|
61
63
|
class _Count:
|
|
@@ -146,9 +148,33 @@ class DirectTransport:
|
|
|
146
148
|
except Exception:
|
|
147
149
|
return False
|
|
148
150
|
|
|
151
|
+
def observe_request(self, t_ms: int, method: str, route: str, status: int) -> None:
|
|
152
|
+
"""Feeds one finished request to this process's 429 spike detector
|
|
153
|
+
(Capability 2b) and counts the spikes of any window it closed. The
|
|
154
|
+
first 429 starts the flush thread, so a process answering nothing but
|
|
155
|
+
429s still reports its spike within a minute. Never raises, never
|
|
156
|
+
does I/O."""
|
|
157
|
+
try:
|
|
158
|
+
self._check_fork()
|
|
159
|
+
closed = self._spikes.observe(t_ms, method, route, status)
|
|
160
|
+
if status == 429:
|
|
161
|
+
self._ensure_running()
|
|
162
|
+
self._record_spikes(closed)
|
|
163
|
+
except Exception:
|
|
164
|
+
pass
|
|
165
|
+
|
|
166
|
+
def end_requests(self) -> None:
|
|
167
|
+
"""Process exit: the open spike window is evaluated as it stands."""
|
|
168
|
+
try:
|
|
169
|
+
self._check_fork()
|
|
170
|
+
self._record_spikes(self._spikes.end())
|
|
171
|
+
except Exception:
|
|
172
|
+
pass
|
|
173
|
+
|
|
149
174
|
def stop(self) -> None:
|
|
150
175
|
"""A last flush (bounded to EXIT_TIMEOUT), then the thread stops."""
|
|
151
176
|
try:
|
|
177
|
+
self.end_requests()
|
|
152
178
|
self.flush(timeout=EXIT_TIMEOUT)
|
|
153
179
|
self._stopped = True
|
|
154
180
|
with self._wake:
|
|
@@ -205,8 +231,34 @@ class DirectTransport:
|
|
|
205
231
|
return self._exception_signal(payload)
|
|
206
232
|
if kind == "handled_failure":
|
|
207
233
|
return self._handled_signal(payload)
|
|
234
|
+
if kind == "request":
|
|
235
|
+
return self._request_signal(payload)
|
|
208
236
|
return None
|
|
209
237
|
|
|
238
|
+
def _request_signal(self, payload: dict) -> Optional[_Signal]:
|
|
239
|
+
"""A failed or slow request, or a 429 spike (Capability 2b): the
|
|
240
|
+
sidecar's handling of the ``request`` message. A spike carries no
|
|
241
|
+
user."""
|
|
242
|
+
signal = payload.get("signal")
|
|
243
|
+
try:
|
|
244
|
+
built = reqsignal.build(signal, payload.get("method"), payload.get("route"), payload.get("status") or 0,
|
|
245
|
+
payload.get("duration_ms") or 0, payload.get("count") or 0, payload.get("total") or 0)
|
|
246
|
+
except reqsignal.InvalidSignal:
|
|
247
|
+
return None
|
|
248
|
+
fp = built.fingerprint(self.service)
|
|
249
|
+
if fp is None:
|
|
250
|
+
return None
|
|
251
|
+
user = None if signal == reqsignal.RATE_LIMIT_SPIKE else _user(payload.get("user"))
|
|
252
|
+
return _Signal(fp, built.kind, built.text, user, [], built.exemplar)
|
|
253
|
+
|
|
254
|
+
def _record_spikes(self, spikes: List[reqsignal.Spike]) -> None:
|
|
255
|
+
for spike in spikes:
|
|
256
|
+
signal = self._request_signal({"kind": "request", "signal": reqsignal.RATE_LIMIT_SPIKE, "method": spike.method,
|
|
257
|
+
"route": spike.route, "status": 429, "count": spike.count, "total": spike.total})
|
|
258
|
+
if signal is not None:
|
|
259
|
+
with self._lock:
|
|
260
|
+
self._record(signal)
|
|
261
|
+
|
|
210
262
|
def _exception_signal(self, payload: dict) -> Optional[_Signal]:
|
|
211
263
|
type_ = _text(payload.get("error"))
|
|
212
264
|
message = _text(payload.get("message"))[:MAX_MESSAGE]
|
|
@@ -270,6 +322,8 @@ class DirectTransport:
|
|
|
270
322
|
self._window: Dict[str, _Count] = {}
|
|
271
323
|
self._details: Dict[str, _Detail] = {}
|
|
272
324
|
self._overflowed = 0
|
|
325
|
+
# Per process: a forked child starts its own windows.
|
|
326
|
+
self._spikes = reqsignal.SpikeDetector()
|
|
273
327
|
self._sensor_id = "py-" + secrets.token_hex(10)
|
|
274
328
|
self._thread: Optional[threading.Thread] = None
|
|
275
329
|
|
|
@@ -317,6 +371,9 @@ class DirectTransport:
|
|
|
317
371
|
"""Phase 1, then whatever the responses asked for: learned rules
|
|
318
372
|
first (a rule arriving with a request for the shape it governs
|
|
319
373
|
applies to that request), then log slices, then evidence."""
|
|
374
|
+
# A spike window that has ended is closed here, so it is reported
|
|
375
|
+
# even when no later request arrives to close it.
|
|
376
|
+
self._record_spikes(self._spikes.tick(reqsignal.now_ms()))
|
|
320
377
|
with self._lock:
|
|
321
378
|
counts, overflowed = self._window, self._overflowed
|
|
322
379
|
self._window = {}
|
|
@@ -383,8 +440,11 @@ class DirectTransport:
|
|
|
383
440
|
return out
|
|
384
441
|
|
|
385
442
|
def _envelope(self, counts: List[dict], overflowed: int) -> bytes:
|
|
443
|
+
"""Every flush request's body: counts, each request of a split window
|
|
444
|
+
and the empty poll are all built here, so all carry ``sdk``."""
|
|
386
445
|
body: Dict[str, Any] = {"v": PROTOCOL_VERSION, "tenant": "", "source": "server", "service": self.service,
|
|
387
|
-
"release": self.release, "sensor_id": self._sensor_id, "
|
|
446
|
+
"release": self.release, "sensor_id": self._sensor_id, "sdk": _SDK_FIELD,
|
|
447
|
+
"counts": counts}
|
|
388
448
|
if overflowed > 0:
|
|
389
449
|
body["overflowed"] = overflowed
|
|
390
450
|
return _json(body)
|
|
@@ -17,7 +17,7 @@ from typing import Any, Callable, Optional
|
|
|
17
17
|
|
|
18
18
|
from asgiref.sync import iscoroutinefunction, markcoroutinefunction
|
|
19
19
|
|
|
20
|
-
from .. import _core, _integration, context
|
|
20
|
+
from .. import _core, _integration, _requests, context
|
|
21
21
|
from ..trace import WSGI_HEADER, Trace
|
|
22
22
|
|
|
23
23
|
|
|
@@ -34,19 +34,23 @@ class DevbenchMiddleware:
|
|
|
34
34
|
def __call__(self, request: Any) -> Any:
|
|
35
35
|
if self._async:
|
|
36
36
|
return self.__acall__(request)
|
|
37
|
-
if not _core.enabled():
|
|
37
|
+
if not _core.enabled() or _requests.observing():
|
|
38
38
|
return self.get_response(request)
|
|
39
|
-
with context.scoped(_parse(request)):
|
|
39
|
+
with context.scoped(_parse(request)) as scope:
|
|
40
|
+
observed = _requests.begin(scope, getattr(request, "method", None))
|
|
40
41
|
response = self.get_response(request)
|
|
41
42
|
_mark_handled(response)
|
|
43
|
+
_finish(observed, request, response)
|
|
42
44
|
return response
|
|
43
45
|
|
|
44
46
|
async def __acall__(self, request: Any) -> Any:
|
|
45
|
-
if not _core.enabled():
|
|
47
|
+
if not _core.enabled() or _requests.observing():
|
|
46
48
|
return await self.get_response(request)
|
|
47
|
-
with context.scoped(_parse(request)):
|
|
49
|
+
with context.scoped(_parse(request)) as scope:
|
|
50
|
+
observed = _requests.begin(scope, getattr(request, "method", None))
|
|
48
51
|
response = await self.get_response(request)
|
|
49
52
|
_mark_handled(response)
|
|
53
|
+
_finish(observed, request, response)
|
|
50
54
|
return response
|
|
51
55
|
|
|
52
56
|
|
|
@@ -76,6 +80,22 @@ def _symbol(request: Any) -> Optional[str]:
|
|
|
76
80
|
return None
|
|
77
81
|
|
|
78
82
|
|
|
83
|
+
def _finish(observed: Any, request: Any, response: Any) -> None:
|
|
84
|
+
"""Capability 2b: the route is the resolved URL pattern
|
|
85
|
+
(``resolver_match.route``, joined across includes); none matched is
|
|
86
|
+
"*"."""
|
|
87
|
+
if observed is None:
|
|
88
|
+
return
|
|
89
|
+
try:
|
|
90
|
+
match = getattr(request, "resolver_match", None)
|
|
91
|
+
observed.route = getattr(match, "route", None) if match is not None else None
|
|
92
|
+
status = getattr(response, "status_code", 0)
|
|
93
|
+
content_type = response.get("Content-Type") if response is not None else None
|
|
94
|
+
_requests.finish(observed, status, _requests.event_stream(content_type))
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
|
|
98
|
+
|
|
79
99
|
def _mark_handled(response: Any) -> None:
|
|
80
100
|
try:
|
|
81
101
|
count = _integration.handled_count()
|
|
@@ -9,16 +9,18 @@
|
|
|
9
9
|
Wraps ``app.wsgi_app`` in the WSGI middleware (trace, identity scope,
|
|
10
10
|
``x-adt-handled``) and subscribes to ``got_request_exception``, which Flask
|
|
11
11
|
sends for an unhandled exception before it answers 500 (an ``abort(404)`` or
|
|
12
|
-
any other HTTPException is a response, not a failure, and never reaches it)
|
|
12
|
+
any other HTTPException is a response, not a failure, and never reaches it),
|
|
13
|
+
and to ``request_started``, which records the matched URL rule as the
|
|
14
|
+
request's route (Capability 2b: failed and slow requests, 429 spikes).
|
|
13
15
|
"""
|
|
14
16
|
|
|
15
17
|
from __future__ import annotations
|
|
16
18
|
|
|
17
19
|
from typing import Any, Optional
|
|
18
20
|
|
|
19
|
-
from flask import got_request_exception, request
|
|
21
|
+
from flask import got_request_exception, request, request_started
|
|
20
22
|
|
|
21
|
-
from . import _integration
|
|
23
|
+
from . import _integration, _requests
|
|
22
24
|
from .wsgi import DevbenchMiddleware
|
|
23
25
|
|
|
24
26
|
|
|
@@ -36,6 +38,18 @@ class Devbench:
|
|
|
36
38
|
_integration.setup(service=getattr(app, "import_name", None))
|
|
37
39
|
app.wsgi_app = DevbenchMiddleware(app.wsgi_app)
|
|
38
40
|
got_request_exception.connect(_on_exception, app, weak=False)
|
|
41
|
+
request_started.connect(_on_request, app, weak=False)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _on_request(sender: Any, **_: Any) -> None:
|
|
45
|
+
"""Routing has run: the matched rule (``/orders/<int:id>``) is the
|
|
46
|
+
request's route for Capability 2b. None matched (a 404) is "*"."""
|
|
47
|
+
try:
|
|
48
|
+
rule = request.url_rule
|
|
49
|
+
if rule is not None:
|
|
50
|
+
_requests.set_route(rule.rule)
|
|
51
|
+
except Exception:
|
|
52
|
+
pass
|
|
39
53
|
|
|
40
54
|
|
|
41
55
|
def _on_exception(sender: Any, exception: Optional[BaseException] = None, **_: Any) -> None:
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""In-process log capture (docs/
|
|
1
|
+
"""In-process log capture (docs/server-sdk/configuration.md, "In-process log capture
|
|
2
2
|
(direct mode)"): recent log lines kept in memory, keyed by the trace that was
|
|
3
3
|
current when each was written, so triage can be handed the server lines of
|
|
4
4
|
one user action without a sidecar.
|