devbench 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devbench/__init__.py +121 -0
- devbench/__main__.py +24 -0
- devbench/_core.py +252 -0
- devbench/_integration.py +107 -0
- devbench/asgi.py +82 -0
- devbench/celery.py +152 -0
- devbench/config.py +189 -0
- devbench/context.py +94 -0
- devbench/direct.py +639 -0
- devbench/django/__init__.py +12 -0
- devbench/django/apps.py +68 -0
- devbench/django/management/__init__.py +0 -0
- devbench/django/management/commands/__init__.py +0 -0
- devbench/django/management/commands/devbench_test.py +18 -0
- devbench/django/middleware.py +87 -0
- devbench/egress_policy.py +101 -0
- devbench/fastapi.py +14 -0
- devbench/fingerprint.py +199 -0
- devbench/flask.py +54 -0
- devbench/identity.py +50 -0
- devbench/logs.py +282 -0
- devbench/py.typed +0 -0
- devbench/reporter.py +233 -0
- devbench/scrub.py +247 -0
- devbench/trace.py +82 -0
- devbench/version.py +12 -0
- devbench/wsgi.py +67 -0
- devbench-0.1.0.dist-info/METADATA +259 -0
- devbench-0.1.0.dist-info/RECORD +32 -0
- devbench-0.1.0.dist-info/WHEEL +5 -0
- devbench-0.1.0.dist-info/licenses/LICENSE +21 -0
- devbench-0.1.0.dist-info/top_level.txt +1 -0
devbench/direct.py
ADDED
|
@@ -0,0 +1,639 @@
|
|
|
1
|
+
"""Direct mode (docs/SERVER_SDK_SPEC.md, "Direct mode"): fingerprint and
|
|
2
|
+
count in-process, flush counts to ingest once a minute, upload one redacted
|
|
3
|
+
evidence bundle per fingerprint only when ingest asks, and answer ingest's
|
|
4
|
+
requests for the log lines of a trace (need_logs) from the in-process buffer,
|
|
5
|
+
redacted. The same phase-1/phase-2 protocol as the browser sensor and the
|
|
6
|
+
Ruby and Go SDKs, and the same signals and bundle the sidecar builds.
|
|
7
|
+
|
|
8
|
+
Never harms the host:
|
|
9
|
+
|
|
10
|
+
* The calling thread fingerprints the report (no I/O) and folds it into a
|
|
11
|
+
count under a lock that is never held across I/O. Nothing is queued per
|
|
12
|
+
occurrence: the table of counts *is* the buffer, bounded at
|
|
13
|
+
MAX_FINGERPRINTS per window; further distinct fingerprints are counted in
|
|
14
|
+
``overflowed``.
|
|
15
|
+
* All network I/O is on a background daemon thread (or an explicit flush),
|
|
16
|
+
with timeouts of at most 5 s, one retry with jitter, then discard.
|
|
17
|
+
* Nothing raises into the application.
|
|
18
|
+
* Fork-safe: gunicorn, uWSGI and Celery's prefork pool fork after import. A
|
|
19
|
+
child notices the PID change on its first report, drops the parent's
|
|
20
|
+
counts (the parent sends those), takes a new sensor id, and starts its own
|
|
21
|
+
thread.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import random
|
|
29
|
+
import re
|
|
30
|
+
import secrets
|
|
31
|
+
import threading
|
|
32
|
+
import time
|
|
33
|
+
import urllib.error
|
|
34
|
+
import urllib.request
|
|
35
|
+
from typing import IO, Any, Dict, List, Optional, Tuple
|
|
36
|
+
|
|
37
|
+
from . import fingerprint, logs, scrub
|
|
38
|
+
from .config import DSN
|
|
39
|
+
from .egress_policy import EgressPolicy
|
|
40
|
+
from .version import PROTOCOL_VERSION, VERSION
|
|
41
|
+
|
|
42
|
+
MAX_FINGERPRINTS = 512
|
|
43
|
+
MAX_USERS = 20
|
|
44
|
+
MAX_DETAILS = 512
|
|
45
|
+
MAX_COUNTS_PER_POST = 512
|
|
46
|
+
MAX_BODY_BYTES = 192 * 1024
|
|
47
|
+
MAX_MESSAGE = 2000
|
|
48
|
+
MAX_FRAMES = 50
|
|
49
|
+
MAX_IDENTITY = 255
|
|
50
|
+
MAX_RESPONSE_BYTES = 64 * 1024
|
|
51
|
+
# Log slices: 200 lines and 64 KiB are what the store keeps of a slice;
|
|
52
|
+
# sending more would spend the customer's bandwidth on lines truncated on
|
|
53
|
+
# arrival. 25 slices is what one delivery may carry.
|
|
54
|
+
MAX_SLICE_LINES = 200
|
|
55
|
+
MAX_SLICE_BYTES = 64 * 1024
|
|
56
|
+
MAX_SLICES_PER_POST = 25
|
|
57
|
+
HTTP_TIMEOUT = 5.0
|
|
58
|
+
EXIT_TIMEOUT = 2.0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class _Count:
|
|
62
|
+
__slots__ = ("kind", "n", "first", "last", "users", "users_overflow")
|
|
63
|
+
|
|
64
|
+
def __init__(self, kind: str, now: int) -> None:
|
|
65
|
+
self.kind = kind
|
|
66
|
+
self.n = 0
|
|
67
|
+
self.first = now
|
|
68
|
+
self.last = now
|
|
69
|
+
self.users: List[dict] = []
|
|
70
|
+
self.users_overflow = 0
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class _Detail:
|
|
74
|
+
__slots__ = ("kind", "text", "exemplar", "frames")
|
|
75
|
+
|
|
76
|
+
def __init__(self, kind: str, text: str, exemplar: str, frames: list) -> None:
|
|
77
|
+
self.kind = kind
|
|
78
|
+
self.text = text
|
|
79
|
+
self.exemplar = exemplar
|
|
80
|
+
self.frames = frames
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class _Signal:
|
|
84
|
+
__slots__ = ("fp", "kind", "text", "user", "frames", "exemplar")
|
|
85
|
+
|
|
86
|
+
def __init__(self, fp: str, kind: str, text: str, user: Optional[dict], frames: list, exemplar: str) -> None:
|
|
87
|
+
self.fp = fp
|
|
88
|
+
self.kind = kind
|
|
89
|
+
self.text = text
|
|
90
|
+
self.user = user
|
|
91
|
+
self.frames = frames
|
|
92
|
+
self.exemplar = exemplar
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class _Response:
|
|
96
|
+
__slots__ = ("code", "body", "reason")
|
|
97
|
+
|
|
98
|
+
def __init__(self, code: int, body: bytes, reason: str) -> None:
|
|
99
|
+
self.code = code
|
|
100
|
+
self.body = body
|
|
101
|
+
self.reason = reason
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class DirectTransport:
|
|
105
|
+
def __init__(self, dsn: DSN, service: str, release: str, interval: float = 60.0,
|
|
106
|
+
warn: Optional[Any] = None) -> None:
|
|
107
|
+
self.dsn = dsn
|
|
108
|
+
self.service = service or ""
|
|
109
|
+
self.release = release or ""
|
|
110
|
+
self.interval = float(interval) if interval and float(interval) > 0 else 60.0
|
|
111
|
+
self._warn = warn or (lambda kind, message: None)
|
|
112
|
+
# Learned redaction rules from the last response that carried a set.
|
|
113
|
+
# Not per process: a forked child keeps them until ingest re-sends.
|
|
114
|
+
self.policy: Optional[EgressPolicy] = None
|
|
115
|
+
self._fork_lock = threading.Lock()
|
|
116
|
+
self._stopped = False
|
|
117
|
+
self._reset_state()
|
|
118
|
+
|
|
119
|
+
# ---- The app's side: called on the application's thread.
|
|
120
|
+
|
|
121
|
+
def deliver(self, payload: dict) -> None:
|
|
122
|
+
"""Takes a report (the dict reporter builds). Never raises, never
|
|
123
|
+
does I/O."""
|
|
124
|
+
try:
|
|
125
|
+
signal = self._signal_for(payload)
|
|
126
|
+
if signal is None:
|
|
127
|
+
return
|
|
128
|
+
self._ensure_running()
|
|
129
|
+
with self._lock:
|
|
130
|
+
self._record(signal)
|
|
131
|
+
except Exception:
|
|
132
|
+
pass
|
|
133
|
+
|
|
134
|
+
def flush(self, timeout: float = HTTP_TIMEOUT) -> bool:
|
|
135
|
+
"""Sends everything counted so far, now, waiting at most ``timeout``
|
|
136
|
+
seconds. Returns True when nothing is left unsent."""
|
|
137
|
+
try:
|
|
138
|
+
self._check_fork()
|
|
139
|
+
deadline = time.monotonic() + float(timeout)
|
|
140
|
+
if not self._cycle.acquire(timeout=max(0.0, deadline - time.monotonic())):
|
|
141
|
+
return False
|
|
142
|
+
try:
|
|
143
|
+
return self._flush_cycle(deadline)
|
|
144
|
+
finally:
|
|
145
|
+
self._cycle.release()
|
|
146
|
+
except Exception:
|
|
147
|
+
return False
|
|
148
|
+
|
|
149
|
+
def stop(self) -> None:
|
|
150
|
+
"""A last flush (bounded to EXIT_TIMEOUT), then the thread stops."""
|
|
151
|
+
try:
|
|
152
|
+
self.flush(timeout=EXIT_TIMEOUT)
|
|
153
|
+
self._stopped = True
|
|
154
|
+
with self._wake:
|
|
155
|
+
self._wake.notify_all()
|
|
156
|
+
thread = self._thread
|
|
157
|
+
if thread is not None and thread is not threading.current_thread():
|
|
158
|
+
thread.join(0.5)
|
|
159
|
+
except Exception:
|
|
160
|
+
pass
|
|
161
|
+
|
|
162
|
+
def start(self) -> None:
|
|
163
|
+
"""Starts the background flush thread if it is not running (in this
|
|
164
|
+
process). Called when this process first holds a log line, so a
|
|
165
|
+
worker that reports nothing still polls for log requests."""
|
|
166
|
+
try:
|
|
167
|
+
self._ensure_running()
|
|
168
|
+
except Exception:
|
|
169
|
+
pass
|
|
170
|
+
|
|
171
|
+
def pending(self) -> Dict[str, int]:
|
|
172
|
+
"""Counts not yet flushed, by fingerprint. For tests and
|
|
173
|
+
diagnostics."""
|
|
174
|
+
with self._lock:
|
|
175
|
+
return {fp: c.n for fp, c in self._window.items()}
|
|
176
|
+
|
|
177
|
+
def self_test(self, out: IO[str]) -> bool:
|
|
178
|
+
"""``python -m devbench test``: asks ingest whether this DSN's key is
|
|
179
|
+
valid, and for which tenant and environment. Creates nothing."""
|
|
180
|
+
out.write("Dev Bench: checking the DSN against %s (service %s)\n" % (self.dsn, json.dumps(self.service)))
|
|
181
|
+
response, error = self._attempt("POST", self.dsn.check_url, b"{}", self._flush_headers(), HTTP_TIMEOUT)
|
|
182
|
+
if error is not None or response is None:
|
|
183
|
+
out.write(" could not reach %s: %s: %s\n" % (self.dsn, type(error).__name__, _describe(error)))
|
|
184
|
+
return False
|
|
185
|
+
if not 200 <= response.code < 300:
|
|
186
|
+
out.write(" HTTP %d: %s\n" % (response.code, self._explain(response)))
|
|
187
|
+
return False
|
|
188
|
+
try:
|
|
189
|
+
info = json.loads(response.body.decode("utf-8", "replace") or "{}")
|
|
190
|
+
except ValueError:
|
|
191
|
+
info = {}
|
|
192
|
+
if not isinstance(info, dict):
|
|
193
|
+
info = {}
|
|
194
|
+
out.write(" HTTP %d: accepted - tenant %s, environment %s. Nothing was recorded; real errors will appear as issues.\n"
|
|
195
|
+
% (response.code, json.dumps(info.get("tenant")), json.dumps(info.get("environment"))))
|
|
196
|
+
return True
|
|
197
|
+
|
|
198
|
+
# ---- Signals: exactly what the sidecar builds (internal/sidecar/control.go)
|
|
199
|
+
|
|
200
|
+
def _signal_for(self, payload: Any) -> Optional[_Signal]:
|
|
201
|
+
if not isinstance(payload, dict):
|
|
202
|
+
return None
|
|
203
|
+
kind = payload.get("kind")
|
|
204
|
+
if kind == "exception":
|
|
205
|
+
return self._exception_signal(payload)
|
|
206
|
+
if kind == "handled_failure":
|
|
207
|
+
return self._handled_signal(payload)
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
def _exception_signal(self, payload: dict) -> Optional[_Signal]:
|
|
211
|
+
type_ = _text(payload.get("error"))
|
|
212
|
+
message = _text(payload.get("message"))[:MAX_MESSAGE]
|
|
213
|
+
frames = list(payload.get("frames") or [])[:MAX_FRAMES]
|
|
214
|
+
fp = fingerprint.compute("error", "server", self.service, type_, message, frames)
|
|
215
|
+
if fp is None:
|
|
216
|
+
return None
|
|
217
|
+
symbol = _text(payload.get("symbol"))
|
|
218
|
+
text = "%s at %s" % (type_, symbol) if symbol else type_
|
|
219
|
+
return _Signal(fp, "error", text, _user(payload.get("user")), frames, _exemplar(payload))
|
|
220
|
+
|
|
221
|
+
def _handled_signal(self, payload: dict) -> Optional[_Signal]:
|
|
222
|
+
# Frames and message are not part of it, so fingerprints never move.
|
|
223
|
+
symbol = _text(payload.get("symbol"))
|
|
224
|
+
fp = fingerprint.compute("handled_failure", "server", self.service, _text(payload.get("error")), symbol)
|
|
225
|
+
if fp is None:
|
|
226
|
+
return None
|
|
227
|
+
frames = list(payload.get("frames") or [])[:MAX_FRAMES]
|
|
228
|
+
return _Signal(fp, "handled_failure", symbol, _user(payload.get("user")), frames, _exemplar(payload))
|
|
229
|
+
|
|
230
|
+
# ---- The table (caller holds self._lock)
|
|
231
|
+
|
|
232
|
+
def _record(self, signal: _Signal) -> None:
|
|
233
|
+
now = int(time.time())
|
|
234
|
+
entry = self._window.get(signal.fp)
|
|
235
|
+
if entry is None:
|
|
236
|
+
if len(self._window) >= MAX_FINGERPRINTS:
|
|
237
|
+
self._overflowed += 1
|
|
238
|
+
return
|
|
239
|
+
entry = self._window[signal.fp] = _Count(signal.kind, now)
|
|
240
|
+
entry.n += 1
|
|
241
|
+
if now > entry.last:
|
|
242
|
+
entry.last = now
|
|
243
|
+
if signal.user is not None and signal.user not in entry.users:
|
|
244
|
+
if len(entry.users) >= MAX_USERS:
|
|
245
|
+
entry.users_overflow += 1
|
|
246
|
+
else:
|
|
247
|
+
entry.users.append(signal.user)
|
|
248
|
+
self._remember(signal)
|
|
249
|
+
|
|
250
|
+
def _remember(self, signal: _Signal) -> None:
|
|
251
|
+
"""The first occurrence's detail, kept across windows so evidence can
|
|
252
|
+
be answered after the flush that reported it. Bounded, least recently
|
|
253
|
+
seen first out."""
|
|
254
|
+
detail = self._details.pop(signal.fp, None)
|
|
255
|
+
if detail is None:
|
|
256
|
+
detail = _Detail(signal.kind, signal.text, signal.exemplar, signal.frames)
|
|
257
|
+
elif not detail.frames and signal.frames:
|
|
258
|
+
detail.frames = signal.frames
|
|
259
|
+
self._details[signal.fp] = detail
|
|
260
|
+
while len(self._details) > MAX_DETAILS:
|
|
261
|
+
self._details.pop(next(iter(self._details)))
|
|
262
|
+
|
|
263
|
+
# ---- Lifecycle
|
|
264
|
+
|
|
265
|
+
def _reset_state(self) -> None:
|
|
266
|
+
self._pid = os.getpid()
|
|
267
|
+
self._lock = threading.Lock()
|
|
268
|
+
self._wake = threading.Condition(self._lock)
|
|
269
|
+
self._cycle = threading.Lock()
|
|
270
|
+
self._window: Dict[str, _Count] = {}
|
|
271
|
+
self._details: Dict[str, _Detail] = {}
|
|
272
|
+
self._overflowed = 0
|
|
273
|
+
self._sensor_id = "py-" + secrets.token_hex(10)
|
|
274
|
+
self._thread: Optional[threading.Thread] = None
|
|
275
|
+
|
|
276
|
+
def _check_fork(self) -> None:
|
|
277
|
+
if self._pid == os.getpid():
|
|
278
|
+
return
|
|
279
|
+
with self._fork_lock:
|
|
280
|
+
if self._pid != os.getpid():
|
|
281
|
+
self._reset_state()
|
|
282
|
+
|
|
283
|
+
def _ensure_running(self) -> None:
|
|
284
|
+
self._check_fork()
|
|
285
|
+
thread = self._thread
|
|
286
|
+
if self._stopped or (thread is not None and thread.is_alive()):
|
|
287
|
+
return
|
|
288
|
+
with self._fork_lock:
|
|
289
|
+
thread = self._thread
|
|
290
|
+
if self._stopped or (thread is not None and thread.is_alive()):
|
|
291
|
+
return
|
|
292
|
+
thread = threading.Thread(target=self._run, name="devbench-flush", daemon=True)
|
|
293
|
+
self._thread = thread
|
|
294
|
+
thread.start()
|
|
295
|
+
|
|
296
|
+
def _run(self) -> None:
|
|
297
|
+
next_at = time.monotonic() + self.interval
|
|
298
|
+
while not self._stopped:
|
|
299
|
+
with self._wake:
|
|
300
|
+
while not self._stopped:
|
|
301
|
+
remaining = next_at - time.monotonic()
|
|
302
|
+
if remaining <= 0:
|
|
303
|
+
break
|
|
304
|
+
self._wake.wait(remaining)
|
|
305
|
+
if self._stopped:
|
|
306
|
+
break
|
|
307
|
+
next_at = time.monotonic() + self.interval
|
|
308
|
+
try:
|
|
309
|
+
with self._cycle:
|
|
310
|
+
self._flush_cycle(time.monotonic() + HTTP_TIMEOUT * 4)
|
|
311
|
+
except Exception:
|
|
312
|
+
pass
|
|
313
|
+
|
|
314
|
+
# ---- Flushing
|
|
315
|
+
|
|
316
|
+
def _flush_cycle(self, deadline: float) -> bool:
|
|
317
|
+
"""Phase 1, then whatever the responses asked for: learned rules
|
|
318
|
+
first (a rule arriving with a request for the shape it governs
|
|
319
|
+
applies to that request), then log slices, then evidence."""
|
|
320
|
+
with self._lock:
|
|
321
|
+
counts, overflowed = self._window, self._overflowed
|
|
322
|
+
self._window = {}
|
|
323
|
+
self._overflowed = 0
|
|
324
|
+
# Nothing counted: poll only while this process holds traced lines,
|
|
325
|
+
# since need_logs only rides a flush response. Holding none, send
|
|
326
|
+
# nothing.
|
|
327
|
+
if not counts:
|
|
328
|
+
if not logs.holding():
|
|
329
|
+
return True
|
|
330
|
+
posts = [self._envelope([], 0)]
|
|
331
|
+
else:
|
|
332
|
+
posts = self._bodies(counts, overflowed)
|
|
333
|
+
|
|
334
|
+
ok = True
|
|
335
|
+
wanted: List[dict] = []
|
|
336
|
+
log_keys: List[str] = []
|
|
337
|
+
rules: Optional[list] = None
|
|
338
|
+
for body in posts:
|
|
339
|
+
response = self._request("POST", self.dsn.flush_url, body, self._flush_headers(), deadline)
|
|
340
|
+
if response is None:
|
|
341
|
+
ok = False
|
|
342
|
+
continue
|
|
343
|
+
if 200 <= response.code < 300:
|
|
344
|
+
evidence, keys, redaction = self._answers(response)
|
|
345
|
+
wanted.extend(evidence)
|
|
346
|
+
log_keys.extend(keys)
|
|
347
|
+
if redaction is not None:
|
|
348
|
+
rules = redaction
|
|
349
|
+
else:
|
|
350
|
+
ok = False
|
|
351
|
+
self._warn("http_%d" % response.code, "Dev Bench refused a flush: HTTP %d: %s" % (response.code, self._explain(response)))
|
|
352
|
+
if rules is not None:
|
|
353
|
+
self.policy = EgressPolicy.compile(rules)
|
|
354
|
+
if log_keys:
|
|
355
|
+
self._deliver_logs(list(dict.fromkeys(log_keys)), deadline)
|
|
356
|
+
for ask in wanted:
|
|
357
|
+
self._upload(ask, deadline)
|
|
358
|
+
return ok
|
|
359
|
+
|
|
360
|
+
def _bodies(self, counts: Dict[str, _Count], overflowed: int) -> List[bytes]:
|
|
361
|
+
"""Counts split so each request stays under MAX_BODY_BYTES (measured,
|
|
362
|
+
not estimated) and MAX_COUNTS_PER_POST. ``overflowed`` rides on the
|
|
363
|
+
first."""
|
|
364
|
+
out: List[bytes] = []
|
|
365
|
+
current: List[dict] = []
|
|
366
|
+
size = 0
|
|
367
|
+
budget = MAX_BODY_BYTES - len(self._envelope([], overflowed))
|
|
368
|
+
for fp, c in counts.items():
|
|
369
|
+
entry: Dict[str, Any] = {"fp": fp, "n": c.n, "first": c.first, "last": c.last, "kind": c.kind}
|
|
370
|
+
if c.users:
|
|
371
|
+
entry["users"] = c.users
|
|
372
|
+
if c.users_overflow > 0:
|
|
373
|
+
entry["users_overflow"] = c.users_overflow
|
|
374
|
+
size_of = len(_json(entry)) + 1
|
|
375
|
+
if current and (len(current) >= MAX_COUNTS_PER_POST or size + size_of > budget):
|
|
376
|
+
out.append(self._envelope(current, 0 if out else overflowed))
|
|
377
|
+
current = []
|
|
378
|
+
size = 0
|
|
379
|
+
current.append(entry)
|
|
380
|
+
size += size_of
|
|
381
|
+
if current:
|
|
382
|
+
out.append(self._envelope(current, 0 if out else overflowed))
|
|
383
|
+
return out
|
|
384
|
+
|
|
385
|
+
def _envelope(self, counts: List[dict], overflowed: int) -> bytes:
|
|
386
|
+
body: Dict[str, Any] = {"v": PROTOCOL_VERSION, "tenant": "", "source": "server", "service": self.service,
|
|
387
|
+
"release": self.release, "sensor_id": self._sensor_id, "counts": counts}
|
|
388
|
+
if overflowed > 0:
|
|
389
|
+
body["overflowed"] = overflowed
|
|
390
|
+
return _json(body)
|
|
391
|
+
|
|
392
|
+
def _flush_headers(self) -> Dict[str, str]:
|
|
393
|
+
return {"content-type": "application/json", "x-adt-key": self.dsn.key,
|
|
394
|
+
"user-agent": "devbench-python/%s" % VERSION}
|
|
395
|
+
|
|
396
|
+
def _answers(self, response: _Response) -> Tuple[List[dict], List[str], Optional[list]]:
|
|
397
|
+
"""What a flush response asked for: evidence, log slices (need_logs
|
|
398
|
+
trace keys) and the learned rule set (a list, or None when the
|
|
399
|
+
response carried none, which leaves the rules in force, unlike an
|
|
400
|
+
empty list, which withdraws them)."""
|
|
401
|
+
body = response.body
|
|
402
|
+
if not body or len(body) > MAX_RESPONSE_BYTES:
|
|
403
|
+
return [], [], None
|
|
404
|
+
try:
|
|
405
|
+
parsed = json.loads(body.decode("utf-8", "replace"))
|
|
406
|
+
except ValueError:
|
|
407
|
+
# Counts were accepted; an unreadable answer only loses the asks.
|
|
408
|
+
return [], [], None
|
|
409
|
+
if not isinstance(parsed, dict):
|
|
410
|
+
return [], [], None
|
|
411
|
+
evidence = parsed.get("need_evidence")
|
|
412
|
+
need_logs = parsed.get("need_logs")
|
|
413
|
+
redaction = parsed.get("redaction")
|
|
414
|
+
return (
|
|
415
|
+
[a for a in evidence if isinstance(a, dict)] if isinstance(evidence, list) else [],
|
|
416
|
+
[k for k in (_log_key(a) for a in need_logs) if k] if isinstance(need_logs, list) else [],
|
|
417
|
+
redaction if isinstance(redaction, list) else None,
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
def _deliver_logs(self, keys: List[str], deadline: float) -> None:
|
|
421
|
+
"""Phase 3: the lines this process holds for each requested trace,
|
|
422
|
+
redacted with the exemplars' strict egress plus the learned rules,
|
|
423
|
+
POSTed with the secret key. A key with no lines here sends nothing:
|
|
424
|
+
another process may hold them."""
|
|
425
|
+
try:
|
|
426
|
+
policy = self.policy
|
|
427
|
+
slices = []
|
|
428
|
+
for key in keys:
|
|
429
|
+
held = logs.lookup(key, MAX_SLICE_LINES)
|
|
430
|
+
if not held:
|
|
431
|
+
continue
|
|
432
|
+
slices.append({"trace": key, "lines": _bound_slice([self._egress_line(line, policy) for line in held])})
|
|
433
|
+
for body in _slice_bodies(slices):
|
|
434
|
+
response = self._request("POST", self.dsn.logs_url, body, self._flush_headers(), deadline)
|
|
435
|
+
if response is None or 200 <= response.code < 300:
|
|
436
|
+
continue
|
|
437
|
+
self._warn("logs_http_%d" % response.code,
|
|
438
|
+
"Dev Bench refused log lines: HTTP %d: %s" % (response.code, self._explain(response)))
|
|
439
|
+
except Exception:
|
|
440
|
+
pass
|
|
441
|
+
|
|
442
|
+
def _egress_line(self, line: str, policy: Optional[EgressPolicy]) -> str:
|
|
443
|
+
"""A held line is "[<trace>] <LEVEL> <message>". The bracketed trace
|
|
444
|
+
is our own framing, kept as is; the rest is redacted exactly as the
|
|
445
|
+
sidecar redacts a line's body."""
|
|
446
|
+
own = _OWN_TRACE.match(line)
|
|
447
|
+
if own is None:
|
|
448
|
+
out = scrub.egress(line, policy, self.service)
|
|
449
|
+
else:
|
|
450
|
+
redacted = scrub.egress(line[own.end():], policy, self.service)
|
|
451
|
+
framed = "[%s] " % own.group(1)
|
|
452
|
+
out = redacted if redacted.startswith(framed) else framed + redacted
|
|
453
|
+
data = out.encode("utf-8", "replace")
|
|
454
|
+
if len(data) > logs.MAX_LINE_BYTES:
|
|
455
|
+
out = data[: logs.MAX_LINE_BYTES].decode("utf-8", "ignore")
|
|
456
|
+
return out
|
|
457
|
+
|
|
458
|
+
def _upload(self, ask: dict, deadline: float) -> None:
|
|
459
|
+
"""Phase 2: the sidecar's bundle shape, PUT to the presigned URL with
|
|
460
|
+
no key (it carries its own authorization)."""
|
|
461
|
+
try:
|
|
462
|
+
fp = ask.get("fp")
|
|
463
|
+
url = ask.get("url")
|
|
464
|
+
if not isinstance(fp, str) or not isinstance(url, str) or not url:
|
|
465
|
+
return
|
|
466
|
+
expires = ask.get("expires")
|
|
467
|
+
if isinstance(expires, (int, float)) and 0 < expires <= time.time():
|
|
468
|
+
return
|
|
469
|
+
with self._lock:
|
|
470
|
+
detail = self._details.get(fp)
|
|
471
|
+
if detail is None:
|
|
472
|
+
return
|
|
473
|
+
policy = self.policy
|
|
474
|
+
bundle: Dict[str, Any] = {
|
|
475
|
+
"v": PROTOCOL_VERSION, "fp": fp, "kind": detail.kind,
|
|
476
|
+
"template": scrub.template_text(detail.text, policy, fp), "service": self.service,
|
|
477
|
+
"exemplars": [scrub.egress(detail.exemplar, policy, self.service)] if detail.exemplar else [],
|
|
478
|
+
}
|
|
479
|
+
if detail.frames:
|
|
480
|
+
bundle["frames"] = detail.frames
|
|
481
|
+
self._request("PUT", url, _json(bundle), {"content-type": "application/json"}, deadline)
|
|
482
|
+
except Exception:
|
|
483
|
+
pass
|
|
484
|
+
|
|
485
|
+
# ---- HTTP
|
|
486
|
+
|
|
487
|
+
def _request(self, method: str, url: str, body: bytes, headers: Dict[str, str], deadline: float) -> Optional[_Response]:
|
|
488
|
+
"""One try, then one retry with jitter on a network error or 5xx,
|
|
489
|
+
inside the deadline. A 4xx is final. Returns the response or None."""
|
|
490
|
+
for attempt in range(2):
|
|
491
|
+
remaining = deadline - time.monotonic()
|
|
492
|
+
if remaining < 0.05:
|
|
493
|
+
return None
|
|
494
|
+
response, error = self._attempt(method, url, body, headers, min(HTTP_TIMEOUT, remaining))
|
|
495
|
+
if error is None and response is not None and response.code < 500:
|
|
496
|
+
return response
|
|
497
|
+
if error is not None:
|
|
498
|
+
self._warn("unreachable", "could not reach Dev Bench at %s (%s); counts for that window were dropped"
|
|
499
|
+
% (self.dsn, type(error).__name__))
|
|
500
|
+
jitter = 0.1 + random.random() * 0.4
|
|
501
|
+
if attempt == 0 and deadline - time.monotonic() > jitter + 0.05:
|
|
502
|
+
time.sleep(jitter)
|
|
503
|
+
return None
|
|
504
|
+
|
|
505
|
+
def _attempt(self, method: str, url: str, body: bytes, headers: Dict[str, str],
|
|
506
|
+
timeout: float) -> Tuple[Optional[_Response], Optional[BaseException]]:
|
|
507
|
+
request = urllib.request.Request(url, data=body, headers=headers, method=method)
|
|
508
|
+
try:
|
|
509
|
+
with _OPENER.open(request, timeout=timeout) as resp:
|
|
510
|
+
return _Response(resp.status, resp.read(MAX_RESPONSE_BYTES + 1), str(resp.reason or "")), None
|
|
511
|
+
except urllib.error.HTTPError as e:
|
|
512
|
+
try:
|
|
513
|
+
data = e.read(MAX_RESPONSE_BYTES + 1)
|
|
514
|
+
except Exception:
|
|
515
|
+
data = b""
|
|
516
|
+
finally:
|
|
517
|
+
e.close()
|
|
518
|
+
return _Response(e.code, data, str(e.reason or "")), None
|
|
519
|
+
except Exception as e: # network errors, timeouts, DNS
|
|
520
|
+
return None, e
|
|
521
|
+
|
|
522
|
+
def _explain(self, response: _Response) -> str:
|
|
523
|
+
detail = None
|
|
524
|
+
try:
|
|
525
|
+
parsed = json.loads(response.body.decode("utf-8", "replace"))
|
|
526
|
+
detail = parsed.get("error") if isinstance(parsed, dict) else None
|
|
527
|
+
except ValueError:
|
|
528
|
+
pass
|
|
529
|
+
if response.code == 401:
|
|
530
|
+
return "the key in DEVBENCH_DSN was rejected (wrong, revoked, or for another environment)"
|
|
531
|
+
if response.code == 429:
|
|
532
|
+
return "rate limited; counts for that window were dropped"
|
|
533
|
+
if isinstance(detail, str):
|
|
534
|
+
return detail
|
|
535
|
+
return response.reason or "unexpected response"
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
# ---- Helpers
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
542
|
+
# The key header must never follow a redirect to another host.
|
|
543
|
+
def redirect_request(self, *args: Any, **kwargs: Any) -> None: # type: ignore[override]
|
|
544
|
+
return None
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
# The default handlers stay (a customer behind an HTTPS proxy configures it
|
|
548
|
+
# in the environment, as for any client); only redirects are refused.
|
|
549
|
+
_OPENER = urllib.request.build_opener(_NoRedirect)
|
|
550
|
+
|
|
551
|
+
_OWN_TRACE = re.compile(r"\[(v1/[A-Za-z0-9_-]{1,64}/[A-Za-z0-9_-]{1,64}/[0-9]{1,3})\] ")
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def _json(value: Any) -> bytes:
|
|
555
|
+
return json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode("utf-8", "replace")
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def _text(value: Any) -> str:
|
|
559
|
+
return "" if value is None else fingerprint.utf8(value)
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _user(raw: Any) -> Optional[dict]:
|
|
563
|
+
"""Identity travels only in the count's users list, never in a
|
|
564
|
+
fingerprint, template or bundle."""
|
|
565
|
+
if not isinstance(raw, dict):
|
|
566
|
+
return None
|
|
567
|
+
email = fingerprint.trim(_text(raw.get("email"))).lower()[:MAX_IDENTITY]
|
|
568
|
+
account = fingerprint.trim(_text(raw.get("account")))[:MAX_IDENTITY]
|
|
569
|
+
out = {}
|
|
570
|
+
if email:
|
|
571
|
+
out["email"] = email
|
|
572
|
+
if account:
|
|
573
|
+
out["account"] = account
|
|
574
|
+
return out or None
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def _exemplar(payload: dict) -> str:
|
|
578
|
+
"""The sidecar's exemplar line: ``Error: message [context, handled]
|
|
579
|
+
reason=...``. Raw here; templated and scrubbed when it leaves."""
|
|
580
|
+
line = _text(payload.get("error"))
|
|
581
|
+
message = _text(payload.get("message"))[:MAX_MESSAGE]
|
|
582
|
+
if message:
|
|
583
|
+
line += ": " + message.replace("\n", " ")
|
|
584
|
+
ctx = _text(payload.get("context"))
|
|
585
|
+
if ctx:
|
|
586
|
+
line += " [" + ctx
|
|
587
|
+
if payload.get("kind") == "exception" and payload.get("handled"):
|
|
588
|
+
line += ", handled"
|
|
589
|
+
line += "]"
|
|
590
|
+
reason = _text(payload.get("reason"))
|
|
591
|
+
if reason:
|
|
592
|
+
line += " reason=" + reason
|
|
593
|
+
return line
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _log_key(ask: Any) -> Optional[str]:
|
|
597
|
+
key = ask.get("trace") if isinstance(ask, dict) else None
|
|
598
|
+
if isinstance(key, str) and key and len(key.encode("utf-8", "replace")) <= 256:
|
|
599
|
+
return key
|
|
600
|
+
return None
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def _bound_slice(lines: List[str]) -> List[str]:
|
|
604
|
+
"""The most recent lines that fit the store's per-slice byte bound."""
|
|
605
|
+
total = 0
|
|
606
|
+
kept: List[str] = []
|
|
607
|
+
for line in reversed(lines):
|
|
608
|
+
total += len(line.encode("utf-8", "replace"))
|
|
609
|
+
if total > MAX_SLICE_BYTES:
|
|
610
|
+
break
|
|
611
|
+
kept.append(line)
|
|
612
|
+
kept.reverse()
|
|
613
|
+
return kept
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def _slice_bodies(slices: List[dict]) -> List[bytes]:
|
|
617
|
+
"""At most MAX_SLICES_PER_POST slices and MAX_BODY_BYTES per request."""
|
|
618
|
+
out: List[bytes] = []
|
|
619
|
+
current: List[dict] = []
|
|
620
|
+
size = 0
|
|
621
|
+
budget = MAX_BODY_BYTES - len(_json({"slices": []}))
|
|
622
|
+
for s in slices:
|
|
623
|
+
size_of = len(_json(s)) + 1
|
|
624
|
+
if current and (len(current) >= MAX_SLICES_PER_POST or size + size_of > budget):
|
|
625
|
+
out.append(_json({"slices": current}))
|
|
626
|
+
current = []
|
|
627
|
+
size = 0
|
|
628
|
+
current.append(s)
|
|
629
|
+
size += size_of
|
|
630
|
+
if current:
|
|
631
|
+
out.append(_json({"slices": current}))
|
|
632
|
+
return out
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _describe(error: Optional[BaseException]) -> str:
|
|
636
|
+
if error is None:
|
|
637
|
+
return "no response"
|
|
638
|
+
reason = getattr(error, "reason", None)
|
|
639
|
+
return str(reason if reason is not None else error)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Django: add one line to settings.py::
|
|
2
|
+
|
|
3
|
+
INSTALLED_APPS = [
|
|
4
|
+
"devbench.django",
|
|
5
|
+
...
|
|
6
|
+
]
|
|
7
|
+
|
|
8
|
+
At startup (``AppConfig.ready``) this puts DevbenchMiddleware first in
|
|
9
|
+
MIDDLEWARE (unless you listed it yourself), subscribes to
|
|
10
|
+
``got_request_exception``, starts log capture, and, when Celery is loaded,
|
|
11
|
+
installs the Celery hooks. ``python manage.py devbench_test`` checks the DSN.
|
|
12
|
+
"""
|