logsetu 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- logsetu/__init__.py +41 -0
- logsetu/apps.py +6 -0
- logsetu/asgi.py +151 -0
- logsetu/client.py +250 -0
- logsetu/conf.py +41 -0
- logsetu/context.py +36 -0
- logsetu/fastapi.py +11 -0
- logsetu/flask.py +141 -0
- logsetu/handler.py +191 -0
- logsetu/middleware.py +186 -0
- logsetu/py.typed +0 -0
- logsetu/web.py +71 -0
- logsetu-0.3.0.dist-info/METADATA +226 -0
- logsetu-0.3.0.dist-info/RECORD +16 -0
- logsetu-0.3.0.dist-info/WHEEL +4 -0
- logsetu-0.3.0.dist-info/licenses/LICENSE +21 -0
logsetu/__init__.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""LogSetu SDK for Python: Django & DRF, Flask (``logsetu.flask``) and FastAPI / ASGI (``logsetu.fastapi``).
|
|
2
|
+
|
|
3
|
+
Quick start (settings.py)::
|
|
4
|
+
|
|
5
|
+
LOGGING = {
|
|
6
|
+
"version": 1,
|
|
7
|
+
"handlers": {
|
|
8
|
+
"logsetu": {
|
|
9
|
+
"class": "logsetu.handler.LogSetuHandler",
|
|
10
|
+
"api_key": env("LOGSETU_API_KEY"),
|
|
11
|
+
"endpoint": env("LOGSETU_ENDPOINT"),
|
|
12
|
+
"environment": "production",
|
|
13
|
+
"source": "django-backend",
|
|
14
|
+
"level": "WARNING",
|
|
15
|
+
},
|
|
16
|
+
},
|
|
17
|
+
"root": {"handlers": ["logsetu"], "level": "INFO"},
|
|
18
|
+
}
|
|
19
|
+
MIDDLEWARE = [..., "logsetu.middleware.LogSetuMiddleware"]
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from .client import LogSetuClient, get_client, __version__
|
|
23
|
+
from .context import get_request_context, update_request_context
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"LogSetuClient",
|
|
27
|
+
"get_client",
|
|
28
|
+
"get_request_context",
|
|
29
|
+
"update_request_context",
|
|
30
|
+
"capture_exception",
|
|
31
|
+
"__version__",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
default_app_config = "logsetu.apps.LogSetuConfig"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def capture_exception(exc: BaseException, message: str = "", **meta: object) -> None:
|
|
38
|
+
"""Report an exception through the configured ``LogSetuHandler`` (via the stdlib logger)."""
|
|
39
|
+
import logging
|
|
40
|
+
|
|
41
|
+
logging.getLogger("logsetu.capture").error(message or f"{type(exc).__name__}: {exc}", exc_info=exc, extra=meta)
|
logsetu/apps.py
ADDED
logsetu/asgi.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""ASGI middleware for FastAPI, Starlette, Litestar, Quart and any other ASGI app.
|
|
2
|
+
|
|
3
|
+
::
|
|
4
|
+
|
|
5
|
+
from fastapi import FastAPI
|
|
6
|
+
from logsetu.asgi import LogSetuMiddleware
|
|
7
|
+
|
|
8
|
+
app = FastAPI()
|
|
9
|
+
app.add_middleware(
|
|
10
|
+
LogSetuMiddleware,
|
|
11
|
+
api_key=os.environ["LOGSETU_API_KEY"],
|
|
12
|
+
endpoint=os.environ["LOGSETU_ENDPOINT"],
|
|
13
|
+
source="fastapi",
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
Every log emitted while a request is handled (through stdlib ``logging``) carries the request id, method and path;
|
|
17
|
+
unhandled exceptions and 5xx responses are reported with redacted headers. No dependency on Starlette itself.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import logging
|
|
23
|
+
import os
|
|
24
|
+
import time
|
|
25
|
+
from typing import Any, Awaitable, Callable, Dict, Iterable, MutableMapping, Optional
|
|
26
|
+
|
|
27
|
+
from .client import LogSetuClient
|
|
28
|
+
from .conf import DEFAULT_REDACT_HEADERS, import_callable
|
|
29
|
+
from .context import get_request_context, reset_request_context, set_request_context
|
|
30
|
+
from .web import attach_handler, client_ip, exception_meta, make_client, mark_reported, redact, request_id
|
|
31
|
+
|
|
32
|
+
__all__ = ["LogSetuMiddleware"]
|
|
33
|
+
|
|
34
|
+
Scope = MutableMapping[str, Any]
|
|
35
|
+
Message = MutableMapping[str, Any]
|
|
36
|
+
Receive = Callable[[], Awaitable[Message]]
|
|
37
|
+
Send = Callable[[Message], Awaitable[None]]
|
|
38
|
+
ASGIApp = Callable[[Scope, Receive, Send], Awaitable[None]]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class LogSetuMiddleware:
|
|
42
|
+
"""Pure ASGI middleware (no ``BaseHTTPMiddleware``), so it doesn't buffer streaming responses."""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
app: ASGIApp,
|
|
47
|
+
*,
|
|
48
|
+
api_key: Optional[str] = None,
|
|
49
|
+
endpoint: Optional[str] = None,
|
|
50
|
+
environment: Optional[str] = None,
|
|
51
|
+
source: Optional[str] = None,
|
|
52
|
+
get_tenant_id: Any = None,
|
|
53
|
+
capture_logging: bool = True,
|
|
54
|
+
log_level: int = logging.WARNING,
|
|
55
|
+
capture_5xx: bool = True,
|
|
56
|
+
log_requests: bool = False,
|
|
57
|
+
ignore_paths: Iterable[str] = (),
|
|
58
|
+
redact_headers: Iterable[str] = DEFAULT_REDACT_HEADERS,
|
|
59
|
+
debug: bool = False,
|
|
60
|
+
client: Optional[LogSetuClient] = None,
|
|
61
|
+
) -> None:
|
|
62
|
+
self.app = app
|
|
63
|
+
self.environment = environment or os.environ.get("LOGSETU_ENVIRONMENT") or "production"
|
|
64
|
+
self.source = source or os.environ.get("LOGSETU_SOURCE") or "asgi"
|
|
65
|
+
self.client = make_client(
|
|
66
|
+
api_key or os.environ.get("LOGSETU_API_KEY"),
|
|
67
|
+
endpoint or os.environ.get("LOGSETU_ENDPOINT"),
|
|
68
|
+
environment=self.environment,
|
|
69
|
+
source=self.source,
|
|
70
|
+
debug=debug,
|
|
71
|
+
client=client,
|
|
72
|
+
)
|
|
73
|
+
if capture_logging:
|
|
74
|
+
attach_handler(self.client, log_level)
|
|
75
|
+
# Called with the ASGI scope; return a tenant id or None.
|
|
76
|
+
self.get_tenant_id: Optional[Callable[[Scope], Any]] = import_callable(get_tenant_id)
|
|
77
|
+
self.capture_5xx = capture_5xx
|
|
78
|
+
self.log_requests = log_requests
|
|
79
|
+
self.ignore_paths = frozenset(ignore_paths)
|
|
80
|
+
self.redact_headers = tuple(redact_headers)
|
|
81
|
+
|
|
82
|
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
83
|
+
if scope["type"] != "http" or scope.get("path") in self.ignore_paths:
|
|
84
|
+
await self.app(scope, receive, send)
|
|
85
|
+
return
|
|
86
|
+
|
|
87
|
+
headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope.get("headers") or []}
|
|
88
|
+
ctx: Dict[str, Any] = {"request_id": request_id(headers), "method": scope.get("method"), "path": scope.get("path")}
|
|
89
|
+
client_addr = scope.get("client")
|
|
90
|
+
ip = client_ip(headers, client_addr[0] if client_addr else None)
|
|
91
|
+
if ip:
|
|
92
|
+
ctx["ip"] = ip
|
|
93
|
+
if self.get_tenant_id is not None:
|
|
94
|
+
try:
|
|
95
|
+
tenant = self.get_tenant_id(scope)
|
|
96
|
+
if tenant is not None:
|
|
97
|
+
ctx["tenant_id"] = str(tenant)
|
|
98
|
+
except Exception:
|
|
99
|
+
pass
|
|
100
|
+
scope.setdefault("state", {})["logsetu_request_id"] = ctx["request_id"]
|
|
101
|
+
|
|
102
|
+
status = 0
|
|
103
|
+
started = time.monotonic()
|
|
104
|
+
|
|
105
|
+
async def send_wrapper(message: Message) -> None:
|
|
106
|
+
nonlocal status
|
|
107
|
+
if message["type"] == "http.response.start":
|
|
108
|
+
status = int(message["status"])
|
|
109
|
+
await send(message)
|
|
110
|
+
|
|
111
|
+
token = set_request_context(ctx)
|
|
112
|
+
try:
|
|
113
|
+
await self.app(scope, receive, send_wrapper)
|
|
114
|
+
except BaseException as exc:
|
|
115
|
+
if isinstance(exc, Exception):
|
|
116
|
+
self._report(scope, headers, "error", f"{type(exc).__name__}: {exc}",
|
|
117
|
+
{**exception_meta(exc), "status": status or 500, "duration_ms": _ms(started)})
|
|
118
|
+
mark_reported(exc)
|
|
119
|
+
raise
|
|
120
|
+
else:
|
|
121
|
+
if self.capture_5xx and status >= 500:
|
|
122
|
+
self._report(scope, headers, "error", f"HTTP {status} on {scope.get('method')} {scope.get('path')}",
|
|
123
|
+
{"status": status, "duration_ms": _ms(started)})
|
|
124
|
+
elif self.log_requests:
|
|
125
|
+
self._report(scope, headers, "warn" if status >= 400 else "info",
|
|
126
|
+
f"{scope.get('method')} {scope.get('path')} {status}",
|
|
127
|
+
{"status": status, "duration_ms": _ms(started)}, with_headers=False)
|
|
128
|
+
finally:
|
|
129
|
+
reset_request_context(token)
|
|
130
|
+
|
|
131
|
+
def _report(self, scope: Scope, headers: Dict[str, str], level: str, message: str, extra: Dict[str, Any],
|
|
132
|
+
*, with_headers: bool = True) -> None:
|
|
133
|
+
try:
|
|
134
|
+
info: Dict[str, Any] = {
|
|
135
|
+
"method": scope.get("method"),
|
|
136
|
+
"path": scope.get("path"),
|
|
137
|
+
"query": (scope.get("query_string") or b"").decode("latin-1")[:2000],
|
|
138
|
+
}
|
|
139
|
+
route = getattr(scope.get("route"), "path", None)
|
|
140
|
+
if route:
|
|
141
|
+
info["route"] = route
|
|
142
|
+
if with_headers:
|
|
143
|
+
info["headers"] = redact(headers.items(), self.redact_headers)
|
|
144
|
+
meta = {**get_request_context(), **extra, "request": info}
|
|
145
|
+
self.client.log(level, message, meta, source=self.source, environment=self.environment)
|
|
146
|
+
except Exception: # never break the response cycle
|
|
147
|
+
pass
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _ms(started: float) -> int:
|
|
151
|
+
return int((time.monotonic() - started) * 1000)
|
logsetu/client.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""Thread-safe, batching HTTP client for the LogSetu ingestion API.
|
|
2
|
+
|
|
3
|
+
Uses only the standard library so the package has zero runtime dependencies.
|
|
4
|
+
Never raises into the host application: transport failures are reported once on
|
|
5
|
+
stderr and the affected batch is dropped.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import atexit
|
|
11
|
+
import json
|
|
12
|
+
import queue
|
|
13
|
+
import random
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import urllib.error
|
|
18
|
+
import urllib.request
|
|
19
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
20
|
+
|
|
21
|
+
__version__ = "0.3.0"
|
|
22
|
+
|
|
23
|
+
LogDict = Dict[str, Any]
|
|
24
|
+
|
|
25
|
+
_MAX_BATCH = 500
|
|
26
|
+
_FLUSH: Any = object() # queue sentinel: send whatever is buffered now
|
|
27
|
+
_STOP: Any = object() # queue sentinel: send and exit
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def to_json_safe(value: Any, depth: int = 0) -> Any:
|
|
31
|
+
"""Coerce arbitrary values into something ``json.dumps`` accepts."""
|
|
32
|
+
if value is None or isinstance(value, (str, int, float, bool)):
|
|
33
|
+
return value
|
|
34
|
+
if depth > 6:
|
|
35
|
+
return repr(value)
|
|
36
|
+
if isinstance(value, dict):
|
|
37
|
+
return {str(k): to_json_safe(v, depth + 1) for k, v in list(value.items())[:200]}
|
|
38
|
+
if isinstance(value, (list, tuple, set, frozenset)):
|
|
39
|
+
return [to_json_safe(v, depth + 1) for v in list(value)[:200]]
|
|
40
|
+
if isinstance(value, bytes):
|
|
41
|
+
return value.decode("utf-8", errors="replace")[:2000]
|
|
42
|
+
if hasattr(value, "isoformat"):
|
|
43
|
+
try:
|
|
44
|
+
return value.isoformat()
|
|
45
|
+
except Exception: # pragma: no cover
|
|
46
|
+
pass
|
|
47
|
+
return repr(value)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class LogSetuClient:
|
|
51
|
+
"""Queues log dicts and ships them in batches from a background daemon thread."""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
api_key: str,
|
|
56
|
+
endpoint: str,
|
|
57
|
+
*,
|
|
58
|
+
environment: str = "production",
|
|
59
|
+
source: str = "django",
|
|
60
|
+
flush_interval: float = 2.0,
|
|
61
|
+
batch_size: int = 10,
|
|
62
|
+
max_queue_size: int = 1000,
|
|
63
|
+
max_retries: int = 3,
|
|
64
|
+
timeout: float = 5.0,
|
|
65
|
+
debug: bool = False,
|
|
66
|
+
enabled: bool = True,
|
|
67
|
+
) -> None:
|
|
68
|
+
self.api_key = api_key or ""
|
|
69
|
+
self.endpoint = (endpoint or "").rstrip("/")
|
|
70
|
+
self.ingest_url = f"{self.endpoint}/api/v1/ingest"
|
|
71
|
+
self.environment = environment
|
|
72
|
+
self.source = source
|
|
73
|
+
self.flush_interval = max(0.05, float(flush_interval))
|
|
74
|
+
self.batch_size = max(1, min(int(batch_size), _MAX_BATCH))
|
|
75
|
+
self.max_queue_size = max(1, int(max_queue_size))
|
|
76
|
+
self.max_retries = max(0, int(max_retries))
|
|
77
|
+
self.timeout = float(timeout)
|
|
78
|
+
self.debug = debug
|
|
79
|
+
self.enabled = bool(enabled and self.api_key and self.endpoint)
|
|
80
|
+
|
|
81
|
+
self._queue: "queue.Queue[Any]" = queue.Queue(maxsize=self.max_queue_size + 2)
|
|
82
|
+
self._dropped = 0
|
|
83
|
+
self._warned = False
|
|
84
|
+
self._lock = threading.Lock()
|
|
85
|
+
self._idle = threading.Event()
|
|
86
|
+
self._idle.set()
|
|
87
|
+
self._stopped = False
|
|
88
|
+
self._thread: Optional[threading.Thread] = None
|
|
89
|
+
|
|
90
|
+
if not self.enabled:
|
|
91
|
+
if enabled:
|
|
92
|
+
self._warn("api_key/endpoint not configured; logs will not be sent")
|
|
93
|
+
return
|
|
94
|
+
self._thread = threading.Thread(target=self._run, name="logsetu-worker", daemon=True)
|
|
95
|
+
self._thread.start()
|
|
96
|
+
atexit.register(self.close)
|
|
97
|
+
|
|
98
|
+
# ---------- public ----------
|
|
99
|
+
|
|
100
|
+
def log(
|
|
101
|
+
self,
|
|
102
|
+
level: str,
|
|
103
|
+
message: str,
|
|
104
|
+
meta: Optional[Dict[str, Any]] = None,
|
|
105
|
+
*,
|
|
106
|
+
timestamp: Optional[str] = None,
|
|
107
|
+
source: Optional[str] = None,
|
|
108
|
+
environment: Optional[str] = None,
|
|
109
|
+
) -> None:
|
|
110
|
+
if not self.enabled or self._stopped:
|
|
111
|
+
return
|
|
112
|
+
entry: LogDict = {
|
|
113
|
+
"level": level,
|
|
114
|
+
"message": str(message)[:10_000],
|
|
115
|
+
"source": source or self.source,
|
|
116
|
+
"environment": environment or self.environment,
|
|
117
|
+
}
|
|
118
|
+
if timestamp:
|
|
119
|
+
entry["timestamp"] = timestamp
|
|
120
|
+
if meta:
|
|
121
|
+
entry["meta"] = to_json_safe(meta)
|
|
122
|
+
self.enqueue(entry)
|
|
123
|
+
|
|
124
|
+
def enqueue(self, entry: LogDict) -> None:
|
|
125
|
+
if not self.enabled or self._stopped:
|
|
126
|
+
return
|
|
127
|
+
self._idle.clear()
|
|
128
|
+
with self._lock:
|
|
129
|
+
if self._queue.qsize() >= self.max_queue_size:
|
|
130
|
+
# Drop the oldest to make room; a log SDK must never block the request cycle.
|
|
131
|
+
try:
|
|
132
|
+
self._queue.get_nowait()
|
|
133
|
+
except queue.Empty: # pragma: no cover
|
|
134
|
+
pass
|
|
135
|
+
self._dropped += 1
|
|
136
|
+
try:
|
|
137
|
+
self._queue.put_nowait(entry)
|
|
138
|
+
except queue.Full: # pragma: no cover
|
|
139
|
+
self._dropped += 1
|
|
140
|
+
|
|
141
|
+
def flush(self, timeout: Optional[float] = 10.0) -> bool:
|
|
142
|
+
"""Block until everything currently queued has been sent (or *timeout* elapses)."""
|
|
143
|
+
if not self.enabled or self._stopped:
|
|
144
|
+
return True
|
|
145
|
+
self._idle.clear()
|
|
146
|
+
try:
|
|
147
|
+
self._queue.put_nowait(_FLUSH)
|
|
148
|
+
except queue.Full: # pragma: no cover
|
|
149
|
+
pass
|
|
150
|
+
return self._idle.wait(timeout)
|
|
151
|
+
|
|
152
|
+
def close(self) -> None:
|
|
153
|
+
if not self.enabled or self._stopped:
|
|
154
|
+
return
|
|
155
|
+
self._stopped = True
|
|
156
|
+
self._queue.put(_STOP)
|
|
157
|
+
if self._thread is not None:
|
|
158
|
+
self._thread.join(timeout=10.0)
|
|
159
|
+
|
|
160
|
+
@property
|
|
161
|
+
def dropped(self) -> int:
|
|
162
|
+
return self._dropped
|
|
163
|
+
|
|
164
|
+
# ---------- worker ----------
|
|
165
|
+
|
|
166
|
+
def _run(self) -> None:
|
|
167
|
+
batch: List[LogDict] = []
|
|
168
|
+
deadline = time.monotonic() + self.flush_interval
|
|
169
|
+
while True:
|
|
170
|
+
try:
|
|
171
|
+
item = self._queue.get(timeout=max(0.0, deadline - time.monotonic()))
|
|
172
|
+
except queue.Empty:
|
|
173
|
+
item = _FLUSH
|
|
174
|
+
if item is _STOP:
|
|
175
|
+
self._send(batch + self._drain())
|
|
176
|
+
self._idle.set()
|
|
177
|
+
return
|
|
178
|
+
if item is not _FLUSH:
|
|
179
|
+
batch.append(item)
|
|
180
|
+
if batch and (len(batch) >= self.batch_size or item is _FLUSH):
|
|
181
|
+
self._send(batch + self._drain(_MAX_BATCH - len(batch)))
|
|
182
|
+
batch = []
|
|
183
|
+
if item is _FLUSH or not batch:
|
|
184
|
+
deadline = time.monotonic() + self.flush_interval
|
|
185
|
+
if not batch and self._queue.empty():
|
|
186
|
+
self._idle.set()
|
|
187
|
+
|
|
188
|
+
def _drain(self, limit: int = _MAX_BATCH) -> List[LogDict]:
|
|
189
|
+
out: List[LogDict] = []
|
|
190
|
+
while len(out) < limit:
|
|
191
|
+
try:
|
|
192
|
+
more = self._queue.get_nowait()
|
|
193
|
+
except queue.Empty:
|
|
194
|
+
break
|
|
195
|
+
if more is _STOP:
|
|
196
|
+
self._queue.put_nowait(_STOP)
|
|
197
|
+
break
|
|
198
|
+
if more is not _FLUSH:
|
|
199
|
+
out.append(more)
|
|
200
|
+
return out
|
|
201
|
+
|
|
202
|
+
def _send(self, batch: List[LogDict]) -> None:
|
|
203
|
+
if not batch:
|
|
204
|
+
return
|
|
205
|
+
body = json.dumps(batch, default=str).encode("utf-8")
|
|
206
|
+
headers = {
|
|
207
|
+
"Content-Type": "application/json",
|
|
208
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
209
|
+
"User-Agent": f"logsetu-python/{__version__}",
|
|
210
|
+
}
|
|
211
|
+
last_error = "unknown error"
|
|
212
|
+
for attempt in range(self.max_retries + 1):
|
|
213
|
+
try:
|
|
214
|
+
req = urllib.request.Request(self.ingest_url, data=body, headers=headers, method="POST")
|
|
215
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as res: # noqa: S310 - user-configured URL
|
|
216
|
+
if 200 <= res.status < 300:
|
|
217
|
+
return
|
|
218
|
+
last_error = f"HTTP {res.status}"
|
|
219
|
+
except urllib.error.HTTPError as e:
|
|
220
|
+
if e.code < 500 and e.code != 429:
|
|
221
|
+
self._warn(f"server rejected batch (HTTP {e.code}); dropped {len(batch)} logs")
|
|
222
|
+
return
|
|
223
|
+
last_error = f"HTTP {e.code}"
|
|
224
|
+
except Exception as e: # network errors, timeouts, DNS
|
|
225
|
+
last_error = repr(e)
|
|
226
|
+
if attempt < self.max_retries:
|
|
227
|
+
time.sleep(min(30.0, 0.5 * (2**attempt)) + random.random() * 0.25)
|
|
228
|
+
self._warn(f"giving up after {self.max_retries + 1} attempts ({last_error}); dropped {len(batch)} logs")
|
|
229
|
+
|
|
230
|
+
def _warn(self, msg: str) -> None:
|
|
231
|
+
# Deliberately bypass the logging module: the LogSetu handler may be attached to the root logger.
|
|
232
|
+
if self.debug or not self._warned:
|
|
233
|
+
self._warned = True
|
|
234
|
+
suffix = "" if self.debug else " (further transport warnings suppressed; set debug=True to see them)"
|
|
235
|
+
sys.stderr.write(f"[logsetu] {msg}{suffix}\n")
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
_registry: Dict[Tuple[str, str], LogSetuClient] = {}
|
|
239
|
+
_registry_lock = threading.Lock()
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def get_client(api_key: str, endpoint: str, **kwargs: Any) -> LogSetuClient:
|
|
243
|
+
"""Return a shared client per (api_key, endpoint) so the handler and middleware use one queue."""
|
|
244
|
+
key = (api_key or "", (endpoint or "").rstrip("/"))
|
|
245
|
+
with _registry_lock:
|
|
246
|
+
client = _registry.get(key)
|
|
247
|
+
if client is None or client._stopped:
|
|
248
|
+
client = LogSetuClient(api_key, endpoint, **kwargs)
|
|
249
|
+
_registry[key] = client
|
|
250
|
+
return client
|
logsetu/conf.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Configuration helpers: resolve callables from dotted paths and read ``settings.LOGSETU``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
from typing import Any, Callable, Dict, Optional, Union
|
|
7
|
+
|
|
8
|
+
TenantGetter = Callable[[Any], Optional[str]]
|
|
9
|
+
|
|
10
|
+
DEFAULT_REDACT_HEADERS = ("authorization", "cookie", "set-cookie", "x-api-key", "x-csrftoken", "proxy-authorization")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def import_callable(value: Union[str, Callable[..., Any], None]) -> Optional[Callable[..., Any]]:
|
|
14
|
+
"""Accept a callable or a dotted path like ``"myapp.tenants.get_tenant_id"``."""
|
|
15
|
+
if value is None:
|
|
16
|
+
return None
|
|
17
|
+
if callable(value):
|
|
18
|
+
return value
|
|
19
|
+
if not isinstance(value, str):
|
|
20
|
+
raise TypeError(f"expected callable or dotted path, got {type(value)!r}")
|
|
21
|
+
module_path, _, attr = value.rpartition(".")
|
|
22
|
+
if not module_path:
|
|
23
|
+
raise ImportError(f"'{value}' is not a dotted path")
|
|
24
|
+
module = importlib.import_module(module_path)
|
|
25
|
+
try:
|
|
26
|
+
return getattr(module, attr)
|
|
27
|
+
except AttributeError as e:
|
|
28
|
+
raise ImportError(f"module '{module_path}' has no attribute '{attr}'") from e
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def django_settings() -> Dict[str, Any]:
|
|
32
|
+
"""``settings.LOGSETU`` if Django is configured, else ``{}``. Never raises."""
|
|
33
|
+
try:
|
|
34
|
+
from django.conf import settings
|
|
35
|
+
|
|
36
|
+
if not settings.configured:
|
|
37
|
+
return {}
|
|
38
|
+
value = getattr(settings, "LOGSETU", None)
|
|
39
|
+
return dict(value) if isinstance(value, dict) else {}
|
|
40
|
+
except Exception:
|
|
41
|
+
return {}
|
logsetu/context.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Per-request context (path, user, tenant, request id) shared between middleware and handler.
|
|
2
|
+
|
|
3
|
+
Backed by ``contextvars`` so it works for both sync (threaded) and async Django.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import contextvars
|
|
9
|
+
from typing import Any, Dict, Optional
|
|
10
|
+
|
|
11
|
+
_request_context: contextvars.ContextVar[Optional[Dict[str, Any]]] = contextvars.ContextVar(
|
|
12
|
+
"logsetu_request_context", default=None
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def set_request_context(ctx: Optional[Dict[str, Any]]) -> contextvars.Token:
|
|
17
|
+
return _request_context.set(ctx)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def reset_request_context(token: contextvars.Token) -> None:
|
|
21
|
+
try:
|
|
22
|
+
_request_context.reset(token)
|
|
23
|
+
except ValueError: # token from another context (async edge cases)
|
|
24
|
+
_request_context.set(None)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_request_context() -> Dict[str, Any]:
|
|
28
|
+
return dict(_request_context.get() or {})
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def update_request_context(**fields: Any) -> None:
|
|
32
|
+
"""Attach extra fields (e.g. tenant_id) to every log emitted during the current request."""
|
|
33
|
+
current = _request_context.get()
|
|
34
|
+
merged = dict(current or {})
|
|
35
|
+
merged.update(fields)
|
|
36
|
+
_request_context.set(merged)
|
logsetu/fastapi.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""FastAPI integration — an alias of :mod:`logsetu.asgi`.
|
|
2
|
+
|
|
3
|
+
::
|
|
4
|
+
|
|
5
|
+
from logsetu.fastapi import LogSetuMiddleware
|
|
6
|
+
app.add_middleware(LogSetuMiddleware, api_key=..., endpoint=..., source="fastapi")
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from .asgi import LogSetuMiddleware
|
|
10
|
+
|
|
11
|
+
__all__ = ["LogSetuMiddleware"]
|
logsetu/flask.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Flask integration: request context on every log, unhandled exceptions and 5xx responses reported.
|
|
2
|
+
|
|
3
|
+
::
|
|
4
|
+
|
|
5
|
+
from flask import Flask
|
|
6
|
+
from logsetu.flask import LogSetu
|
|
7
|
+
|
|
8
|
+
app = Flask(__name__)
|
|
9
|
+
LogSetu(app, api_key=os.environ["LOGSETU_API_KEY"], endpoint=os.environ["LOGSETU_ENDPOINT"], source="flask-api")
|
|
10
|
+
|
|
11
|
+
or with an app factory: ``logsetu = LogSetu()`` at module level, then ``logsetu.init_app(app)``. Options not passed
|
|
12
|
+
explicitly are read from ``app.config`` (``LOGSETU_API_KEY``, ``LOGSETU_ENDPOINT``, ``LOGSETU_ENVIRONMENT``,
|
|
13
|
+
``LOGSETU_SOURCE``).
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
from typing import Any, Callable, Iterable, Optional
|
|
20
|
+
|
|
21
|
+
from flask import Flask, g, got_request_exception, request
|
|
22
|
+
|
|
23
|
+
from .client import LogSetuClient
|
|
24
|
+
from .conf import DEFAULT_REDACT_HEADERS, import_callable
|
|
25
|
+
from .context import get_request_context, reset_request_context, set_request_context
|
|
26
|
+
from .web import attach_handler, client_ip, exception_meta, make_client, mark_reported, redact, request_id
|
|
27
|
+
|
|
28
|
+
__all__ = ["LogSetu"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class LogSetu:
|
|
32
|
+
"""Flask extension. ``capture_logging`` also forwards stdlib ``logging`` records at ``log_level`` and above."""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
app: Optional[Flask] = None,
|
|
37
|
+
*,
|
|
38
|
+
api_key: Optional[str] = None,
|
|
39
|
+
endpoint: Optional[str] = None,
|
|
40
|
+
environment: Optional[str] = None,
|
|
41
|
+
source: Optional[str] = None,
|
|
42
|
+
get_tenant_id: Any = None,
|
|
43
|
+
capture_logging: bool = True,
|
|
44
|
+
log_level: int = logging.WARNING,
|
|
45
|
+
capture_5xx: bool = True,
|
|
46
|
+
log_requests: bool = False,
|
|
47
|
+
redact_headers: Iterable[str] = DEFAULT_REDACT_HEADERS,
|
|
48
|
+
debug: bool = False,
|
|
49
|
+
client: Optional[LogSetuClient] = None,
|
|
50
|
+
) -> None:
|
|
51
|
+
self._options = dict(api_key=api_key, endpoint=endpoint, environment=environment, source=source)
|
|
52
|
+
self.get_tenant_id: Optional[Callable[[Any], Any]] = import_callable(get_tenant_id)
|
|
53
|
+
self.capture_logging = capture_logging
|
|
54
|
+
self.log_level = log_level
|
|
55
|
+
self.capture_5xx = capture_5xx
|
|
56
|
+
self.log_requests = log_requests
|
|
57
|
+
self.redact_headers = tuple(redact_headers)
|
|
58
|
+
self.debug = debug
|
|
59
|
+
self.client: Optional[LogSetuClient] = client
|
|
60
|
+
if app is not None:
|
|
61
|
+
self.init_app(app)
|
|
62
|
+
|
|
63
|
+
def init_app(self, app: Flask) -> None:
|
|
64
|
+
cfg = app.config
|
|
65
|
+
self.environment = self._options["environment"] or cfg.get("LOGSETU_ENVIRONMENT") or cfg.get("ENV") or "production"
|
|
66
|
+
self.source = self._options["source"] or cfg.get("LOGSETU_SOURCE") or "flask"
|
|
67
|
+
self.client = make_client(
|
|
68
|
+
self._options["api_key"] or cfg.get("LOGSETU_API_KEY"),
|
|
69
|
+
self._options["endpoint"] or cfg.get("LOGSETU_ENDPOINT"),
|
|
70
|
+
environment=self.environment,
|
|
71
|
+
source=self.source,
|
|
72
|
+
debug=self.debug,
|
|
73
|
+
client=self.client,
|
|
74
|
+
)
|
|
75
|
+
if self.capture_logging:
|
|
76
|
+
attach_handler(self.client, self.log_level)
|
|
77
|
+
|
|
78
|
+
app.before_request(self._before_request)
|
|
79
|
+
app.after_request(self._after_request)
|
|
80
|
+
app.teardown_request(self._teardown_request)
|
|
81
|
+
got_request_exception.connect(self._on_exception, app, weak=False)
|
|
82
|
+
app.extensions["logsetu"] = self
|
|
83
|
+
|
|
84
|
+
# ---------- request lifecycle ----------
|
|
85
|
+
|
|
86
|
+
def _before_request(self) -> None:
|
|
87
|
+
headers = {k.lower(): v for k, v in request.headers.items()}
|
|
88
|
+
ctx = {"request_id": request_id(headers), "method": request.method, "path": request.path}
|
|
89
|
+
ip = client_ip(headers, request.remote_addr)
|
|
90
|
+
if ip:
|
|
91
|
+
ctx["ip"] = ip
|
|
92
|
+
if self.get_tenant_id is not None:
|
|
93
|
+
try:
|
|
94
|
+
tenant = self.get_tenant_id(request)
|
|
95
|
+
if tenant is not None:
|
|
96
|
+
ctx["tenant_id"] = str(tenant)
|
|
97
|
+
except Exception:
|
|
98
|
+
pass
|
|
99
|
+
g._logsetu_token = set_request_context(ctx)
|
|
100
|
+
g.logsetu_request_id = ctx["request_id"]
|
|
101
|
+
|
|
102
|
+
def _after_request(self, response: Any) -> Any:
|
|
103
|
+
try:
|
|
104
|
+
status = response.status_code
|
|
105
|
+
if self.capture_5xx and status >= 500 and not g.get("_logsetu_reported"):
|
|
106
|
+
self._report("error", f"HTTP {status} on {request.method} {request.path}", {"status": status})
|
|
107
|
+
elif self.log_requests:
|
|
108
|
+
level = "warn" if status >= 400 else "info"
|
|
109
|
+
self._report(level, f"{request.method} {request.path} {status}", {"status": status}, with_headers=False)
|
|
110
|
+
except Exception:
|
|
111
|
+
pass
|
|
112
|
+
return response
|
|
113
|
+
|
|
114
|
+
def _teardown_request(self, _exc: Optional[BaseException]) -> None:
|
|
115
|
+
token = g.pop("_logsetu_token", None)
|
|
116
|
+
if token is not None:
|
|
117
|
+
reset_request_context(token)
|
|
118
|
+
|
|
119
|
+
def _on_exception(self, _sender: Any, exception: BaseException, **_: Any) -> None:
|
|
120
|
+
self._report("error", f"{type(exception).__name__}: {exception}", exception_meta(exception))
|
|
121
|
+
mark_reported(exception)
|
|
122
|
+
g._logsetu_reported = True
|
|
123
|
+
|
|
124
|
+
# ---------- helpers ----------
|
|
125
|
+
|
|
126
|
+
def _report(self, level: str, message: str, extra: dict, *, with_headers: bool = True) -> None:
|
|
127
|
+
if self.client is None:
|
|
128
|
+
return
|
|
129
|
+
try:
|
|
130
|
+
info = {"method": request.method, "path": request.path, "query": request.query_string.decode("latin-1")[:2000]}
|
|
131
|
+
endpoint = request.endpoint
|
|
132
|
+
if endpoint:
|
|
133
|
+
info["endpoint"] = endpoint
|
|
134
|
+
if request.url_rule is not None:
|
|
135
|
+
info["route"] = request.url_rule.rule
|
|
136
|
+
if with_headers:
|
|
137
|
+
info["headers"] = redact(request.headers.items(), self.redact_headers)
|
|
138
|
+
meta = {**get_request_context(), **extra, "request": info}
|
|
139
|
+
self.client.log(level, message, meta, source=self.source, environment=self.environment)
|
|
140
|
+
except Exception: # never break the response cycle
|
|
141
|
+
pass
|