watchtower-observability 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- watchtower/__init__.py +8 -0
- watchtower/appsec.py +209 -0
- watchtower/buffer.py +71 -0
- watchtower/capture.py +163 -0
- watchtower/client.py +227 -0
- watchtower/errors.py +138 -0
- watchtower/iast.py +202 -0
- watchtower/integrations/__init__.py +1 -0
- watchtower/integrations/aws_lambda.py +24 -0
- watchtower/integrations/azure_functions.py +90 -0
- watchtower/integrations/celery.py +14 -0
- watchtower/integrations/datastores.py +335 -0
- watchtower/integrations/django.py +39 -0
- watchtower/integrations/fastapi.py +58 -0
- watchtower/integrations/flask.py +26 -0
- watchtower/integrations/genai.py +216 -0
- watchtower/integrations/genai_agents.py +166 -0
- watchtower/integrations/genai_frameworks.py +131 -0
- watchtower/integrations/genai_more.py +246 -0
- watchtower/integrations/http_clients.py +16 -0
- watchtower/integrations/kvstores.py +191 -0
- watchtower/integrations/sqlalchemy.py +4 -0
- watchtower/log_handler.py +22 -0
- watchtower/origin.py +35 -0
- watchtower/probes.py +404 -0
- watchtower/profiling.py +152 -0
- watchtower/pytest_plugin.py +274 -0
- watchtower/sampling.py +95 -0
- watchtower/signals.py +101 -0
- watchtower/sqlnorm.py +210 -0
- watchtower_observability-0.1.2.dist-info/METADATA +45 -0
- watchtower_observability-0.1.2.dist-info/RECORD +36 -0
- watchtower_observability-0.1.2.dist-info/WHEEL +5 -0
- watchtower_observability-0.1.2.dist-info/entry_points.txt +2 -0
- watchtower_observability-0.1.2.dist-info/licenses/LICENSE +202 -0
- watchtower_observability-0.1.2.dist-info/top_level.txt +1 -0
watchtower/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Watchtower: key header, endpoint default, offline buffer, redaction, prompt/score/evaluate helpers."""
|
|
2
|
+
from .client import Client, init, observe, get_prompt, score, evaluate, evaluate_resume, shutdown, WT_PREFIX
|
|
3
|
+
|
|
4
|
+
__all__ = ["Client", "init", "observe", "get_prompt", "score", "evaluate", "evaluate_resume", "shutdown", "WT_PREFIX"]
|
|
5
|
+
from .errors import capture_exception, install_hooks, install_asyncio # noqa: E402
|
|
6
|
+
from .log_handler import WatchtowerHandler # noqa: E402
|
|
7
|
+
|
|
8
|
+
__all__ += ["capture_exception", "install_hooks", "install_asyncio", "WatchtowerHandler"]
|
watchtower/appsec.py
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""App and API Protection in-app WAF (W52).
|
|
2
|
+
|
|
3
|
+
`start(endpoint, key, service)` polls the project's ruleset from the edge
|
|
4
|
+
(`GET /v1/appsec/ruleset`, an ingestion key with scope `probes:read`, ETag, every `poll_s`) in a
|
|
5
|
+
daemon thread: mode (`monitor` or `block`), the enabled pattern rules and the denylist.
|
|
6
|
+
`AppsecMiddleware(app)` (WSGI: Flask, Django) and `AppsecASGIMiddleware(app)` (FastAPI,
|
|
7
|
+
Starlette) inspect every request before the app: path, query, User-Agent and the other header
|
|
8
|
+
values, percent-decoded once, against the rules the worker runs over stored spans. A match is
|
|
9
|
+
recorded on the current span (`appsec.event`, `appsec.rule_id`, `appsec.category`,
|
|
10
|
+
`appsec.blocked`); in `block` mode, and for denylisted IPs or users in either mode, the request
|
|
11
|
+
gets 403 problem+json and the app never runs. Install the middleware inside the OpenTelemetry
|
|
12
|
+
one so the server span is current. Rules are matched case-insensitively; one that Python cannot
|
|
13
|
+
compile is skipped. Nothing happens until `start` (or `set_ruleset`) is called.
|
|
14
|
+
"""
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
import urllib.parse
|
|
21
|
+
import urllib.request
|
|
22
|
+
|
|
23
|
+
from opentelemetry import trace
|
|
24
|
+
|
|
25
|
+
DENYLIST = "denylist"
|
|
26
|
+
_PROBLEM = json.dumps({"type": "about:blank", "title": "Forbidden", "status": 403,
|
|
27
|
+
"detail": "Request blocked by Watchtower App and API Protection"}).encode()
|
|
28
|
+
_state = {"mode": "monitor", "rules": [], "ip": {}, "user": {}}
|
|
29
|
+
_lock = threading.Lock()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _expiry(entries):
|
|
33
|
+
return {e["value"]: (e.get("expires_at") or float("inf")) for e in entries or [] if isinstance(e, dict) and e.get("value")}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def set_ruleset(r):
|
|
37
|
+
"""Replace the active ruleset (a dict as the edge serves it; None clears it)."""
|
|
38
|
+
r = r or {}
|
|
39
|
+
rules = []
|
|
40
|
+
for rule in r.get("rules") or []:
|
|
41
|
+
try:
|
|
42
|
+
rules.append((rule, re.compile(rule["pattern"], re.I)))
|
|
43
|
+
except (re.error, KeyError, TypeError):
|
|
44
|
+
continue # not portable to Python: the worker still evaluates it
|
|
45
|
+
deny = r.get("denylist") or {}
|
|
46
|
+
with _lock:
|
|
47
|
+
_state.update(mode="block" if r.get("mode") == "block" else "monitor", rules=rules,
|
|
48
|
+
ip=_expiry(deny.get("ip")), user=_expiry(deny.get("user")))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _listed(kind, value, now):
|
|
52
|
+
return bool(value) and _state[kind].get(value, 0) > now
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def is_user_blocked(user_id, now=None):
|
|
56
|
+
"""Whether `user_id` is denylisted (call after authentication to refuse a listed user)."""
|
|
57
|
+
return _listed("user", user_id, time.time() if now is None else now)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def active():
|
|
61
|
+
return bool(_state["rules"] or _state["ip"] or _state["user"])
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def inspect(req, now=None):
|
|
65
|
+
"""Judge one request: a dict with `path`, `query`, `user_agent`, `body`, `headers`, `ip`, `user`."""
|
|
66
|
+
now = time.time() if now is None else now
|
|
67
|
+
if _listed("ip", req.get("ip"), now) or _listed("user", req.get("user"), now):
|
|
68
|
+
return {"action": "block", "rule_id": DENYLIST, "category": "abuse"}
|
|
69
|
+
for rule, rx in _state["rules"]:
|
|
70
|
+
for f in rule.get("fields") or []:
|
|
71
|
+
v = req.get(f)
|
|
72
|
+
m = rx.search(v) if v else None
|
|
73
|
+
if m:
|
|
74
|
+
action = "block" if _state["mode"] == "block" else "monitor"
|
|
75
|
+
return {"action": action, "rule_id": rule.get("id", ""), "category": rule.get("category", ""),
|
|
76
|
+
"match": m.group(0)[:200]}
|
|
77
|
+
return {"action": "allow"}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _client_ip(headers, peer):
|
|
81
|
+
xff = headers.get("x-forwarded-for")
|
|
82
|
+
if xff:
|
|
83
|
+
return xff.split(",")[0].strip()
|
|
84
|
+
return headers.get("x-real-ip") or peer or ""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _request(raw_path, raw_query, headers, peer, user):
|
|
88
|
+
"""shortcut: trusts X-Forwarded-For for the client IP; run behind a proxy that overwrites it."""
|
|
89
|
+
return {
|
|
90
|
+
"path": urllib.parse.unquote(raw_path),
|
|
91
|
+
"query": urllib.parse.unquote_plus(raw_query),
|
|
92
|
+
"user_agent": headers.get("user-agent", ""),
|
|
93
|
+
"headers": "\n".join(v for k, v in headers.items() if k != "user-agent"),
|
|
94
|
+
"ip": _client_ip(headers, peer),
|
|
95
|
+
"user": user,
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def from_wsgi(environ, user=None):
|
|
100
|
+
headers = {k[5:].replace("_", "-").lower(): v for k, v in environ.items() if k.startswith("HTTP_")}
|
|
101
|
+
path = environ.get("RAW_URI") or environ.get("REQUEST_URI") or ""
|
|
102
|
+
path = path.split("?", 1)[0] or urllib.parse.quote(environ.get("PATH_INFO", "/").encode("latin-1", "replace"))
|
|
103
|
+
return _request(path, environ.get("QUERY_STRING", ""), headers, environ.get("REMOTE_ADDR"), user)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def from_asgi(scope, user=None):
|
|
107
|
+
headers = {}
|
|
108
|
+
for k, v in scope.get("headers") or []:
|
|
109
|
+
name = k.decode("latin-1").lower()
|
|
110
|
+
headers[name] = f"{headers[name]}, {v.decode('latin-1')}" if name in headers else v.decode("latin-1")
|
|
111
|
+
raw = scope.get("raw_path")
|
|
112
|
+
path = raw.decode("latin-1") if raw else urllib.parse.quote(scope.get("path", "/"))
|
|
113
|
+
peer = (scope.get("client") or ("",))[0]
|
|
114
|
+
return _request(path, (scope.get("query_string") or b"").decode("latin-1"), headers, peer, user)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def annotate(decision, ip=None):
|
|
118
|
+
span = trace.get_current_span()
|
|
119
|
+
if not span.is_recording():
|
|
120
|
+
return
|
|
121
|
+
if ip:
|
|
122
|
+
span.set_attribute("appsec.client_ip", ip)
|
|
123
|
+
if decision["action"] == "allow":
|
|
124
|
+
return
|
|
125
|
+
span.set_attributes({"appsec.event": True, "appsec.rule_id": decision.get("rule_id", ""),
|
|
126
|
+
"appsec.category": decision.get("category", ""),
|
|
127
|
+
"appsec.blocked": decision["action"] == "block"})
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _judge(req):
|
|
131
|
+
d = inspect(req)
|
|
132
|
+
annotate(d, req.get("ip"))
|
|
133
|
+
return d["action"] == "block"
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def track_login(login, success):
|
|
137
|
+
"""Mark the current request as a login attempt; failed ones feed the account-takeover rules."""
|
|
138
|
+
span = trace.get_current_span()
|
|
139
|
+
span.set_attributes({"appsec.login": login, "appsec.login.success": bool(success)})
|
|
140
|
+
if success:
|
|
141
|
+
span.set_attribute("appsec.auth", True)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class AppsecMiddleware:
|
|
145
|
+
"""WSGI middleware; `user(environ)` names the caller when known."""
|
|
146
|
+
|
|
147
|
+
def __init__(self, app, user=None):
|
|
148
|
+
self.app, self.user = app, user
|
|
149
|
+
|
|
150
|
+
def __call__(self, environ, start_response):
|
|
151
|
+
if active() and _judge(from_wsgi(environ, self.user(environ) if self.user else None)):
|
|
152
|
+
start_response("403 Forbidden", [("Content-Type", "application/problem+json"),
|
|
153
|
+
("Content-Length", str(len(_PROBLEM)))])
|
|
154
|
+
return [_PROBLEM]
|
|
155
|
+
return self.app(environ, start_response)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class AppsecASGIMiddleware:
|
|
159
|
+
"""ASGI middleware (HTTP scopes only); `user(scope)` names the caller when known."""
|
|
160
|
+
|
|
161
|
+
def __init__(self, app, user=None):
|
|
162
|
+
self.app, self.user = app, user
|
|
163
|
+
|
|
164
|
+
async def __call__(self, scope, receive, send):
|
|
165
|
+
if scope.get("type") == "http" and active() and _judge(from_asgi(scope, self.user(scope) if self.user else None)):
|
|
166
|
+
await send({"type": "http.response.start", "status": 403,
|
|
167
|
+
"headers": [(b"content-type", b"application/problem+json"),
|
|
168
|
+
(b"content-length", str(len(_PROBLEM)).encode())]})
|
|
169
|
+
await send({"type": "http.response.body", "body": _PROBLEM})
|
|
170
|
+
return
|
|
171
|
+
await self.app(scope, receive, send)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def fetch(endpoint, key, service, etag="", timeout_s=5):
|
|
175
|
+
"""One poll: (ruleset or None when unchanged or failed, etag)."""
|
|
176
|
+
q = urllib.parse.urlencode({"service": service})
|
|
177
|
+
headers = {"Authorization": f"Bearer {key}"}
|
|
178
|
+
if etag:
|
|
179
|
+
headers["If-None-Match"] = etag
|
|
180
|
+
req = urllib.request.Request(f"{endpoint}/v1/appsec/ruleset?{q}", headers=headers)
|
|
181
|
+
try:
|
|
182
|
+
with urllib.request.urlopen(req, timeout=timeout_s) as r: # noqa: S310 - endpoint is the operator's
|
|
183
|
+
return json.loads(r.read(4 << 20)), r.headers.get("ETag", "")
|
|
184
|
+
except (OSError, ValueError): # 304 arrives as HTTPError, an OSError
|
|
185
|
+
return None, etag
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def start(endpoint=None, key=None, service=None, environ=None):
|
|
189
|
+
"""Fetch the ruleset now and keep polling in a daemon thread; returns a `threading.Event` that stops it."""
|
|
190
|
+
environ = os.environ if environ is None else environ
|
|
191
|
+
base = (endpoint or environ.get("WT_API_URL", "https://api.watchtower.dev")).rstrip("/")
|
|
192
|
+
key = key or environ.get("WT_INGEST_KEY", "")
|
|
193
|
+
service = service or environ.get("OTEL_SERVICE_NAME", "unknown")
|
|
194
|
+
stop = threading.Event()
|
|
195
|
+
state = {"etag": "", "poll": 30}
|
|
196
|
+
|
|
197
|
+
def once():
|
|
198
|
+
body, state["etag"] = fetch(base, key, service, state["etag"])
|
|
199
|
+
if isinstance(body, dict):
|
|
200
|
+
set_ruleset(body)
|
|
201
|
+
state["poll"] = max(10, int(body.get("poll_s") or 30))
|
|
202
|
+
|
|
203
|
+
def loop():
|
|
204
|
+
while not stop.wait(state["poll"]):
|
|
205
|
+
once()
|
|
206
|
+
|
|
207
|
+
once()
|
|
208
|
+
threading.Thread(target=loop, name="watchtower-appsec", daemon=True).start()
|
|
209
|
+
return stop
|
watchtower/buffer.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Offline buffer: failed OTLP payloads are spooled to disk and replayed on the next export."""
|
|
2
|
+
import os
|
|
3
|
+
import urllib.request
|
|
4
|
+
import uuid
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class SpoolExporter:
|
|
8
|
+
"""Wraps an OTLP span, log or metric exporter; on failure the encoded batch goes to `directory`.
|
|
9
|
+
|
|
10
|
+
Other attributes (the metric reader reads the exporter's temporality preferences) pass through."""
|
|
11
|
+
|
|
12
|
+
def __init__(self, inner, directory, url, headers, encode=None, max_files=500):
|
|
13
|
+
self.inner, self.dir, self.url, self.headers = inner, directory, url, headers
|
|
14
|
+
self.max_files = max_files
|
|
15
|
+
if encode is None:
|
|
16
|
+
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
|
|
17
|
+
|
|
18
|
+
def encode(spans):
|
|
19
|
+
return encode_spans(spans).SerializeToString()
|
|
20
|
+
|
|
21
|
+
self.encode = encode
|
|
22
|
+
os.makedirs(directory, exist_ok=True)
|
|
23
|
+
|
|
24
|
+
def __getattr__(self, name):
|
|
25
|
+
if name == "inner": # not set yet; avoid recursing
|
|
26
|
+
raise AttributeError(name)
|
|
27
|
+
return getattr(self.inner, name)
|
|
28
|
+
|
|
29
|
+
def export(self, batch, *a, **kw):
|
|
30
|
+
self.replay()
|
|
31
|
+
try:
|
|
32
|
+
result = self.inner.export(batch, *a, **kw)
|
|
33
|
+
except Exception:
|
|
34
|
+
self._spool(self.encode(batch)) # some exporter versions raise on connection errors
|
|
35
|
+
raise
|
|
36
|
+
if getattr(result, "name", "SUCCESS") != "SUCCESS":
|
|
37
|
+
self._spool(self.encode(batch))
|
|
38
|
+
return result
|
|
39
|
+
|
|
40
|
+
def _files(self):
|
|
41
|
+
"""Batches oldest first; subdirectories (another signal's spool) are skipped."""
|
|
42
|
+
names = [n for n in os.listdir(self.dir) if os.path.isfile(os.path.join(self.dir, n))]
|
|
43
|
+
return sorted(names, key=lambda n: os.path.getmtime(os.path.join(self.dir, n)))
|
|
44
|
+
|
|
45
|
+
def _spool(self, payload):
|
|
46
|
+
files = self._files()
|
|
47
|
+
for old in files[: max(0, len(files) - self.max_files + 1)]: # drop-oldest
|
|
48
|
+
os.remove(os.path.join(self.dir, old))
|
|
49
|
+
with open(os.path.join(self.dir, f"{uuid.uuid4().hex}.pb"), "wb") as f:
|
|
50
|
+
f.write(payload)
|
|
51
|
+
|
|
52
|
+
def replay(self):
|
|
53
|
+
for name in self._files():
|
|
54
|
+
path = os.path.join(self.dir, name)
|
|
55
|
+
with open(path, "rb") as f:
|
|
56
|
+
body = f.read()
|
|
57
|
+
req = urllib.request.Request(
|
|
58
|
+
self.url, data=body, method="POST",
|
|
59
|
+
headers={**self.headers, "Content-Type": "application/x-protobuf"},
|
|
60
|
+
)
|
|
61
|
+
try:
|
|
62
|
+
urllib.request.urlopen(req, timeout=5).close()
|
|
63
|
+
except Exception: # noqa: BLE001
|
|
64
|
+
return # still offline; keep the rest in order
|
|
65
|
+
os.remove(path)
|
|
66
|
+
|
|
67
|
+
def shutdown(self, *a, **kw):
|
|
68
|
+
self.inner.shutdown(*a, **kw)
|
|
69
|
+
|
|
70
|
+
def force_flush(self, *a, **kw):
|
|
71
|
+
return self.inner.force_flush(*a, **kw)
|
watchtower/capture.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Payload and header capture rules (APM-016), set in Watchtower per service and route.
|
|
2
|
+
|
|
3
|
+
`apply(endpoint, key, service)` fetches `GET /v1/capture-rules?service=` (an ingestion key with
|
|
4
|
+
scope `probes:read`) once and turns the header lists into the OpenTelemetry Python settings the
|
|
5
|
+
HTTP server instrumentations read when they instrument an app
|
|
6
|
+
(`OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`/`_RESPONSE`), so it must run before
|
|
7
|
+
`instrument(app)`. A variable the user already set wins. Credential headers never come back from
|
|
8
|
+
the server. `init(capture_rules=True)` (or `WT_CAPTURE_RULES=1`) calls it.
|
|
9
|
+
|
|
10
|
+
Bodies and GraphQL: `record(span, route, ...)` keeps at most the matching rule's `body_bytes`
|
|
11
|
+
(hard cap 8 KiB) of text-like request and response bodies as `http.request.body` /
|
|
12
|
+
`http.response.body`, after the default secret and PII detectors (a port of the pipeline's
|
|
13
|
+
redact stage; ingest applies the project's detectors again), plus `graphql.operation.type`,
|
|
14
|
+
`graphql.operation.name` and (with `graphql_query`) `graphql.document`. The Flask, Django
|
|
15
|
+
and FastAPI integrations call it. shortcut: rules are fetched once at init; poll when live edits ask for it.
|
|
16
|
+
"""
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import urllib.parse
|
|
21
|
+
import urllib.request
|
|
22
|
+
|
|
23
|
+
REQ = "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST"
|
|
24
|
+
RESP = "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def env_for(rules):
|
|
28
|
+
"""The header settings for `rules` (a list from the server): the union of every rule's names."""
|
|
29
|
+
req, resp = [], []
|
|
30
|
+
for r in rules:
|
|
31
|
+
for src, dst in ((r.get("request_headers") or [], req), (r.get("response_headers") or [], resp)):
|
|
32
|
+
for h in src:
|
|
33
|
+
if isinstance(h, str) and h and h.lower() not in dst:
|
|
34
|
+
dst.append(h.lower())
|
|
35
|
+
out = {}
|
|
36
|
+
if req:
|
|
37
|
+
out[REQ] = ",".join(req)
|
|
38
|
+
if resp:
|
|
39
|
+
out[RESP] = ",".join(resp)
|
|
40
|
+
return out
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def apply(endpoint=None, key=None, service=None, timeout_s=3, environ=None):
|
|
44
|
+
"""Fetch the rules and set the variables; returns the rules (empty on any failure)."""
|
|
45
|
+
environ = os.environ if environ is None else environ
|
|
46
|
+
base = (endpoint or environ.get("WT_API_URL", "https://api.watchtower.dev")).rstrip("/")
|
|
47
|
+
q = urllib.parse.urlencode({"service": service or environ.get("OTEL_SERVICE_NAME", "unknown")})
|
|
48
|
+
req = urllib.request.Request(f"{base}/v1/capture-rules?{q}",
|
|
49
|
+
headers={"Authorization": f"Bearer {key or environ.get('WT_INGEST_KEY', '')}"})
|
|
50
|
+
try:
|
|
51
|
+
with urllib.request.urlopen(req, timeout=timeout_s) as r: # noqa: S310 - endpoint is the operator's
|
|
52
|
+
rules = json.loads(r.read(1 << 20)).get("rules", [])
|
|
53
|
+
except (OSError, ValueError):
|
|
54
|
+
return []
|
|
55
|
+
for k, v in env_for(rules).items():
|
|
56
|
+
environ.setdefault(k, v)
|
|
57
|
+
RULES[:] = [r for r in rules if isinstance(r, dict)]
|
|
58
|
+
return rules
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
HARD_MAX = 8192
|
|
62
|
+
RULES = []
|
|
63
|
+
_R = "[REDACTED]"
|
|
64
|
+
_SECRET_KEYS = r"password|passwd|secret|token|access_token|refresh_token|client_secret|api_?key|authorization"
|
|
65
|
+
_JSON_SECRET = re.compile(r'("(?:' + _SECRET_KEYS + r')"\s*:\s*)"(?:[^"\\]|\\.)*"', re.I)
|
|
66
|
+
_FORM_SECRET = re.compile(r"\b((?:" + _SECRET_KEYS + r")=)[^&\s]*", re.I)
|
|
67
|
+
_PATTERNS = [re.compile(p, f) for p, f in (
|
|
68
|
+
(r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----", 0),
|
|
69
|
+
(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", 0),
|
|
70
|
+
(r"bearer\s+[A-Za-z0-9._~+/-]+=*", re.I),
|
|
71
|
+
(r"wt[ik]_[a-z0-9]{8}_[A-Za-z0-9_-]{43}", 0),
|
|
72
|
+
(r"(?:AKIA|ASIA)[0-9A-Z]{16}", 0),
|
|
73
|
+
(r"gh[pousr]_[A-Za-z0-9]{36,}", 0),
|
|
74
|
+
(r"[rs]k_(?:live|test)_[A-Za-z0-9]{16,}", 0),
|
|
75
|
+
(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}", 0),
|
|
76
|
+
(r"\b\d{3}-\d{2}-\d{4}\b", 0),
|
|
77
|
+
)]
|
|
78
|
+
_CARD = re.compile(r"\b\d{4}[ -]?\d{4}[ -]?\d{4}[ -]?\d{1,7}\b")
|
|
79
|
+
_GQL = re.compile(r"(?:^|\})\s*(query|mutation|subscription)\b\s*([_A-Za-z][_0-9A-Za-z]*)?")
|
|
80
|
+
_TEXT = re.compile(r"json|^text/|xml|x-www-form-urlencoded|graphql", re.I)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _glob(pat, s):
|
|
84
|
+
return re.fullmatch(".*".join(re.escape(p) for p in pat.split("*")), s, re.S) is not None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def rule_for(route):
|
|
88
|
+
return next((r for r in RULES if not r.get("route") or _glob(r["route"], route)), None)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _luhn(s):
|
|
92
|
+
d = [int(c) for c in s if c.isdigit()]
|
|
93
|
+
total = sum(x if i % 2 == 0 else (x * 2 - 9 if x * 2 > 9 else x * 2) for i, x in enumerate(reversed(d)))
|
|
94
|
+
return len(d) >= 13 and total % 10 == 0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def redact_text(s):
|
|
98
|
+
"""Default secret and PII detectors: secret-named JSON and form fields, keys, tokens, emails, SSNs, Luhn-valid cards."""
|
|
99
|
+
s = _FORM_SECRET.sub(r"\1" + _R, _JSON_SECRET.sub(r'\1"' + _R + '"', s))
|
|
100
|
+
for p in _PATTERNS:
|
|
101
|
+
s = p.sub(_R, s)
|
|
102
|
+
return _CARD.sub(lambda m: _R if _luhn(m.group(0)) else m.group(0), s)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _cap(s, n):
|
|
106
|
+
return s.encode("utf-8")[:n].decode("utf-8", "ignore")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def excerpt(s, n):
|
|
110
|
+
"""Redact a wider window than kept so a secret cut at the edge is still masked, then cap."""
|
|
111
|
+
return _cap(redact_text(_cap(s, n * 2)), n)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def graphql_op(doc):
|
|
115
|
+
"""(type, name) of a GraphQL document, `{ ... }` being an anonymous query; None when it is not one."""
|
|
116
|
+
d = re.sub(r"#[^\n]*", "", doc)
|
|
117
|
+
m = _GQL.search(d)
|
|
118
|
+
if m:
|
|
119
|
+
return m.group(1), m.group(2)
|
|
120
|
+
return ("query", None) if d.lstrip().startswith("{") else None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def text_like(ctype):
|
|
124
|
+
return bool(ctype) and _TEXT.search(ctype) is not None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _text(body, ctype):
|
|
128
|
+
if body is None or (ctype and not _TEXT.search(ctype)):
|
|
129
|
+
return None
|
|
130
|
+
if isinstance(body, bytes):
|
|
131
|
+
return body[: HARD_MAX * 2].decode("utf-8", "ignore")
|
|
132
|
+
return body if isinstance(body, str) else None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def record(span, route, req_type=None, req_body=None, res_type=None, res_body=None):
|
|
136
|
+
"""Set the body and GraphQL attributes the rule matching `route` asks for; no rule, nothing."""
|
|
137
|
+
rule = rule_for(route or "")
|
|
138
|
+
if rule is None or span is None:
|
|
139
|
+
return
|
|
140
|
+
n = min(int(rule.get("body_bytes") or 0), HARD_MAX)
|
|
141
|
+
req, res = _text(req_body, req_type), _text(res_body, res_type)
|
|
142
|
+
if n and req:
|
|
143
|
+
span.set_attribute("http.request.body", excerpt(req, n))
|
|
144
|
+
if n and res:
|
|
145
|
+
span.set_attribute("http.response.body", excerpt(res, n))
|
|
146
|
+
doc, name = None, None
|
|
147
|
+
if req and "graphql" in (req_type or "").lower():
|
|
148
|
+
doc = req
|
|
149
|
+
elif req:
|
|
150
|
+
try:
|
|
151
|
+
j = json.loads(req)
|
|
152
|
+
except ValueError:
|
|
153
|
+
j = None
|
|
154
|
+
if isinstance(j, dict) and isinstance(j.get("query"), str):
|
|
155
|
+
doc, name = j["query"], j.get("operationName") if isinstance(j.get("operationName"), str) else None
|
|
156
|
+
op = graphql_op(doc) if doc else None
|
|
157
|
+
if op is None:
|
|
158
|
+
return
|
|
159
|
+
span.set_attribute("graphql.operation.type", op[0])
|
|
160
|
+
if name or op[1]:
|
|
161
|
+
span.set_attribute("graphql.operation.name", name or op[1])
|
|
162
|
+
if rule.get("graphql_query"):
|
|
163
|
+
span.set_attribute("graphql.document", excerpt(doc, HARD_MAX))
|
watchtower/client.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import urllib.request
|
|
6
|
+
|
|
7
|
+
from opentelemetry.sdk.resources import Resource
|
|
8
|
+
from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
|
|
9
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
10
|
+
|
|
11
|
+
from . import signals
|
|
12
|
+
|
|
13
|
+
WT_PREFIX = "wt."
|
|
14
|
+
DEFAULT_ENDPOINT = "http://localhost:4318"
|
|
15
|
+
_client = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _Redactor(SpanProcessor):
|
|
19
|
+
"""Runs the user's redaction hook over span attributes before export."""
|
|
20
|
+
|
|
21
|
+
def __init__(self, hook):
|
|
22
|
+
self.hook = hook
|
|
23
|
+
|
|
24
|
+
def on_end(self, span):
|
|
25
|
+
attrs = span._attributes # noqa: SLF001 - ReadableSpan has no setter
|
|
26
|
+
if attrs is None:
|
|
27
|
+
return
|
|
28
|
+
# The live mapping is immutable once the span ended, so swap in a redacted copy.
|
|
29
|
+
span._attributes = {k: self.hook(k, v) for k, v in attrs.items()} # noqa: SLF001
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def safe_resource(attrs):
|
|
33
|
+
"""Drop caller-supplied `wt.*` keys: tenant identity comes only from the key."""
|
|
34
|
+
return {k: v for k, v in (attrs or {}).items() if not k.startswith(WT_PREFIX)}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Client:
|
|
38
|
+
def __init__(self, api_key, endpoint=None, api_url=None, project=None, service_name=None,
|
|
39
|
+
resource=None, environment=None, release=None, redact=None, buffer_dir=None, exporter=None,
|
|
40
|
+
logs=None, metrics=None, remote_sampling=None, sampling_interval=None, code_origin=None):
|
|
41
|
+
if not api_key:
|
|
42
|
+
raise ValueError("api_key is required")
|
|
43
|
+
self.endpoint = (endpoint or os.environ.get("WT_OTLP_ENDPOINT") or DEFAULT_ENDPOINT).rstrip("/")
|
|
44
|
+
self.api_url = (api_url or os.environ.get("WT_API_URL") or self.endpoint).rstrip("/")
|
|
45
|
+
self.project = project or os.environ.get("WT_PROJECT_ID")
|
|
46
|
+
self.headers = {"Authorization": f"Bearer {api_key}"}
|
|
47
|
+
base = {"service.name": service_name or os.environ.get("WT_SERVICE") or "app"}
|
|
48
|
+
env, rel = environment or os.environ.get("WT_ENVIRONMENT"), release or os.environ.get("WT_RELEASE")
|
|
49
|
+
if env:
|
|
50
|
+
base["deployment.environment.name"] = env
|
|
51
|
+
if rel:
|
|
52
|
+
base["service.version"] = rel
|
|
53
|
+
res = Resource.create({**base, **safe_resource(resource)})
|
|
54
|
+
self.remote_sampler = None
|
|
55
|
+
sampler = None
|
|
56
|
+
if remote_sampling or (remote_sampling is None and os.environ.get("WT_REMOTE_SAMPLING") == "1"):
|
|
57
|
+
from opentelemetry.sdk.trace.sampling import ParentBased
|
|
58
|
+
|
|
59
|
+
from .sampling import RemoteSampler
|
|
60
|
+
|
|
61
|
+
self.remote_sampler = RemoteSampler(
|
|
62
|
+
self.endpoint, self.headers, base["service.name"], env,
|
|
63
|
+
sampling_interval or float(os.environ.get("WT_SAMPLING_INTERVAL_S") or 30))
|
|
64
|
+
sampler = ParentBased(self.remote_sampler)
|
|
65
|
+
self.provider = TracerProvider(resource=res, **({"sampler": sampler} if sampler else {}))
|
|
66
|
+
if code_origin or (code_origin is None and os.environ.get("WT_CODE_ORIGIN") == "1"):
|
|
67
|
+
from .origin import CodeOrigin
|
|
68
|
+
|
|
69
|
+
self.provider.add_span_processor(CodeOrigin())
|
|
70
|
+
if redact:
|
|
71
|
+
self.provider.add_span_processor(_Redactor(redact))
|
|
72
|
+
if exporter is None:
|
|
73
|
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
74
|
+
|
|
75
|
+
url = f"{self.endpoint}/v1/traces"
|
|
76
|
+
exporter = OTLPSpanExporter(endpoint=url, headers=self.headers)
|
|
77
|
+
if buffer_dir:
|
|
78
|
+
from .buffer import SpoolExporter
|
|
79
|
+
|
|
80
|
+
exporter = SpoolExporter(exporter, buffer_dir, url, self.headers)
|
|
81
|
+
self.exporter = exporter
|
|
82
|
+
self.provider.add_span_processor(BatchSpanProcessor(exporter))
|
|
83
|
+
self.tracer = self.provider.get_tracer("watchtower")
|
|
84
|
+
self.logger_provider = self.meter_provider = self._log_handler = None
|
|
85
|
+
if signals.enabled(logs, "WATCHTOWER_LOGS", "OTEL_LOGS_EXPORTER"):
|
|
86
|
+
self.logger_provider, self._log_handler = signals.start_logs(self.endpoint, self.headers, res, buffer_dir)
|
|
87
|
+
if signals.enabled(metrics, "WATCHTOWER_METRICS", "OTEL_METRICS_EXPORTER"):
|
|
88
|
+
self.meter_provider = signals.start_metrics(self.endpoint, self.headers, res, buffer_dir)
|
|
89
|
+
|
|
90
|
+
def _call(self, method, path, body=None):
|
|
91
|
+
url = f"{self.api_url}/v1/projects/{self.project}/llm/{path}"
|
|
92
|
+
data = None if body is None else json.dumps(body).encode()
|
|
93
|
+
req = urllib.request.Request(url, data=data, method=method,
|
|
94
|
+
headers={**self.headers, "Content-Type": "application/json"})
|
|
95
|
+
with urllib.request.urlopen(req, timeout=10) as r:
|
|
96
|
+
return json.load(r)
|
|
97
|
+
|
|
98
|
+
def get_prompt(self, prompt_id):
|
|
99
|
+
return self._call("GET", f"prompts/{prompt_id}")
|
|
100
|
+
|
|
101
|
+
def score(self, trace_id, value, name="user_feedback", span_id=None, comment=None):
|
|
102
|
+
body = {"trace_id": trace_id, "value": value, "name": name, "source": "sdk"}
|
|
103
|
+
if span_id:
|
|
104
|
+
body["span_id"] = span_id
|
|
105
|
+
if comment:
|
|
106
|
+
body["comment"] = comment
|
|
107
|
+
return self._call("POST", "scores", body)
|
|
108
|
+
|
|
109
|
+
def evaluate_resume(self, dataset, task, scorers, previous, **kw):
|
|
110
|
+
"""Re-run only the cases that errored or are missing in `previous` (rows of an earlier run)."""
|
|
111
|
+
return self.evaluate(dataset, task, scorers, resume=previous, **kw)
|
|
112
|
+
|
|
113
|
+
def evaluate(self, dataset, task, scorers, name="evaluate", error_tolerance=True, resume=None):
|
|
114
|
+
"""Run `task` per case under a span; each scorer result is recorded as a score (when a project is set).
|
|
115
|
+
|
|
116
|
+
`resume`: rows of an earlier run; cases that finished without errors are kept, not re-run."""
|
|
117
|
+
rows = []
|
|
118
|
+
done = {r["case"]: r for r in (resume or []) if not r.get("errors")}
|
|
119
|
+
for i, case in enumerate(dataset):
|
|
120
|
+
if i in done:
|
|
121
|
+
rows.append(done[i])
|
|
122
|
+
continue
|
|
123
|
+
with self.tracer.start_as_current_span(f"{name}.case") as span:
|
|
124
|
+
tid = format(span.get_span_context().trace_id, "032x")
|
|
125
|
+
row = {"case": i, "trace_id": tid, "scores": {}, "errors": {}}
|
|
126
|
+
rows.append(row)
|
|
127
|
+
try:
|
|
128
|
+
out = task(case["input"] if isinstance(case, dict) and "input" in case else case)
|
|
129
|
+
except Exception as e: # noqa: BLE001
|
|
130
|
+
if not error_tolerance:
|
|
131
|
+
raise
|
|
132
|
+
row["errors"]["task"] = str(e)
|
|
133
|
+
continue
|
|
134
|
+
for sname, fn in scorers.items():
|
|
135
|
+
try:
|
|
136
|
+
v = fn(case, out)
|
|
137
|
+
except Exception as e: # noqa: BLE001
|
|
138
|
+
if not error_tolerance:
|
|
139
|
+
raise
|
|
140
|
+
row["errors"][sname] = str(e)
|
|
141
|
+
continue
|
|
142
|
+
row["scores"][sname] = v
|
|
143
|
+
if self.project:
|
|
144
|
+
self.score(tid, v, name=sname)
|
|
145
|
+
return rows
|
|
146
|
+
|
|
147
|
+
def shutdown(self):
|
|
148
|
+
if self.remote_sampler is not None:
|
|
149
|
+
self.remote_sampler.stop()
|
|
150
|
+
self.provider.shutdown()
|
|
151
|
+
if self._log_handler is not None:
|
|
152
|
+
logging.getLogger().removeHandler(self._log_handler)
|
|
153
|
+
self._log_handler = None
|
|
154
|
+
for p in (self.logger_provider, self.meter_provider):
|
|
155
|
+
if p is not None:
|
|
156
|
+
p.shutdown()
|
|
157
|
+
self.logger_provider = self.meter_provider = None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
_profiler = None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def init(api_key=None, capture_errors=True, capture_rules=None, capture_locals=None, **kw):
|
|
164
|
+
global _client
|
|
165
|
+
_client = Client(api_key or os.environ.get("WT_INGEST_KEY"), **kw)
|
|
166
|
+
if capture_rules or (capture_rules is None and os.environ.get("WT_CAPTURE_RULES") == "1"):
|
|
167
|
+
from . import capture
|
|
168
|
+
|
|
169
|
+
capture.apply(_client.api_url, api_key or os.environ.get("WT_INGEST_KEY"), kw.get("service_name"))
|
|
170
|
+
if capture_errors:
|
|
171
|
+
from .errors import enable_locals, install_hooks
|
|
172
|
+
|
|
173
|
+
install_hooks()
|
|
174
|
+
# exception replay: opt-in here (or WT_EXCEPTION_LOCALS=1) and in the project's issue settings
|
|
175
|
+
enable_locals(capture_locals if capture_locals is not None else os.environ.get("WT_EXCEPTION_LOCALS") == "1")
|
|
176
|
+
if os.environ.get("WT_PROFILING") in ("1", "true"): # also set by the agent's lambda_instrument command
|
|
177
|
+
from . import profiling
|
|
178
|
+
|
|
179
|
+
global _profiler
|
|
180
|
+
if _profiler is not None:
|
|
181
|
+
_profiler.stop()
|
|
182
|
+
try:
|
|
183
|
+
_profiler = profiling.start(api_key=api_key, service_name=kw.get("service_name"))
|
|
184
|
+
except ValueError: # no key: profiling is best effort, never break init
|
|
185
|
+
_profiler = None
|
|
186
|
+
return _client
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _need():
|
|
190
|
+
if _client is None:
|
|
191
|
+
raise RuntimeError("watchtower.init() has not been called")
|
|
192
|
+
return _client
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def observe(name=None):
|
|
196
|
+
"""Decorator: wrap a function in a span named after it."""
|
|
197
|
+
def deco(fn):
|
|
198
|
+
@functools.wraps(fn)
|
|
199
|
+
def wrapper(*a, **kw):
|
|
200
|
+
with _need().tracer.start_as_current_span(name or fn.__qualname__):
|
|
201
|
+
return fn(*a, **kw)
|
|
202
|
+
return wrapper
|
|
203
|
+
return deco
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def get_prompt(prompt_id):
|
|
207
|
+
return _need().get_prompt(prompt_id)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def score(trace_id, value, **kw):
|
|
211
|
+
return _need().score(trace_id, value, **kw)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def evaluate(dataset, task, scorers, **kw):
|
|
215
|
+
return _need().evaluate(dataset, task, scorers, **kw)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def evaluate_resume(dataset, task, scorers, previous, **kw):
|
|
219
|
+
return _need().evaluate_resume(dataset, task, scorers, previous, **kw)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def shutdown():
|
|
223
|
+
from .errors import uninstall_hooks
|
|
224
|
+
|
|
225
|
+
uninstall_hooks()
|
|
226
|
+
if _client:
|
|
227
|
+
_client.shutdown()
|