watchtower-observability 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. watchtower/__init__.py +8 -0
  2. watchtower/appsec.py +209 -0
  3. watchtower/buffer.py +71 -0
  4. watchtower/capture.py +163 -0
  5. watchtower/client.py +227 -0
  6. watchtower/errors.py +138 -0
  7. watchtower/iast.py +202 -0
  8. watchtower/integrations/__init__.py +1 -0
  9. watchtower/integrations/aws_lambda.py +24 -0
  10. watchtower/integrations/azure_functions.py +90 -0
  11. watchtower/integrations/celery.py +14 -0
  12. watchtower/integrations/datastores.py +335 -0
  13. watchtower/integrations/django.py +39 -0
  14. watchtower/integrations/fastapi.py +58 -0
  15. watchtower/integrations/flask.py +26 -0
  16. watchtower/integrations/genai.py +216 -0
  17. watchtower/integrations/genai_agents.py +166 -0
  18. watchtower/integrations/genai_frameworks.py +131 -0
  19. watchtower/integrations/genai_more.py +246 -0
  20. watchtower/integrations/http_clients.py +16 -0
  21. watchtower/integrations/kvstores.py +191 -0
  22. watchtower/integrations/sqlalchemy.py +4 -0
  23. watchtower/log_handler.py +22 -0
  24. watchtower/origin.py +35 -0
  25. watchtower/probes.py +404 -0
  26. watchtower/profiling.py +152 -0
  27. watchtower/pytest_plugin.py +274 -0
  28. watchtower/sampling.py +95 -0
  29. watchtower/signals.py +101 -0
  30. watchtower/sqlnorm.py +210 -0
  31. watchtower_observability-0.1.2.dist-info/METADATA +45 -0
  32. watchtower_observability-0.1.2.dist-info/RECORD +36 -0
  33. watchtower_observability-0.1.2.dist-info/WHEEL +5 -0
  34. watchtower_observability-0.1.2.dist-info/entry_points.txt +2 -0
  35. watchtower_observability-0.1.2.dist-info/licenses/LICENSE +202 -0
  36. watchtower_observability-0.1.2.dist-info/top_level.txt +1 -0
watchtower/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """Watchtower: key header, endpoint default, offline buffer, redaction, prompt/score/evaluate helpers."""
2
+ from .client import Client, init, observe, get_prompt, score, evaluate, evaluate_resume, shutdown, WT_PREFIX
3
+
4
+ __all__ = ["Client", "init", "observe", "get_prompt", "score", "evaluate", "evaluate_resume", "shutdown", "WT_PREFIX"]
5
+ from .errors import capture_exception, install_hooks, install_asyncio # noqa: E402
6
+ from .log_handler import WatchtowerHandler # noqa: E402
7
+
8
+ __all__ += ["capture_exception", "install_hooks", "install_asyncio", "WatchtowerHandler"]
watchtower/appsec.py ADDED
@@ -0,0 +1,209 @@
1
+ """App and API Protection in-app WAF (W52).
2
+
3
+ `start(endpoint, key, service)` polls the project's ruleset from the edge
4
+ (`GET /v1/appsec/ruleset`, an ingestion key with scope `probes:read`, ETag, every `poll_s`) in a
5
+ daemon thread: mode (`monitor` or `block`), the enabled pattern rules and the denylist.
6
+ `AppsecMiddleware(app)` (WSGI: Flask, Django) and `AppsecASGIMiddleware(app)` (FastAPI,
7
+ Starlette) inspect every request before the app: path, query, User-Agent and the other header
8
+ values, percent-decoded once, against the rules the worker runs over stored spans. A match is
9
+ recorded on the current span (`appsec.event`, `appsec.rule_id`, `appsec.category`,
10
+ `appsec.blocked`); in `block` mode, and for denylisted IPs or users in either mode, the request
11
+ gets 403 problem+json and the app never runs. Install the middleware inside the OpenTelemetry
12
+ one so the server span is current. Rules are matched case-insensitively; one that Python cannot
13
+ compile is skipped. Nothing happens until `start` (or `set_ruleset`) is called.
14
+ """
15
+ import json
16
+ import os
17
+ import re
18
+ import threading
19
+ import time
20
+ import urllib.parse
21
+ import urllib.request
22
+
23
+ from opentelemetry import trace
24
+
25
+ DENYLIST = "denylist"
26
+ _PROBLEM = json.dumps({"type": "about:blank", "title": "Forbidden", "status": 403,
27
+ "detail": "Request blocked by Watchtower App and API Protection"}).encode()
28
+ _state = {"mode": "monitor", "rules": [], "ip": {}, "user": {}}
29
+ _lock = threading.Lock()
30
+
31
+
32
+ def _expiry(entries):
33
+ return {e["value"]: (e.get("expires_at") or float("inf")) for e in entries or [] if isinstance(e, dict) and e.get("value")}
34
+
35
+
36
+ def set_ruleset(r):
37
+ """Replace the active ruleset (a dict as the edge serves it; None clears it)."""
38
+ r = r or {}
39
+ rules = []
40
+ for rule in r.get("rules") or []:
41
+ try:
42
+ rules.append((rule, re.compile(rule["pattern"], re.I)))
43
+ except (re.error, KeyError, TypeError):
44
+ continue # not portable to Python: the worker still evaluates it
45
+ deny = r.get("denylist") or {}
46
+ with _lock:
47
+ _state.update(mode="block" if r.get("mode") == "block" else "monitor", rules=rules,
48
+ ip=_expiry(deny.get("ip")), user=_expiry(deny.get("user")))
49
+
50
+
51
+ def _listed(kind, value, now):
52
+ return bool(value) and _state[kind].get(value, 0) > now
53
+
54
+
55
+ def is_user_blocked(user_id, now=None):
56
+ """Whether `user_id` is denylisted (call after authentication to refuse a listed user)."""
57
+ return _listed("user", user_id, time.time() if now is None else now)
58
+
59
+
60
+ def active():
61
+ return bool(_state["rules"] or _state["ip"] or _state["user"])
62
+
63
+
64
+ def inspect(req, now=None):
65
+ """Judge one request: a dict with `path`, `query`, `user_agent`, `body`, `headers`, `ip`, `user`."""
66
+ now = time.time() if now is None else now
67
+ if _listed("ip", req.get("ip"), now) or _listed("user", req.get("user"), now):
68
+ return {"action": "block", "rule_id": DENYLIST, "category": "abuse"}
69
+ for rule, rx in _state["rules"]:
70
+ for f in rule.get("fields") or []:
71
+ v = req.get(f)
72
+ m = rx.search(v) if v else None
73
+ if m:
74
+ action = "block" if _state["mode"] == "block" else "monitor"
75
+ return {"action": action, "rule_id": rule.get("id", ""), "category": rule.get("category", ""),
76
+ "match": m.group(0)[:200]}
77
+ return {"action": "allow"}
78
+
79
+
80
+ def _client_ip(headers, peer):
81
+ xff = headers.get("x-forwarded-for")
82
+ if xff:
83
+ return xff.split(",")[0].strip()
84
+ return headers.get("x-real-ip") or peer or ""
85
+
86
+
87
+ def _request(raw_path, raw_query, headers, peer, user):
88
+ """shortcut: trusts X-Forwarded-For for the client IP; run behind a proxy that overwrites it."""
89
+ return {
90
+ "path": urllib.parse.unquote(raw_path),
91
+ "query": urllib.parse.unquote_plus(raw_query),
92
+ "user_agent": headers.get("user-agent", ""),
93
+ "headers": "\n".join(v for k, v in headers.items() if k != "user-agent"),
94
+ "ip": _client_ip(headers, peer),
95
+ "user": user,
96
+ }
97
+
98
+
99
+ def from_wsgi(environ, user=None):
100
+ headers = {k[5:].replace("_", "-").lower(): v for k, v in environ.items() if k.startswith("HTTP_")}
101
+ path = environ.get("RAW_URI") or environ.get("REQUEST_URI") or ""
102
+ path = path.split("?", 1)[0] or urllib.parse.quote(environ.get("PATH_INFO", "/").encode("latin-1", "replace"))
103
+ return _request(path, environ.get("QUERY_STRING", ""), headers, environ.get("REMOTE_ADDR"), user)
104
+
105
+
106
+ def from_asgi(scope, user=None):
107
+ headers = {}
108
+ for k, v in scope.get("headers") or []:
109
+ name = k.decode("latin-1").lower()
110
+ headers[name] = f"{headers[name]}, {v.decode('latin-1')}" if name in headers else v.decode("latin-1")
111
+ raw = scope.get("raw_path")
112
+ path = raw.decode("latin-1") if raw else urllib.parse.quote(scope.get("path", "/"))
113
+ peer = (scope.get("client") or ("",))[0]
114
+ return _request(path, (scope.get("query_string") or b"").decode("latin-1"), headers, peer, user)
115
+
116
+
117
+ def annotate(decision, ip=None):
118
+ span = trace.get_current_span()
119
+ if not span.is_recording():
120
+ return
121
+ if ip:
122
+ span.set_attribute("appsec.client_ip", ip)
123
+ if decision["action"] == "allow":
124
+ return
125
+ span.set_attributes({"appsec.event": True, "appsec.rule_id": decision.get("rule_id", ""),
126
+ "appsec.category": decision.get("category", ""),
127
+ "appsec.blocked": decision["action"] == "block"})
128
+
129
+
130
+ def _judge(req):
131
+ d = inspect(req)
132
+ annotate(d, req.get("ip"))
133
+ return d["action"] == "block"
134
+
135
+
136
+ def track_login(login, success):
137
+ """Mark the current request as a login attempt; failed ones feed the account-takeover rules."""
138
+ span = trace.get_current_span()
139
+ span.set_attributes({"appsec.login": login, "appsec.login.success": bool(success)})
140
+ if success:
141
+ span.set_attribute("appsec.auth", True)
142
+
143
+
144
+ class AppsecMiddleware:
145
+ """WSGI middleware; `user(environ)` names the caller when known."""
146
+
147
+ def __init__(self, app, user=None):
148
+ self.app, self.user = app, user
149
+
150
+ def __call__(self, environ, start_response):
151
+ if active() and _judge(from_wsgi(environ, self.user(environ) if self.user else None)):
152
+ start_response("403 Forbidden", [("Content-Type", "application/problem+json"),
153
+ ("Content-Length", str(len(_PROBLEM)))])
154
+ return [_PROBLEM]
155
+ return self.app(environ, start_response)
156
+
157
+
158
+ class AppsecASGIMiddleware:
159
+ """ASGI middleware (HTTP scopes only); `user(scope)` names the caller when known."""
160
+
161
+ def __init__(self, app, user=None):
162
+ self.app, self.user = app, user
163
+
164
+ async def __call__(self, scope, receive, send):
165
+ if scope.get("type") == "http" and active() and _judge(from_asgi(scope, self.user(scope) if self.user else None)):
166
+ await send({"type": "http.response.start", "status": 403,
167
+ "headers": [(b"content-type", b"application/problem+json"),
168
+ (b"content-length", str(len(_PROBLEM)).encode())]})
169
+ await send({"type": "http.response.body", "body": _PROBLEM})
170
+ return
171
+ await self.app(scope, receive, send)
172
+
173
+
174
+ def fetch(endpoint, key, service, etag="", timeout_s=5):
175
+ """One poll: (ruleset or None when unchanged or failed, etag)."""
176
+ q = urllib.parse.urlencode({"service": service})
177
+ headers = {"Authorization": f"Bearer {key}"}
178
+ if etag:
179
+ headers["If-None-Match"] = etag
180
+ req = urllib.request.Request(f"{endpoint}/v1/appsec/ruleset?{q}", headers=headers)
181
+ try:
182
+ with urllib.request.urlopen(req, timeout=timeout_s) as r: # noqa: S310 - endpoint is the operator's
183
+ return json.loads(r.read(4 << 20)), r.headers.get("ETag", "")
184
+ except (OSError, ValueError): # 304 arrives as HTTPError, an OSError
185
+ return None, etag
186
+
187
+
188
+ def start(endpoint=None, key=None, service=None, environ=None):
189
+ """Fetch the ruleset now and keep polling in a daemon thread; returns a `threading.Event` that stops it."""
190
+ environ = os.environ if environ is None else environ
191
+ base = (endpoint or environ.get("WT_API_URL", "https://api.watchtower.dev")).rstrip("/")
192
+ key = key or environ.get("WT_INGEST_KEY", "")
193
+ service = service or environ.get("OTEL_SERVICE_NAME", "unknown")
194
+ stop = threading.Event()
195
+ state = {"etag": "", "poll": 30}
196
+
197
+ def once():
198
+ body, state["etag"] = fetch(base, key, service, state["etag"])
199
+ if isinstance(body, dict):
200
+ set_ruleset(body)
201
+ state["poll"] = max(10, int(body.get("poll_s") or 30))
202
+
203
+ def loop():
204
+ while not stop.wait(state["poll"]):
205
+ once()
206
+
207
+ once()
208
+ threading.Thread(target=loop, name="watchtower-appsec", daemon=True).start()
209
+ return stop
watchtower/buffer.py ADDED
@@ -0,0 +1,71 @@
1
+ """Offline buffer: failed OTLP payloads are spooled to disk and replayed on the next export."""
2
+ import os
3
+ import urllib.request
4
+ import uuid
5
+
6
+
7
+ class SpoolExporter:
8
+ """Wraps an OTLP span, log or metric exporter; on failure the encoded batch goes to `directory`.
9
+
10
+ Other attributes (the metric reader reads the exporter's temporality preferences) pass through."""
11
+
12
+ def __init__(self, inner, directory, url, headers, encode=None, max_files=500):
13
+ self.inner, self.dir, self.url, self.headers = inner, directory, url, headers
14
+ self.max_files = max_files
15
+ if encode is None:
16
+ from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
17
+
18
+ def encode(spans):
19
+ return encode_spans(spans).SerializeToString()
20
+
21
+ self.encode = encode
22
+ os.makedirs(directory, exist_ok=True)
23
+
24
+ def __getattr__(self, name):
25
+ if name == "inner": # not set yet; avoid recursing
26
+ raise AttributeError(name)
27
+ return getattr(self.inner, name)
28
+
29
+ def export(self, batch, *a, **kw):
30
+ self.replay()
31
+ try:
32
+ result = self.inner.export(batch, *a, **kw)
33
+ except Exception:
34
+ self._spool(self.encode(batch)) # some exporter versions raise on connection errors
35
+ raise
36
+ if getattr(result, "name", "SUCCESS") != "SUCCESS":
37
+ self._spool(self.encode(batch))
38
+ return result
39
+
40
+ def _files(self):
41
+ """Batches oldest first; subdirectories (another signal's spool) are skipped."""
42
+ names = [n for n in os.listdir(self.dir) if os.path.isfile(os.path.join(self.dir, n))]
43
+ return sorted(names, key=lambda n: os.path.getmtime(os.path.join(self.dir, n)))
44
+
45
+ def _spool(self, payload):
46
+ files = self._files()
47
+ for old in files[: max(0, len(files) - self.max_files + 1)]: # drop-oldest
48
+ os.remove(os.path.join(self.dir, old))
49
+ with open(os.path.join(self.dir, f"{uuid.uuid4().hex}.pb"), "wb") as f:
50
+ f.write(payload)
51
+
52
+ def replay(self):
53
+ for name in self._files():
54
+ path = os.path.join(self.dir, name)
55
+ with open(path, "rb") as f:
56
+ body = f.read()
57
+ req = urllib.request.Request(
58
+ self.url, data=body, method="POST",
59
+ headers={**self.headers, "Content-Type": "application/x-protobuf"},
60
+ )
61
+ try:
62
+ urllib.request.urlopen(req, timeout=5).close()
63
+ except Exception: # noqa: BLE001
64
+ return # still offline; keep the rest in order
65
+ os.remove(path)
66
+
67
+ def shutdown(self, *a, **kw):
68
+ self.inner.shutdown(*a, **kw)
69
+
70
+ def force_flush(self, *a, **kw):
71
+ return self.inner.force_flush(*a, **kw)
watchtower/capture.py ADDED
@@ -0,0 +1,163 @@
1
+ """Payload and header capture rules (APM-016), set in Watchtower per service and route.
2
+
3
+ `apply(endpoint, key, service)` fetches `GET /v1/capture-rules?service=` (an ingestion key with
4
+ scope `probes:read`) once and turns the header lists into the OpenTelemetry Python settings the
5
+ HTTP server instrumentations read when they instrument an app
6
+ (`OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`/`_RESPONSE`), so it must run before
7
+ `instrument(app)`. A variable the user already set wins. Credential headers never come back from
8
+ the server. `init(capture_rules=True)` (or `WT_CAPTURE_RULES=1`) calls it.
9
+
10
+ Bodies and GraphQL: `record(span, route, ...)` keeps at most the matching rule's `body_bytes`
11
+ (hard cap 8 KiB) of text-like request and response bodies as `http.request.body` /
12
+ `http.response.body`, after the default secret and PII detectors (a port of the pipeline's
13
+ redact stage; ingest applies the project's detectors again), plus `graphql.operation.type`,
14
+ `graphql.operation.name` and (with `graphql_query`) `graphql.document`. The Flask, Django
15
+ and FastAPI integrations call it. shortcut: rules are fetched once at init; poll when live edits ask for it.
16
+ """
17
+ import json
18
+ import os
19
+ import re
20
+ import urllib.parse
21
+ import urllib.request
22
+
23
+ REQ = "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST"
24
+ RESP = "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE"
25
+
26
+
27
+ def env_for(rules):
28
+ """The header settings for `rules` (a list from the server): the union of every rule's names."""
29
+ req, resp = [], []
30
+ for r in rules:
31
+ for src, dst in ((r.get("request_headers") or [], req), (r.get("response_headers") or [], resp)):
32
+ for h in src:
33
+ if isinstance(h, str) and h and h.lower() not in dst:
34
+ dst.append(h.lower())
35
+ out = {}
36
+ if req:
37
+ out[REQ] = ",".join(req)
38
+ if resp:
39
+ out[RESP] = ",".join(resp)
40
+ return out
41
+
42
+
43
+ def apply(endpoint=None, key=None, service=None, timeout_s=3, environ=None):
44
+ """Fetch the rules and set the variables; returns the rules (empty on any failure)."""
45
+ environ = os.environ if environ is None else environ
46
+ base = (endpoint or environ.get("WT_API_URL", "https://api.watchtower.dev")).rstrip("/")
47
+ q = urllib.parse.urlencode({"service": service or environ.get("OTEL_SERVICE_NAME", "unknown")})
48
+ req = urllib.request.Request(f"{base}/v1/capture-rules?{q}",
49
+ headers={"Authorization": f"Bearer {key or environ.get('WT_INGEST_KEY', '')}"})
50
+ try:
51
+ with urllib.request.urlopen(req, timeout=timeout_s) as r: # noqa: S310 - endpoint is the operator's
52
+ rules = json.loads(r.read(1 << 20)).get("rules", [])
53
+ except (OSError, ValueError):
54
+ return []
55
+ for k, v in env_for(rules).items():
56
+ environ.setdefault(k, v)
57
+ RULES[:] = [r for r in rules if isinstance(r, dict)]
58
+ return rules
59
+
60
+
61
+ HARD_MAX = 8192
62
+ RULES = []
63
+ _R = "[REDACTED]"
64
+ _SECRET_KEYS = r"password|passwd|secret|token|access_token|refresh_token|client_secret|api_?key|authorization"
65
+ _JSON_SECRET = re.compile(r'("(?:' + _SECRET_KEYS + r')"\s*:\s*)"(?:[^"\\]|\\.)*"', re.I)
66
+ _FORM_SECRET = re.compile(r"\b((?:" + _SECRET_KEYS + r")=)[^&\s]*", re.I)
67
+ _PATTERNS = [re.compile(p, f) for p, f in (
68
+ (r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----", 0),
69
+ (r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", 0),
70
+ (r"bearer\s+[A-Za-z0-9._~+/-]+=*", re.I),
71
+ (r"wt[ik]_[a-z0-9]{8}_[A-Za-z0-9_-]{43}", 0),
72
+ (r"(?:AKIA|ASIA)[0-9A-Z]{16}", 0),
73
+ (r"gh[pousr]_[A-Za-z0-9]{36,}", 0),
74
+ (r"[rs]k_(?:live|test)_[A-Za-z0-9]{16,}", 0),
75
+ (r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}", 0),
76
+ (r"\b\d{3}-\d{2}-\d{4}\b", 0),
77
+ )]
78
+ _CARD = re.compile(r"\b\d{4}[ -]?\d{4}[ -]?\d{4}[ -]?\d{1,7}\b")
79
+ _GQL = re.compile(r"(?:^|\})\s*(query|mutation|subscription)\b\s*([_A-Za-z][_0-9A-Za-z]*)?")
80
+ _TEXT = re.compile(r"json|^text/|xml|x-www-form-urlencoded|graphql", re.I)
81
+
82
+
83
+ def _glob(pat, s):
84
+ return re.fullmatch(".*".join(re.escape(p) for p in pat.split("*")), s, re.S) is not None
85
+
86
+
87
+ def rule_for(route):
88
+ return next((r for r in RULES if not r.get("route") or _glob(r["route"], route)), None)
89
+
90
+
91
+ def _luhn(s):
92
+ d = [int(c) for c in s if c.isdigit()]
93
+ total = sum(x if i % 2 == 0 else (x * 2 - 9 if x * 2 > 9 else x * 2) for i, x in enumerate(reversed(d)))
94
+ return len(d) >= 13 and total % 10 == 0
95
+
96
+
97
+ def redact_text(s):
98
+ """Default secret and PII detectors: secret-named JSON and form fields, keys, tokens, emails, SSNs, Luhn-valid cards."""
99
+ s = _FORM_SECRET.sub(r"\1" + _R, _JSON_SECRET.sub(r'\1"' + _R + '"', s))
100
+ for p in _PATTERNS:
101
+ s = p.sub(_R, s)
102
+ return _CARD.sub(lambda m: _R if _luhn(m.group(0)) else m.group(0), s)
103
+
104
+
105
+ def _cap(s, n):
106
+ return s.encode("utf-8")[:n].decode("utf-8", "ignore")
107
+
108
+
109
+ def excerpt(s, n):
110
+ """Redact a wider window than kept so a secret cut at the edge is still masked, then cap."""
111
+ return _cap(redact_text(_cap(s, n * 2)), n)
112
+
113
+
114
+ def graphql_op(doc):
115
+ """(type, name) of a GraphQL document, `{ ... }` being an anonymous query; None when it is not one."""
116
+ d = re.sub(r"#[^\n]*", "", doc)
117
+ m = _GQL.search(d)
118
+ if m:
119
+ return m.group(1), m.group(2)
120
+ return ("query", None) if d.lstrip().startswith("{") else None
121
+
122
+
123
+ def text_like(ctype):
124
+ return bool(ctype) and _TEXT.search(ctype) is not None
125
+
126
+
127
+ def _text(body, ctype):
128
+ if body is None or (ctype and not _TEXT.search(ctype)):
129
+ return None
130
+ if isinstance(body, bytes):
131
+ return body[: HARD_MAX * 2].decode("utf-8", "ignore")
132
+ return body if isinstance(body, str) else None
133
+
134
+
135
+ def record(span, route, req_type=None, req_body=None, res_type=None, res_body=None):
136
+ """Set the body and GraphQL attributes the rule matching `route` asks for; no rule, nothing."""
137
+ rule = rule_for(route or "")
138
+ if rule is None or span is None:
139
+ return
140
+ n = min(int(rule.get("body_bytes") or 0), HARD_MAX)
141
+ req, res = _text(req_body, req_type), _text(res_body, res_type)
142
+ if n and req:
143
+ span.set_attribute("http.request.body", excerpt(req, n))
144
+ if n and res:
145
+ span.set_attribute("http.response.body", excerpt(res, n))
146
+ doc, name = None, None
147
+ if req and "graphql" in (req_type or "").lower():
148
+ doc = req
149
+ elif req:
150
+ try:
151
+ j = json.loads(req)
152
+ except ValueError:
153
+ j = None
154
+ if isinstance(j, dict) and isinstance(j.get("query"), str):
155
+ doc, name = j["query"], j.get("operationName") if isinstance(j.get("operationName"), str) else None
156
+ op = graphql_op(doc) if doc else None
157
+ if op is None:
158
+ return
159
+ span.set_attribute("graphql.operation.type", op[0])
160
+ if name or op[1]:
161
+ span.set_attribute("graphql.operation.name", name or op[1])
162
+ if rule.get("graphql_query"):
163
+ span.set_attribute("graphql.document", excerpt(doc, HARD_MAX))
watchtower/client.py ADDED
@@ -0,0 +1,227 @@
1
+ import functools
2
+ import json
3
+ import logging
4
+ import os
5
+ import urllib.request
6
+
7
+ from opentelemetry.sdk.resources import Resource
8
+ from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
9
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
10
+
11
+ from . import signals
12
+
13
+ WT_PREFIX = "wt."
14
+ DEFAULT_ENDPOINT = "http://localhost:4318"
15
+ _client = None
16
+
17
+
18
+ class _Redactor(SpanProcessor):
19
+ """Runs the user's redaction hook over span attributes before export."""
20
+
21
+ def __init__(self, hook):
22
+ self.hook = hook
23
+
24
+ def on_end(self, span):
25
+ attrs = span._attributes # noqa: SLF001 - ReadableSpan has no setter
26
+ if attrs is None:
27
+ return
28
+ # The live mapping is immutable once the span ended, so swap in a redacted copy.
29
+ span._attributes = {k: self.hook(k, v) for k, v in attrs.items()} # noqa: SLF001
30
+
31
+
32
+ def safe_resource(attrs):
33
+ """Drop caller-supplied `wt.*` keys: tenant identity comes only from the key."""
34
+ return {k: v for k, v in (attrs or {}).items() if not k.startswith(WT_PREFIX)}
35
+
36
+
37
+ class Client:
38
+ def __init__(self, api_key, endpoint=None, api_url=None, project=None, service_name=None,
39
+ resource=None, environment=None, release=None, redact=None, buffer_dir=None, exporter=None,
40
+ logs=None, metrics=None, remote_sampling=None, sampling_interval=None, code_origin=None):
41
+ if not api_key:
42
+ raise ValueError("api_key is required")
43
+ self.endpoint = (endpoint or os.environ.get("WT_OTLP_ENDPOINT") or DEFAULT_ENDPOINT).rstrip("/")
44
+ self.api_url = (api_url or os.environ.get("WT_API_URL") or self.endpoint).rstrip("/")
45
+ self.project = project or os.environ.get("WT_PROJECT_ID")
46
+ self.headers = {"Authorization": f"Bearer {api_key}"}
47
+ base = {"service.name": service_name or os.environ.get("WT_SERVICE") or "app"}
48
+ env, rel = environment or os.environ.get("WT_ENVIRONMENT"), release or os.environ.get("WT_RELEASE")
49
+ if env:
50
+ base["deployment.environment.name"] = env
51
+ if rel:
52
+ base["service.version"] = rel
53
+ res = Resource.create({**base, **safe_resource(resource)})
54
+ self.remote_sampler = None
55
+ sampler = None
56
+ if remote_sampling or (remote_sampling is None and os.environ.get("WT_REMOTE_SAMPLING") == "1"):
57
+ from opentelemetry.sdk.trace.sampling import ParentBased
58
+
59
+ from .sampling import RemoteSampler
60
+
61
+ self.remote_sampler = RemoteSampler(
62
+ self.endpoint, self.headers, base["service.name"], env,
63
+ sampling_interval or float(os.environ.get("WT_SAMPLING_INTERVAL_S") or 30))
64
+ sampler = ParentBased(self.remote_sampler)
65
+ self.provider = TracerProvider(resource=res, **({"sampler": sampler} if sampler else {}))
66
+ if code_origin or (code_origin is None and os.environ.get("WT_CODE_ORIGIN") == "1"):
67
+ from .origin import CodeOrigin
68
+
69
+ self.provider.add_span_processor(CodeOrigin())
70
+ if redact:
71
+ self.provider.add_span_processor(_Redactor(redact))
72
+ if exporter is None:
73
+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
74
+
75
+ url = f"{self.endpoint}/v1/traces"
76
+ exporter = OTLPSpanExporter(endpoint=url, headers=self.headers)
77
+ if buffer_dir:
78
+ from .buffer import SpoolExporter
79
+
80
+ exporter = SpoolExporter(exporter, buffer_dir, url, self.headers)
81
+ self.exporter = exporter
82
+ self.provider.add_span_processor(BatchSpanProcessor(exporter))
83
+ self.tracer = self.provider.get_tracer("watchtower")
84
+ self.logger_provider = self.meter_provider = self._log_handler = None
85
+ if signals.enabled(logs, "WATCHTOWER_LOGS", "OTEL_LOGS_EXPORTER"):
86
+ self.logger_provider, self._log_handler = signals.start_logs(self.endpoint, self.headers, res, buffer_dir)
87
+ if signals.enabled(metrics, "WATCHTOWER_METRICS", "OTEL_METRICS_EXPORTER"):
88
+ self.meter_provider = signals.start_metrics(self.endpoint, self.headers, res, buffer_dir)
89
+
90
+ def _call(self, method, path, body=None):
91
+ url = f"{self.api_url}/v1/projects/{self.project}/llm/{path}"
92
+ data = None if body is None else json.dumps(body).encode()
93
+ req = urllib.request.Request(url, data=data, method=method,
94
+ headers={**self.headers, "Content-Type": "application/json"})
95
+ with urllib.request.urlopen(req, timeout=10) as r:
96
+ return json.load(r)
97
+
98
+ def get_prompt(self, prompt_id):
99
+ return self._call("GET", f"prompts/{prompt_id}")
100
+
101
+ def score(self, trace_id, value, name="user_feedback", span_id=None, comment=None):
102
+ body = {"trace_id": trace_id, "value": value, "name": name, "source": "sdk"}
103
+ if span_id:
104
+ body["span_id"] = span_id
105
+ if comment:
106
+ body["comment"] = comment
107
+ return self._call("POST", "scores", body)
108
+
109
+ def evaluate_resume(self, dataset, task, scorers, previous, **kw):
110
+ """Re-run only the cases that errored or are missing in `previous` (rows of an earlier run)."""
111
+ return self.evaluate(dataset, task, scorers, resume=previous, **kw)
112
+
113
+ def evaluate(self, dataset, task, scorers, name="evaluate", error_tolerance=True, resume=None):
114
+ """Run `task` per case under a span; each scorer result is recorded as a score (when a project is set).
115
+
116
+ `resume`: rows of an earlier run; cases that finished without errors are kept, not re-run."""
117
+ rows = []
118
+ done = {r["case"]: r for r in (resume or []) if not r.get("errors")}
119
+ for i, case in enumerate(dataset):
120
+ if i in done:
121
+ rows.append(done[i])
122
+ continue
123
+ with self.tracer.start_as_current_span(f"{name}.case") as span:
124
+ tid = format(span.get_span_context().trace_id, "032x")
125
+ row = {"case": i, "trace_id": tid, "scores": {}, "errors": {}}
126
+ rows.append(row)
127
+ try:
128
+ out = task(case["input"] if isinstance(case, dict) and "input" in case else case)
129
+ except Exception as e: # noqa: BLE001
130
+ if not error_tolerance:
131
+ raise
132
+ row["errors"]["task"] = str(e)
133
+ continue
134
+ for sname, fn in scorers.items():
135
+ try:
136
+ v = fn(case, out)
137
+ except Exception as e: # noqa: BLE001
138
+ if not error_tolerance:
139
+ raise
140
+ row["errors"][sname] = str(e)
141
+ continue
142
+ row["scores"][sname] = v
143
+ if self.project:
144
+ self.score(tid, v, name=sname)
145
+ return rows
146
+
147
+ def shutdown(self):
148
+ if self.remote_sampler is not None:
149
+ self.remote_sampler.stop()
150
+ self.provider.shutdown()
151
+ if self._log_handler is not None:
152
+ logging.getLogger().removeHandler(self._log_handler)
153
+ self._log_handler = None
154
+ for p in (self.logger_provider, self.meter_provider):
155
+ if p is not None:
156
+ p.shutdown()
157
+ self.logger_provider = self.meter_provider = None
158
+
159
+
160
+ _profiler = None
161
+
162
+
163
+ def init(api_key=None, capture_errors=True, capture_rules=None, capture_locals=None, **kw):
164
+ global _client
165
+ _client = Client(api_key or os.environ.get("WT_INGEST_KEY"), **kw)
166
+ if capture_rules or (capture_rules is None and os.environ.get("WT_CAPTURE_RULES") == "1"):
167
+ from . import capture
168
+
169
+ capture.apply(_client.api_url, api_key or os.environ.get("WT_INGEST_KEY"), kw.get("service_name"))
170
+ if capture_errors:
171
+ from .errors import enable_locals, install_hooks
172
+
173
+ install_hooks()
174
+ # exception replay: opt-in here (or WT_EXCEPTION_LOCALS=1) and in the project's issue settings
175
+ enable_locals(capture_locals if capture_locals is not None else os.environ.get("WT_EXCEPTION_LOCALS") == "1")
176
+ if os.environ.get("WT_PROFILING") in ("1", "true"): # also set by the agent's lambda_instrument command
177
+ from . import profiling
178
+
179
+ global _profiler
180
+ if _profiler is not None:
181
+ _profiler.stop()
182
+ try:
183
+ _profiler = profiling.start(api_key=api_key, service_name=kw.get("service_name"))
184
+ except ValueError: # no key: profiling is best effort, never break init
185
+ _profiler = None
186
+ return _client
187
+
188
+
189
+ def _need():
190
+ if _client is None:
191
+ raise RuntimeError("watchtower.init() has not been called")
192
+ return _client
193
+
194
+
195
+ def observe(name=None):
196
+ """Decorator: wrap a function in a span named after it."""
197
+ def deco(fn):
198
+ @functools.wraps(fn)
199
+ def wrapper(*a, **kw):
200
+ with _need().tracer.start_as_current_span(name or fn.__qualname__):
201
+ return fn(*a, **kw)
202
+ return wrapper
203
+ return deco
204
+
205
+
206
+ def get_prompt(prompt_id):
207
+ return _need().get_prompt(prompt_id)
208
+
209
+
210
+ def score(trace_id, value, **kw):
211
+ return _need().score(trace_id, value, **kw)
212
+
213
+
214
+ def evaluate(dataset, task, scorers, **kw):
215
+ return _need().evaluate(dataset, task, scorers, **kw)
216
+
217
+
218
+ def evaluate_resume(dataset, task, scorers, previous, **kw):
219
+ return _need().evaluate_resume(dataset, task, scorers, previous, **kw)
220
+
221
+
222
+ def shutdown():
223
+ from .errors import uninstall_hooks
224
+
225
+ uninstall_hooks()
226
+ if _client:
227
+ _client.shutdown()