deapi-mcp 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
deapi_mcp/webhooks.py ADDED
@@ -0,0 +1,349 @@
1
+ """Internal completion webhooks: the API calls this server back when a job ends.
2
+
3
+ With DEAPI_WEBHOOKS_INTERNAL on, every job this server submits carries our own
4
+ receiver URL, and the tool that submitted it waits on the callback instead of
5
+ polling the status endpoint every few seconds. The callback is only a wake-up
6
+ signal: on receiving it the tool makes one final, authoritative status request,
7
+ because the webhook payload carries `result_url` but not inline `result`
8
+ (e.g. a transcript), and reusing the status path keeps the two modes identical.
9
+
10
+ Polling continues underneath at a much lower rate as a safety net, so a lost
11
+ or misrouted callback costs latency, never correctness.
12
+
13
+ Contract, from the deAPI webhook delivery scheme:
14
+
15
+ POST <webhook_url>
16
+ Content-Type: application/json
17
+ X-DeAPI-Signature: sha256=<hex HMAC-SHA256(secret, f"{timestamp}.{raw_body}")>
18
+ X-DeAPI-Timestamp: <unix seconds>
19
+ X-DeAPI-Event: job.processing | job.completed | job.failed
20
+ X-DeAPI-Delivery-Id: <id>
21
+
22
+ {"event": ..., "delivery_id": ..., "timestamp": <iso8601>,
23
+ "data": {"job_request_id": <uuid>, "status": ..., ...}}
24
+
25
+ Any 2xx counts as delivered; anything else is retried up to 10 times.
26
+
27
+ The job registry is pluggable. The default, ``JobCompletionRegistry``, lives in
28
+ process memory: with more than one replica a callback can land on a replica
29
+ that is not waiting for it, and that job then completes on the safety-net poll.
30
+ A deployment with several replicas passes a shared implementation of
31
+ ``JobRegistry`` to ``create_app(server, job_registry=...)``, for example one that
32
+ relays signals over a pub/sub channel.
33
+
34
+ Deliveries are deduplicated on ``delivery_id``: a signature stays valid for the
35
+ whole timestamp tolerance window, so the same delivery can arrive again. Only
36
+ the first terminal delivery wakes the waiter.
37
+ """
38
+
39
+ import asyncio
40
+ import hashlib
41
+ import hmac
42
+ import json
43
+ import logging
44
+ import time
45
+ from collections import OrderedDict
46
+ from typing import Dict, Optional, Protocol
47
+
48
+ from starlette.requests import Request
49
+ from starlette.responses import JSONResponse
50
+
51
+ from .config import settings
52
+
53
+ logger = logging.getLogger(__name__)
54
+
55
+ WEBHOOK_ROUTE = "/webhooks/deapi"
56
+
57
+ # Events that mean the job has reached a terminal state. `job.processing` is
58
+ # also delivered but says nothing a waiting tool can act on.
59
+ TERMINAL_EVENTS = frozenset({"job.completed", "job.failed"})
60
+
61
+ # How long a registry entry survives after its last touch. Long enough to outlive
62
+ # the longest polling timeout (video: 900s) so a legitimately slow job is never
63
+ # forgotten, short enough that abandoned entries do not accumulate.
64
+ REGISTRY_ENTRY_TTL = 3600.0
65
+
66
+ # Real callbacks are a few hundred bytes. The cap applies before the signature is
67
+ # checked, so an unauthenticated sender cannot make us buffer an arbitrary body.
68
+ MAX_BODY_BYTES = 64 * 1024
69
+
70
+
71
+ # Upper bound on remembered delivery ids in the in-process registry. Reached only
72
+ # under a flood of distinct signed deliveries; the oldest ids are dropped first.
73
+ MAX_REMEMBERED_DELIVERIES = 10_000
74
+
75
+
76
+ class JobRegistry(Protocol):
77
+ """What the client, the poller and the receiver need from a job registry.
78
+
79
+ ``track``, ``is_tracked`` and ``forget`` run on the replica that submitted the
80
+ job and waits for it. ``signal`` runs on the replica that received the
81
+ callback, which may be another one: a shared implementation relays it to
82
+ every replica and sets ``track``'s event there. ``claim_delivery`` returns
83
+ True the first time it sees a delivery id and False afterwards.
84
+
85
+ ``start`` and ``stop`` run with the app's lifespan while webhooks are on, so an
86
+ implementation can open and close its connections there.
87
+ """
88
+
89
+ def track(self, job_id: str) -> asyncio.Event: ...
90
+
91
+ def is_tracked(self, job_id: str) -> bool: ...
92
+
93
+ def forget(self, job_id: str) -> None: ...
94
+
95
+ async def signal(self, job_id: str) -> None: ...
96
+
97
+ async def claim_delivery(self, delivery_id: str) -> bool: ...
98
+
99
+ async def start(self) -> None: ...
100
+
101
+ async def stop(self) -> None: ...
102
+
103
+
104
+ class JobCompletionRegistry:
105
+ """The default ``JobRegistry``: job ids mapped to events, in process memory.
106
+
107
+ Order-independent by design: the callback can arrive before the submitting
108
+ tool has learned its job id (the API assigns it in the submit response, and
109
+ a fast job could finish in the gap). Both `track` and `wake` therefore
110
+ create the entry if it is missing, and a signal that arrives first is not
111
+ lost — the later `track` finds the event already set.
112
+
113
+ A shared implementation can subclass it, keep the local events, relay
114
+ ``signal`` to other replicas, and call ``wake`` where the relay arrives.
115
+ """
116
+
117
+ def __init__(
118
+ self,
119
+ *,
120
+ delivery_ttl: Optional[float] = None,
121
+ max_deliveries: int = MAX_REMEMBERED_DELIVERIES,
122
+ ) -> None:
123
+ self._events: Dict[str, asyncio.Event] = {}
124
+ self._touched: Dict[str, float] = {}
125
+ # delivery id -> monotonic expiry, oldest first
126
+ self._deliveries: "OrderedDict[str, float]" = OrderedDict()
127
+ self._delivery_ttl = delivery_ttl
128
+ self._max_deliveries = max_deliveries
129
+
130
+ def track(self, job_id: str) -> asyncio.Event:
131
+ """Register interest in a job and return the event to wait on."""
132
+ self._sweep()
133
+ event = self._events.setdefault(job_id, asyncio.Event())
134
+ self._touched[job_id] = time.monotonic()
135
+ return event
136
+
137
+ def wake(self, job_id: str) -> None:
138
+ """Mark a job as finished in this process, waking any local waiter."""
139
+ self._sweep()
140
+ event = self._events.setdefault(job_id, asyncio.Event())
141
+ self._touched[job_id] = time.monotonic()
142
+ event.set()
143
+
144
+ async def signal(self, job_id: str) -> None:
145
+ """Mark a job as finished. In process memory this is ``wake``."""
146
+ self.wake(job_id)
147
+
148
+ async def claim_delivery(self, delivery_id: str) -> bool:
149
+ """True the first time a delivery id is seen within the dedupe window.
150
+
151
+ The window defaults to twice the signature tolerance: a replay older than
152
+ the tolerance already fails the signature check, and twice covers clock
153
+ skew in either direction.
154
+ """
155
+ now = time.monotonic()
156
+ ttl = self._delivery_ttl
157
+ if ttl is None:
158
+ ttl = 2 * settings.webhook_timestamp_tolerance
159
+ while self._deliveries:
160
+ expires = next(iter(self._deliveries.values()))
161
+ if expires > now:
162
+ break
163
+ self._deliveries.popitem(last=False)
164
+ if delivery_id in self._deliveries:
165
+ return False
166
+ self._deliveries[delivery_id] = now + ttl
167
+ while len(self._deliveries) > self._max_deliveries:
168
+ self._deliveries.popitem(last=False)
169
+ return True
170
+
171
+ async def start(self) -> None:
172
+ """Nothing to open in process memory."""
173
+
174
+ async def stop(self) -> None:
175
+ """Nothing to close in process memory."""
176
+
177
+ def is_tracked(self, job_id: str) -> bool:
178
+ return job_id in self._events
179
+
180
+ def forget(self, job_id: str) -> None:
181
+ """Drop a job once its tool has returned."""
182
+ self._events.pop(job_id, None)
183
+ self._touched.pop(job_id, None)
184
+
185
+ def _sweep(self) -> None:
186
+ cutoff = time.monotonic() - REGISTRY_ENTRY_TTL
187
+ stale = [job_id for job_id, touched in self._touched.items() if touched < cutoff]
188
+ for job_id in stale:
189
+ self.forget(job_id)
190
+
191
+
192
+ _job_registry: JobRegistry = JobCompletionRegistry()
193
+
194
+
195
+ def get_job_registry() -> JobRegistry:
196
+ """The registry the client, the poller and the receiver use."""
197
+ return _job_registry
198
+
199
+
200
+ def set_job_registry(job_registry: JobRegistry) -> None:
201
+ """Replace the registry. ``create_app(job_registry=...)`` calls this at startup."""
202
+ global _job_registry
203
+ _job_registry = job_registry
204
+
205
+
206
+ def internal_webhooks_enabled() -> bool:
207
+ return bool(settings.webhooks_internal and settings.webhook_receiver_url)
208
+
209
+
210
+ def compute_signature(secret: str, timestamp: str, raw_body: bytes) -> str:
211
+ """Produce the value the API puts in X-DeAPI-Signature.
212
+
213
+ Mirrors the deAPI webhook signing scheme: HMAC-SHA256 over the
214
+ timestamp, a literal ".", and the body bytes exactly as sent.
215
+ """
216
+ message = timestamp.encode("ascii") + b"." + raw_body
217
+ digest = hmac.new(secret.encode("utf-8"), message, hashlib.sha256).hexdigest()
218
+ return f"sha256={digest}"
219
+
220
+
221
+ def verify_signature(
222
+ secret: str,
223
+ timestamp: Optional[str],
224
+ signature: Optional[str],
225
+ raw_body: bytes,
226
+ tolerance: float,
227
+ now: Optional[float] = None,
228
+ ) -> bool:
229
+ """Check a callback's signature and freshness.
230
+
231
+ The body must be the raw bytes received, not a re-serialisation: the API
232
+ signed what it sent, and any re-encoding (key order, whitespace, unicode
233
+ escaping) breaks the match.
234
+ """
235
+ if not timestamp or not signature:
236
+ return False
237
+
238
+ try:
239
+ sent_at = float(timestamp)
240
+ except ValueError:
241
+ return False
242
+
243
+ current = time.time() if now is None else now
244
+ if abs(current - sent_at) > tolerance:
245
+ return False
246
+
247
+ # Compare bytes: compare_digest raises TypeError on a non-ASCII str, which would be a 500.
248
+ expected = compute_signature(secret, timestamp, raw_body).encode("ascii")
249
+ received = signature if isinstance(signature, bytes) else signature.encode("utf-8", "replace")
250
+ return hmac.compare_digest(expected, received)
251
+
252
+
253
+ async def read_body_limited(request: Request, limit: int) -> Optional[bytes]:
254
+ """Read the request body, or return None as soon as it exceeds `limit` bytes.
255
+
256
+ Checks Content-Length first, then counts while streaming, so an oversized or
257
+ unannounced chunked body is never buffered in full.
258
+ """
259
+ declared = request.headers.get("content-length")
260
+ if declared is not None:
261
+ try:
262
+ if int(declared) > limit:
263
+ return None
264
+ except ValueError:
265
+ return None
266
+
267
+ chunks = []
268
+ size = 0
269
+ async for chunk in request.stream():
270
+ size += len(chunk)
271
+ if size > limit:
272
+ return None
273
+ chunks.append(chunk)
274
+ return b"".join(chunks)
275
+
276
+
277
+ async def webhook_receiver(request: Request) -> JSONResponse:
278
+ """Starlette endpoint the API posts job events to.
279
+
280
+ Returns 2xx for everything it accepts, including events for jobs it does
281
+ not know about (a different replica may be waiting, and a retry from the
282
+ API would not change that) and non-terminal events. Only a bad signature is
283
+ refused, and that is the one case where the API retrying is harmless.
284
+ """
285
+ secret = settings.webhook_secret
286
+ if not internal_webhooks_enabled() or not secret:
287
+ # The route is only mounted when enabled, so this is belt and braces.
288
+ return JSONResponse({"error": "webhook receiver not enabled"}, status_code=404)
289
+
290
+ raw_body = await read_body_limited(request, MAX_BODY_BYTES)
291
+ if raw_body is None:
292
+ logger.warning(
293
+ "Rejected oversized webhook (delivery %s)", request.headers.get("X-DeAPI-Delivery-Id")
294
+ )
295
+ return JSONResponse({"error": "payload too large"}, status_code=413)
296
+
297
+ if not verify_signature(
298
+ secret,
299
+ request.headers.get("X-DeAPI-Timestamp"),
300
+ request.headers.get("X-DeAPI-Signature"),
301
+ raw_body,
302
+ settings.webhook_timestamp_tolerance,
303
+ ):
304
+ logger.warning(
305
+ "Rejected webhook with bad signature (delivery %s)",
306
+ request.headers.get("X-DeAPI-Delivery-Id"),
307
+ )
308
+ return JSONResponse({"error": "invalid signature"}, status_code=401)
309
+
310
+ try:
311
+ payload = json.loads(raw_body)
312
+ event = payload.get("event") or request.headers.get("X-DeAPI-Event")
313
+ job_id = payload["data"]["job_request_id"]
314
+ delivery_id = payload.get("delivery_id") or request.headers.get("X-DeAPI-Delivery-Id")
315
+ except (ValueError, KeyError, TypeError, AttributeError):
316
+ return JSONResponse({"error": "malformed payload"}, status_code=400)
317
+ if not isinstance(job_id, str) or not job_id:
318
+ return JSONResponse({"error": "malformed payload"}, status_code=400)
319
+
320
+ if event not in TERMINAL_EVENTS:
321
+ logger.debug("Webhook %s for job %s: not terminal, ignored", event, job_id)
322
+ return JSONResponse({"received": True})
323
+
324
+ # The callback only wakes a waiter, which then reads the job status itself. A registry
325
+ # failure is therefore logged and acknowledged: the job completes on the fallback poll,
326
+ # and a retry from the API would not fix the registry.
327
+ job_registry = get_job_registry()
328
+ if isinstance(delivery_id, str) and delivery_id:
329
+ try:
330
+ first = await job_registry.claim_delivery(delivery_id)
331
+ except Exception:
332
+ logger.warning(
333
+ "Could not check webhook delivery %s for reuse", delivery_id, exc_info=True
334
+ )
335
+ first = True
336
+ if not first:
337
+ logger.info("Webhook delivery %s for job %s already handled", delivery_id, job_id)
338
+ return JSONResponse({"received": True, "duplicate": True})
339
+
340
+ try:
341
+ await job_registry.signal(job_id)
342
+ except Exception:
343
+ logger.warning(
344
+ "Could not signal job %s; it completes on the fallback poll", job_id, exc_info=True
345
+ )
346
+ else:
347
+ logger.debug("Webhook %s for job %s: waiter signalled", event, job_id)
348
+
349
+ return JSONResponse({"received": True})