deapi-mcp 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deapi_mcp/__init__.py +24 -0
- deapi_mcp/auth.py +33 -0
- deapi_mcp/auth_code_store.py +226 -0
- deapi_mcp/config.py +297 -0
- deapi_mcp/consent.py +549 -0
- deapi_mcp/deapi_client.py +466 -0
- deapi_mcp/fastmcp_auth.py +227 -0
- deapi_mcp/key_verifier.py +225 -0
- deapi_mcp/landing.py +431 -0
- deapi_mcp/middleware.py +756 -0
- deapi_mcp/oauth_clients.py +443 -0
- deapi_mcp/oauth_endpoints.py +973 -0
- deapi_mcp/observability.py +199 -0
- deapi_mcp/polling_manager.py +319 -0
- deapi_mcp/py.typed +0 -0
- deapi_mcp/schemas.py +397 -0
- deapi_mcp/server_remote.py +416 -0
- deapi_mcp/tools/__init__.py +0 -0
- deapi_mcp/tools/_price_helpers.py +167 -0
- deapi_mcp/tools/_responses.py +157 -0
- deapi_mcp/tools/audio.py +1195 -0
- deapi_mcp/tools/embedding.py +102 -0
- deapi_mcp/tools/image.py +884 -0
- deapi_mcp/tools/prompt.py +155 -0
- deapi_mcp/tools/utility.py +230 -0
- deapi_mcp/tools/video.py +1117 -0
- deapi_mcp/utils.py +666 -0
- deapi_mcp/webhooks.py +349 -0
- deapi_mcp-0.7.0.dist-info/METADATA +642 -0
- deapi_mcp-0.7.0.dist-info/RECORD +33 -0
- deapi_mcp-0.7.0.dist-info/WHEEL +4 -0
- deapi_mcp-0.7.0.dist-info/entry_points.txt +2 -0
- deapi_mcp-0.7.0.dist-info/licenses/LICENSE +21 -0
deapi_mcp/webhooks.py
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
"""Internal completion webhooks: the API calls this server back when a job ends.
|
|
2
|
+
|
|
3
|
+
With DEAPI_WEBHOOKS_INTERNAL on, every job this server submits carries our own
|
|
4
|
+
receiver URL, and the tool that submitted it waits on the callback instead of
|
|
5
|
+
polling the status endpoint every few seconds. The callback is only a wake-up
|
|
6
|
+
signal: on receiving it the tool makes one final, authoritative status request,
|
|
7
|
+
because the webhook payload carries `result_url` but not inline `result`
|
|
8
|
+
(e.g. a transcript), and reusing the status path keeps the two modes identical.
|
|
9
|
+
|
|
10
|
+
Polling continues underneath at a much lower rate as a safety net, so a lost
|
|
11
|
+
or misrouted callback costs latency, never correctness.
|
|
12
|
+
|
|
13
|
+
Contract, from the deAPI webhook delivery scheme:
|
|
14
|
+
|
|
15
|
+
POST <webhook_url>
|
|
16
|
+
Content-Type: application/json
|
|
17
|
+
X-DeAPI-Signature: sha256=<hex HMAC-SHA256(secret, f"{timestamp}.{raw_body}")>
|
|
18
|
+
X-DeAPI-Timestamp: <unix seconds>
|
|
19
|
+
X-DeAPI-Event: job.processing | job.completed | job.failed
|
|
20
|
+
X-DeAPI-Delivery-Id: <id>
|
|
21
|
+
|
|
22
|
+
{"event": ..., "delivery_id": ..., "timestamp": <iso8601>,
|
|
23
|
+
"data": {"job_request_id": <uuid>, "status": ..., ...}}
|
|
24
|
+
|
|
25
|
+
Any 2xx counts as delivered; anything else is retried up to 10 times.
|
|
26
|
+
|
|
27
|
+
The job registry is pluggable. The default, ``JobCompletionRegistry``, lives in
|
|
28
|
+
process memory: with more than one replica a callback can land on a replica
|
|
29
|
+
that is not waiting for it, and that job then completes on the safety-net poll.
|
|
30
|
+
A deployment with several replicas passes a shared implementation of
|
|
31
|
+
``JobRegistry`` to ``create_app(server, job_registry=...)``, for example one that
|
|
32
|
+
relays signals over a pub/sub channel.
|
|
33
|
+
|
|
34
|
+
Deliveries are deduplicated on ``delivery_id``: a signature stays valid for the
|
|
35
|
+
whole timestamp tolerance window, so the same delivery can arrive again. Only
|
|
36
|
+
the first terminal delivery wakes the waiter.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
import asyncio
|
|
40
|
+
import hashlib
|
|
41
|
+
import hmac
|
|
42
|
+
import json
|
|
43
|
+
import logging
|
|
44
|
+
import time
|
|
45
|
+
from collections import OrderedDict
|
|
46
|
+
from typing import Dict, Optional, Protocol
|
|
47
|
+
|
|
48
|
+
from starlette.requests import Request
|
|
49
|
+
from starlette.responses import JSONResponse
|
|
50
|
+
|
|
51
|
+
from .config import settings
|
|
52
|
+
|
|
53
|
+
logger = logging.getLogger(__name__)
|
|
54
|
+
|
|
55
|
+
WEBHOOK_ROUTE = "/webhooks/deapi"
|
|
56
|
+
|
|
57
|
+
# Events that mean the job has reached a terminal state. `job.processing` is
|
|
58
|
+
# also delivered but says nothing a waiting tool can act on.
|
|
59
|
+
TERMINAL_EVENTS = frozenset({"job.completed", "job.failed"})
|
|
60
|
+
|
|
61
|
+
# How long a registry entry survives after its last touch. Long enough to outlive
|
|
62
|
+
# the longest polling timeout (video: 900s) so a legitimately slow job is never
|
|
63
|
+
# forgotten, short enough that abandoned entries do not accumulate.
|
|
64
|
+
REGISTRY_ENTRY_TTL = 3600.0
|
|
65
|
+
|
|
66
|
+
# Real callbacks are a few hundred bytes. The cap applies before the signature is
|
|
67
|
+
# checked, so an unauthenticated sender cannot make us buffer an arbitrary body.
|
|
68
|
+
MAX_BODY_BYTES = 64 * 1024
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# Upper bound on remembered delivery ids in the in-process registry. Reached only
|
|
72
|
+
# under a flood of distinct signed deliveries; the oldest ids are dropped first.
|
|
73
|
+
MAX_REMEMBERED_DELIVERIES = 10_000
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class JobRegistry(Protocol):
|
|
77
|
+
"""What the client, the poller and the receiver need from a job registry.
|
|
78
|
+
|
|
79
|
+
``track``, ``is_tracked`` and ``forget`` run on the replica that submitted the
|
|
80
|
+
job and waits for it. ``signal`` runs on the replica that received the
|
|
81
|
+
callback, which may be another one: a shared implementation relays it to
|
|
82
|
+
every replica and sets ``track``'s event there. ``claim_delivery`` returns
|
|
83
|
+
True the first time it sees a delivery id and False afterwards.
|
|
84
|
+
|
|
85
|
+
``start`` and ``stop`` run with the app's lifespan while webhooks are on, so an
|
|
86
|
+
implementation can open and close its connections there.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
def track(self, job_id: str) -> asyncio.Event: ...
|
|
90
|
+
|
|
91
|
+
def is_tracked(self, job_id: str) -> bool: ...
|
|
92
|
+
|
|
93
|
+
def forget(self, job_id: str) -> None: ...
|
|
94
|
+
|
|
95
|
+
async def signal(self, job_id: str) -> None: ...
|
|
96
|
+
|
|
97
|
+
async def claim_delivery(self, delivery_id: str) -> bool: ...
|
|
98
|
+
|
|
99
|
+
async def start(self) -> None: ...
|
|
100
|
+
|
|
101
|
+
async def stop(self) -> None: ...
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class JobCompletionRegistry:
|
|
105
|
+
"""The default ``JobRegistry``: job ids mapped to events, in process memory.
|
|
106
|
+
|
|
107
|
+
Order-independent by design: the callback can arrive before the submitting
|
|
108
|
+
tool has learned its job id (the API assigns it in the submit response, and
|
|
109
|
+
a fast job could finish in the gap). Both `track` and `wake` therefore
|
|
110
|
+
create the entry if it is missing, and a signal that arrives first is not
|
|
111
|
+
lost — the later `track` finds the event already set.
|
|
112
|
+
|
|
113
|
+
A shared implementation can subclass it, keep the local events, relay
|
|
114
|
+
``signal`` to other replicas, and call ``wake`` where the relay arrives.
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
def __init__(
|
|
118
|
+
self,
|
|
119
|
+
*,
|
|
120
|
+
delivery_ttl: Optional[float] = None,
|
|
121
|
+
max_deliveries: int = MAX_REMEMBERED_DELIVERIES,
|
|
122
|
+
) -> None:
|
|
123
|
+
self._events: Dict[str, asyncio.Event] = {}
|
|
124
|
+
self._touched: Dict[str, float] = {}
|
|
125
|
+
# delivery id -> monotonic expiry, oldest first
|
|
126
|
+
self._deliveries: "OrderedDict[str, float]" = OrderedDict()
|
|
127
|
+
self._delivery_ttl = delivery_ttl
|
|
128
|
+
self._max_deliveries = max_deliveries
|
|
129
|
+
|
|
130
|
+
def track(self, job_id: str) -> asyncio.Event:
|
|
131
|
+
"""Register interest in a job and return the event to wait on."""
|
|
132
|
+
self._sweep()
|
|
133
|
+
event = self._events.setdefault(job_id, asyncio.Event())
|
|
134
|
+
self._touched[job_id] = time.monotonic()
|
|
135
|
+
return event
|
|
136
|
+
|
|
137
|
+
def wake(self, job_id: str) -> None:
|
|
138
|
+
"""Mark a job as finished in this process, waking any local waiter."""
|
|
139
|
+
self._sweep()
|
|
140
|
+
event = self._events.setdefault(job_id, asyncio.Event())
|
|
141
|
+
self._touched[job_id] = time.monotonic()
|
|
142
|
+
event.set()
|
|
143
|
+
|
|
144
|
+
async def signal(self, job_id: str) -> None:
|
|
145
|
+
"""Mark a job as finished. In process memory this is ``wake``."""
|
|
146
|
+
self.wake(job_id)
|
|
147
|
+
|
|
148
|
+
async def claim_delivery(self, delivery_id: str) -> bool:
|
|
149
|
+
"""True the first time a delivery id is seen within the dedupe window.
|
|
150
|
+
|
|
151
|
+
The window defaults to twice the signature tolerance: a replay older than
|
|
152
|
+
the tolerance already fails the signature check, and twice covers clock
|
|
153
|
+
skew in either direction.
|
|
154
|
+
"""
|
|
155
|
+
now = time.monotonic()
|
|
156
|
+
ttl = self._delivery_ttl
|
|
157
|
+
if ttl is None:
|
|
158
|
+
ttl = 2 * settings.webhook_timestamp_tolerance
|
|
159
|
+
while self._deliveries:
|
|
160
|
+
expires = next(iter(self._deliveries.values()))
|
|
161
|
+
if expires > now:
|
|
162
|
+
break
|
|
163
|
+
self._deliveries.popitem(last=False)
|
|
164
|
+
if delivery_id in self._deliveries:
|
|
165
|
+
return False
|
|
166
|
+
self._deliveries[delivery_id] = now + ttl
|
|
167
|
+
while len(self._deliveries) > self._max_deliveries:
|
|
168
|
+
self._deliveries.popitem(last=False)
|
|
169
|
+
return True
|
|
170
|
+
|
|
171
|
+
async def start(self) -> None:
|
|
172
|
+
"""Nothing to open in process memory."""
|
|
173
|
+
|
|
174
|
+
async def stop(self) -> None:
|
|
175
|
+
"""Nothing to close in process memory."""
|
|
176
|
+
|
|
177
|
+
def is_tracked(self, job_id: str) -> bool:
|
|
178
|
+
return job_id in self._events
|
|
179
|
+
|
|
180
|
+
def forget(self, job_id: str) -> None:
|
|
181
|
+
"""Drop a job once its tool has returned."""
|
|
182
|
+
self._events.pop(job_id, None)
|
|
183
|
+
self._touched.pop(job_id, None)
|
|
184
|
+
|
|
185
|
+
def _sweep(self) -> None:
|
|
186
|
+
cutoff = time.monotonic() - REGISTRY_ENTRY_TTL
|
|
187
|
+
stale = [job_id for job_id, touched in self._touched.items() if touched < cutoff]
|
|
188
|
+
for job_id in stale:
|
|
189
|
+
self.forget(job_id)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
_job_registry: JobRegistry = JobCompletionRegistry()
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def get_job_registry() -> JobRegistry:
|
|
196
|
+
"""The registry the client, the poller and the receiver use."""
|
|
197
|
+
return _job_registry
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def set_job_registry(job_registry: JobRegistry) -> None:
|
|
201
|
+
"""Replace the registry. ``create_app(job_registry=...)`` calls this at startup."""
|
|
202
|
+
global _job_registry
|
|
203
|
+
_job_registry = job_registry
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def internal_webhooks_enabled() -> bool:
|
|
207
|
+
return bool(settings.webhooks_internal and settings.webhook_receiver_url)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def compute_signature(secret: str, timestamp: str, raw_body: bytes) -> str:
|
|
211
|
+
"""Produce the value the API puts in X-DeAPI-Signature.
|
|
212
|
+
|
|
213
|
+
Mirrors the deAPI webhook signing scheme: HMAC-SHA256 over the
|
|
214
|
+
timestamp, a literal ".", and the body bytes exactly as sent.
|
|
215
|
+
"""
|
|
216
|
+
message = timestamp.encode("ascii") + b"." + raw_body
|
|
217
|
+
digest = hmac.new(secret.encode("utf-8"), message, hashlib.sha256).hexdigest()
|
|
218
|
+
return f"sha256={digest}"
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def verify_signature(
|
|
222
|
+
secret: str,
|
|
223
|
+
timestamp: Optional[str],
|
|
224
|
+
signature: Optional[str],
|
|
225
|
+
raw_body: bytes,
|
|
226
|
+
tolerance: float,
|
|
227
|
+
now: Optional[float] = None,
|
|
228
|
+
) -> bool:
|
|
229
|
+
"""Check a callback's signature and freshness.
|
|
230
|
+
|
|
231
|
+
The body must be the raw bytes received, not a re-serialisation: the API
|
|
232
|
+
signed what it sent, and any re-encoding (key order, whitespace, unicode
|
|
233
|
+
escaping) breaks the match.
|
|
234
|
+
"""
|
|
235
|
+
if not timestamp or not signature:
|
|
236
|
+
return False
|
|
237
|
+
|
|
238
|
+
try:
|
|
239
|
+
sent_at = float(timestamp)
|
|
240
|
+
except ValueError:
|
|
241
|
+
return False
|
|
242
|
+
|
|
243
|
+
current = time.time() if now is None else now
|
|
244
|
+
if abs(current - sent_at) > tolerance:
|
|
245
|
+
return False
|
|
246
|
+
|
|
247
|
+
# Compare bytes: compare_digest raises TypeError on a non-ASCII str, which would be a 500.
|
|
248
|
+
expected = compute_signature(secret, timestamp, raw_body).encode("ascii")
|
|
249
|
+
received = signature if isinstance(signature, bytes) else signature.encode("utf-8", "replace")
|
|
250
|
+
return hmac.compare_digest(expected, received)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
async def read_body_limited(request: Request, limit: int) -> Optional[bytes]:
|
|
254
|
+
"""Read the request body, or return None as soon as it exceeds `limit` bytes.
|
|
255
|
+
|
|
256
|
+
Checks Content-Length first, then counts while streaming, so an oversized or
|
|
257
|
+
unannounced chunked body is never buffered in full.
|
|
258
|
+
"""
|
|
259
|
+
declared = request.headers.get("content-length")
|
|
260
|
+
if declared is not None:
|
|
261
|
+
try:
|
|
262
|
+
if int(declared) > limit:
|
|
263
|
+
return None
|
|
264
|
+
except ValueError:
|
|
265
|
+
return None
|
|
266
|
+
|
|
267
|
+
chunks = []
|
|
268
|
+
size = 0
|
|
269
|
+
async for chunk in request.stream():
|
|
270
|
+
size += len(chunk)
|
|
271
|
+
if size > limit:
|
|
272
|
+
return None
|
|
273
|
+
chunks.append(chunk)
|
|
274
|
+
return b"".join(chunks)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
async def webhook_receiver(request: Request) -> JSONResponse:
|
|
278
|
+
"""Starlette endpoint the API posts job events to.
|
|
279
|
+
|
|
280
|
+
Returns 2xx for everything it accepts, including events for jobs it does
|
|
281
|
+
not know about (a different replica may be waiting, and a retry from the
|
|
282
|
+
API would not change that) and non-terminal events. Only a bad signature is
|
|
283
|
+
refused, and that is the one case where the API retrying is harmless.
|
|
284
|
+
"""
|
|
285
|
+
secret = settings.webhook_secret
|
|
286
|
+
if not internal_webhooks_enabled() or not secret:
|
|
287
|
+
# The route is only mounted when enabled, so this is belt and braces.
|
|
288
|
+
return JSONResponse({"error": "webhook receiver not enabled"}, status_code=404)
|
|
289
|
+
|
|
290
|
+
raw_body = await read_body_limited(request, MAX_BODY_BYTES)
|
|
291
|
+
if raw_body is None:
|
|
292
|
+
logger.warning(
|
|
293
|
+
"Rejected oversized webhook (delivery %s)", request.headers.get("X-DeAPI-Delivery-Id")
|
|
294
|
+
)
|
|
295
|
+
return JSONResponse({"error": "payload too large"}, status_code=413)
|
|
296
|
+
|
|
297
|
+
if not verify_signature(
|
|
298
|
+
secret,
|
|
299
|
+
request.headers.get("X-DeAPI-Timestamp"),
|
|
300
|
+
request.headers.get("X-DeAPI-Signature"),
|
|
301
|
+
raw_body,
|
|
302
|
+
settings.webhook_timestamp_tolerance,
|
|
303
|
+
):
|
|
304
|
+
logger.warning(
|
|
305
|
+
"Rejected webhook with bad signature (delivery %s)",
|
|
306
|
+
request.headers.get("X-DeAPI-Delivery-Id"),
|
|
307
|
+
)
|
|
308
|
+
return JSONResponse({"error": "invalid signature"}, status_code=401)
|
|
309
|
+
|
|
310
|
+
try:
|
|
311
|
+
payload = json.loads(raw_body)
|
|
312
|
+
event = payload.get("event") or request.headers.get("X-DeAPI-Event")
|
|
313
|
+
job_id = payload["data"]["job_request_id"]
|
|
314
|
+
delivery_id = payload.get("delivery_id") or request.headers.get("X-DeAPI-Delivery-Id")
|
|
315
|
+
except (ValueError, KeyError, TypeError, AttributeError):
|
|
316
|
+
return JSONResponse({"error": "malformed payload"}, status_code=400)
|
|
317
|
+
if not isinstance(job_id, str) or not job_id:
|
|
318
|
+
return JSONResponse({"error": "malformed payload"}, status_code=400)
|
|
319
|
+
|
|
320
|
+
if event not in TERMINAL_EVENTS:
|
|
321
|
+
logger.debug("Webhook %s for job %s: not terminal, ignored", event, job_id)
|
|
322
|
+
return JSONResponse({"received": True})
|
|
323
|
+
|
|
324
|
+
# The callback only wakes a waiter, which then reads the job status itself. A registry
|
|
325
|
+
# failure is therefore logged and acknowledged: the job completes on the fallback poll,
|
|
326
|
+
# and a retry from the API would not fix the registry.
|
|
327
|
+
job_registry = get_job_registry()
|
|
328
|
+
if isinstance(delivery_id, str) and delivery_id:
|
|
329
|
+
try:
|
|
330
|
+
first = await job_registry.claim_delivery(delivery_id)
|
|
331
|
+
except Exception:
|
|
332
|
+
logger.warning(
|
|
333
|
+
"Could not check webhook delivery %s for reuse", delivery_id, exc_info=True
|
|
334
|
+
)
|
|
335
|
+
first = True
|
|
336
|
+
if not first:
|
|
337
|
+
logger.info("Webhook delivery %s for job %s already handled", delivery_id, job_id)
|
|
338
|
+
return JSONResponse({"received": True, "duplicate": True})
|
|
339
|
+
|
|
340
|
+
try:
|
|
341
|
+
await job_registry.signal(job_id)
|
|
342
|
+
except Exception:
|
|
343
|
+
logger.warning(
|
|
344
|
+
"Could not signal job %s; it completes on the fallback poll", job_id, exc_info=True
|
|
345
|
+
)
|
|
346
|
+
else:
|
|
347
|
+
logger.debug("Webhook %s for job %s: waiter signalled", event, job_id)
|
|
348
|
+
|
|
349
|
+
return JSONResponse({"received": True})
|