spot-sdk-python 2.0.0b6__tar.gz → 2.0.0b7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/PKG-INFO +2 -2
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/README.md +1 -1
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/pyproject.toml +1 -1
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/app.py +155 -17
- spot_sdk_python-2.0.0b7/spot_sdk/artifacts.py +205 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/clients.py +58 -7
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/email.py +2 -1
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/manifest.py +91 -1
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/mime.py +71 -8
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/README.md +3 -1
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/__init__.py +4 -1
- spot_sdk_python-2.0.0b7/spot_sdk/testing/contract.py +219 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/fake_knowledge_client.py +7 -3
- spot_sdk_python-2.0.0b7/spot_sdk/testing/settings.py +128 -0
- spot_sdk_python-2.0.0b7/spot_sdk/urls.py +36 -0
- spot_sdk_python-2.0.0b7/spot_sdk/verdict.py +127 -0
- spot_sdk_python-2.0.0b6/spot_sdk/testing/contract.py +0 -51
- spot_sdk_python-2.0.0b6/spot_sdk/verdict.py +0 -53
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/__init__.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/_json.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/config.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/errors.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/knowledge.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/knowledge_tags.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/llm.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/logging.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/orchestrator.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/py.typed +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/retriever.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/settings.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/signals.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/fake_llm.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/fake_spot.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/views.py +0 -0
- {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/workflow.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: spot-sdk-python
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.0b7
|
|
4
4
|
Summary: SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Author: SPOT Project
|
|
@@ -50,7 +50,7 @@ plugins share:
|
|
|
50
50
|
## Install
|
|
51
51
|
|
|
52
52
|
```bash
|
|
53
|
-
pip install spot-sdk-python==2.0.
|
|
53
|
+
pip install spot-sdk-python==2.0.0b7
|
|
54
54
|
```
|
|
55
55
|
|
|
56
56
|
Python 3.11 to 3.15. Version 2 implements plugin contract 2, for
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "spot-sdk-python"
|
|
7
|
-
version = "2.0.
|
|
7
|
+
version = "2.0.0b7"
|
|
8
8
|
description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
|
|
9
9
|
authors = ["SPOT Project <spot@sonn.lu>"]
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -15,6 +15,10 @@ An ``async def`` handler, or an object whose ``__call__`` is ``async def``,
|
|
|
15
15
|
runs on the event loop; any other callable runs in a worker thread, so it may
|
|
16
16
|
block. Either way the clients send the call's ``X-Correlation-ID`` on to SPOT.
|
|
17
17
|
|
|
18
|
+
A plugin that loads a model gives ``load``: run in a daemon thread once the app
|
|
19
|
+
starts, it makes the plugin ``starting`` until it returns, and the analyzer
|
|
20
|
+
answers ``unknown`` meanwhile; then ``analyze`` gets what it returned.
|
|
21
|
+
|
|
18
22
|
Neither the logs nor the error answers hold the request's content: a
|
|
19
23
|
handler's exception is logged by its type and place only, and an invalid
|
|
20
24
|
request is answered with the fields refused, never the values sent.
|
|
@@ -29,11 +33,12 @@ from __future__ import annotations
|
|
|
29
33
|
import asyncio
|
|
30
34
|
import hmac
|
|
31
35
|
import inspect
|
|
36
|
+
import threading
|
|
32
37
|
import traceback
|
|
33
|
-
from collections.abc import Awaitable, Callable, Iterator
|
|
34
|
-
from contextlib import contextmanager
|
|
38
|
+
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
|
|
39
|
+
from contextlib import asynccontextmanager, contextmanager
|
|
35
40
|
from datetime import UTC, datetime
|
|
36
|
-
from typing import Any, Literal, TypeAlias, TypeVar
|
|
41
|
+
from typing import Any, Generic, Literal, TypeAlias, TypeVar, overload
|
|
37
42
|
|
|
38
43
|
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
|
|
39
44
|
from fastapi.responses import JSONResponse
|
|
@@ -55,6 +60,7 @@ logger = get_logger(__name__)
|
|
|
55
60
|
|
|
56
61
|
SettingsT = TypeVar("SettingsT", bound=PluginSettings)
|
|
57
62
|
ModelT = TypeVar("ModelT", bound=BaseModel)
|
|
63
|
+
LoadedT = TypeVar("LoadedT")
|
|
58
64
|
|
|
59
65
|
AnalyzeHandler: TypeAlias = Callable[
|
|
60
66
|
[AnalyzeRequest, SettingsT], AnalyzerVerdict | Awaitable[AnalyzerVerdict]
|
|
@@ -74,6 +80,12 @@ action idempotent (a ticket keyed by the job id).
|
|
|
74
80
|
"""
|
|
75
81
|
SyncHandler: TypeAlias = Callable[[SettingsT], int | Awaitable[int]]
|
|
76
82
|
"""``sync(settings) -> int``: the number of documents written."""
|
|
83
|
+
Loader: TypeAlias = Callable[[SettingsT], LoadedT]
|
|
84
|
+
"""``load(settings) -> object``: a model, say, loaded once in a daemon thread."""
|
|
85
|
+
LoadedAnalyzeHandler: TypeAlias = Callable[
|
|
86
|
+
[AnalyzeRequest, SettingsT, LoadedT], AnalyzerVerdict | Awaitable[AnalyzerVerdict]
|
|
87
|
+
]
|
|
88
|
+
"""``analyze(request, settings, loaded) -> AnalyzerVerdict``, with what ``load`` returned."""
|
|
77
89
|
|
|
78
90
|
|
|
79
91
|
class SyncState(BaseModel):
|
|
@@ -117,6 +129,13 @@ class PluginHealth(BaseModel):
|
|
|
117
129
|
)
|
|
118
130
|
|
|
119
131
|
|
|
132
|
+
class HealthAnswer(PluginHealth):
|
|
133
|
+
"""``GET /health``'s body: the plugin's state and the version of its
|
|
134
|
+
manifest. ``reason`` is left out when there is none."""
|
|
135
|
+
|
|
136
|
+
version: str = Field(description="The manifest's version")
|
|
137
|
+
|
|
138
|
+
|
|
120
139
|
class ErrorAnswer(BaseModel):
|
|
121
140
|
"""An error the app answers itself: 401 without the plugin's own token, 500
|
|
122
141
|
when the handler fails. It never holds the request's content."""
|
|
@@ -152,8 +171,10 @@ _SCHEMAS = "#/components/schemas/{model}"
|
|
|
152
171
|
_HEALTH_CHECK_FAILED = PluginHealth(
|
|
153
172
|
status="unhealthy", reason="the health check failed"
|
|
154
173
|
)
|
|
174
|
+
_LOADING = PluginHealth(status="starting", reason="loading")
|
|
155
175
|
|
|
156
176
|
|
|
177
|
+
@overload
|
|
157
178
|
def create_plugin_app(
|
|
158
179
|
manifest: Manifest,
|
|
159
180
|
settings_cls: type[SettingsT],
|
|
@@ -163,6 +184,34 @@ def create_plugin_app(
|
|
|
163
184
|
sync: SyncHandler[SettingsT] | None = None,
|
|
164
185
|
lifespan: Lifespan[FastAPI] | None = None,
|
|
165
186
|
health: HealthCheck | None = None,
|
|
187
|
+
load: None = None,
|
|
188
|
+
) -> FastAPI: ...
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@overload
|
|
192
|
+
def create_plugin_app(
|
|
193
|
+
manifest: Manifest,
|
|
194
|
+
settings_cls: type[SettingsT],
|
|
195
|
+
*,
|
|
196
|
+
load: Loader[SettingsT, LoadedT],
|
|
197
|
+
analyze: LoadedAnalyzeHandler[SettingsT, LoadedT],
|
|
198
|
+
respond: RespondHandler[SettingsT] | None = None,
|
|
199
|
+
sync: SyncHandler[SettingsT] | None = None,
|
|
200
|
+
lifespan: Lifespan[FastAPI] | None = None,
|
|
201
|
+
health: HealthCheck | None = None,
|
|
202
|
+
) -> FastAPI: ...
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def create_plugin_app(
|
|
206
|
+
manifest: Manifest,
|
|
207
|
+
settings_cls: type[SettingsT],
|
|
208
|
+
*,
|
|
209
|
+
analyze: Callable[..., AnalyzerVerdict | Awaitable[AnalyzerVerdict]] | None = None,
|
|
210
|
+
respond: RespondHandler[SettingsT] | None = None,
|
|
211
|
+
sync: SyncHandler[SettingsT] | None = None,
|
|
212
|
+
lifespan: Lifespan[FastAPI] | None = None,
|
|
213
|
+
health: HealthCheck | None = None,
|
|
214
|
+
load: Loader[SettingsT, Any] | None = None,
|
|
166
215
|
) -> FastAPI:
|
|
167
216
|
"""Build the plugin's app; the settings are read once, here.
|
|
168
217
|
|
|
@@ -181,6 +230,16 @@ def create_plugin_app(
|
|
|
181
230
|
``PluginHealth``, makes the plugin ``unhealthy`` for a fixed reason, and
|
|
182
231
|
is logged as the handlers are: never with the exception's message.
|
|
183
232
|
|
|
233
|
+
``load(settings)``, a plain function, loads what the analyzer needs (a
|
|
234
|
+
model) once the app starts, after ``lifespan``'s startup, in a daemon
|
|
235
|
+
thread: the app answers meanwhile. Until it returns, the plugin is
|
|
236
|
+
``starting`` (``loading``) and ``/internal/analyze`` answers ``unknown``;
|
|
237
|
+
when it raises, ``unhealthy`` for good, with the exception's type
|
|
238
|
+
(``the load failed (OSError)``), logged as the handlers' failures are.
|
|
239
|
+
Then ``analyze(request, settings, loaded)`` gets what it returned, and
|
|
240
|
+
``health``, if given, reports the plugin's state. ``load`` needs
|
|
241
|
+
``analyze``, and may not be async, or ``ValueError`` is raised.
|
|
242
|
+
|
|
184
243
|
``POST /internal/sync`` answers 202 at once and runs ``sync`` in the
|
|
185
244
|
background. While a sync runs, another call starts nothing and answers
|
|
186
245
|
202 with the running sync's state.
|
|
@@ -197,6 +256,17 @@ def create_plugin_app(
|
|
|
197
256
|
"mounts it as /run/secrets/spot_plugin_token; outside a container, set "
|
|
198
257
|
"SPOT_PLUGIN_TOKEN."
|
|
199
258
|
)
|
|
259
|
+
if load is not None:
|
|
260
|
+
if analyze is None:
|
|
261
|
+
raise ValueError(
|
|
262
|
+
"load is given, but no analyze handler to get what it loads"
|
|
263
|
+
)
|
|
264
|
+
if _is_async(load):
|
|
265
|
+
raise ValueError("load runs in a thread: a plain function, not async")
|
|
266
|
+
loading = _BackgroundLoad(load, settings)
|
|
267
|
+
lifespan = loading.lifespan(lifespan)
|
|
268
|
+
health = loading.health(health)
|
|
269
|
+
analyze = loading.analyze(analyze)
|
|
200
270
|
app = _PluginApp(
|
|
201
271
|
title=manifest.id,
|
|
202
272
|
version=manifest.version,
|
|
@@ -204,13 +274,10 @@ def create_plugin_app(
|
|
|
204
274
|
lifespan=lifespan,
|
|
205
275
|
)
|
|
206
276
|
|
|
207
|
-
@app.get("/health")
|
|
208
|
-
async def health_route() ->
|
|
277
|
+
@app.get("/health", response_model_exclude_none=True)
|
|
278
|
+
async def health_route() -> HealthAnswer:
|
|
209
279
|
state = await _health(health)
|
|
210
|
-
|
|
211
|
-
if state.reason is not None:
|
|
212
|
-
answer["reason"] = state.reason
|
|
213
|
-
return answer
|
|
280
|
+
return HealthAnswer(**state.model_dump(), version=manifest.version)
|
|
214
281
|
|
|
215
282
|
@app.get("/manifest")
|
|
216
283
|
async def get_manifest() -> Manifest:
|
|
@@ -346,6 +413,75 @@ def _check_kinds(manifest: Manifest, handlers: dict[str, object]) -> None:
|
|
|
346
413
|
)
|
|
347
414
|
|
|
348
415
|
|
|
416
|
+
class _BackgroundLoad(Generic[LoadedT]):
|
|
417
|
+
"""Runs ``load`` in a daemon thread once the app starts, and holds what it
|
|
418
|
+
returned; the plugin's state is ``starting`` until then."""
|
|
419
|
+
|
|
420
|
+
_loaded: LoadedT
|
|
421
|
+
|
|
422
|
+
def __init__(self, load: Loader[Any, LoadedT], settings: PluginSettings) -> None:
|
|
423
|
+
self._load = load
|
|
424
|
+
self._settings = settings
|
|
425
|
+
self.state = _LOADING
|
|
426
|
+
|
|
427
|
+
def lifespan(self, lifespan: Lifespan[FastAPI] | None) -> Lifespan[FastAPI]:
|
|
428
|
+
"""``lifespan``, with the load started after its startup: once the
|
|
429
|
+
plugin has set its logging up, say."""
|
|
430
|
+
|
|
431
|
+
@asynccontextmanager
|
|
432
|
+
async def loading(app: FastAPI) -> AsyncIterator[Any]:
|
|
433
|
+
if lifespan is None:
|
|
434
|
+
self._start()
|
|
435
|
+
yield None
|
|
436
|
+
return
|
|
437
|
+
async with lifespan(app) as state:
|
|
438
|
+
self._start()
|
|
439
|
+
yield state
|
|
440
|
+
|
|
441
|
+
return loading
|
|
442
|
+
|
|
443
|
+
def health(self, check: HealthCheck | None) -> HealthCheck:
|
|
444
|
+
"""The plugin's state: the load's until it succeeded, then ``check``'s."""
|
|
445
|
+
|
|
446
|
+
async def health() -> PluginHealth:
|
|
447
|
+
if self.state.status != "healthy":
|
|
448
|
+
return self.state
|
|
449
|
+
return await _health(check)
|
|
450
|
+
|
|
451
|
+
return health
|
|
452
|
+
|
|
453
|
+
def analyze(
|
|
454
|
+
self, handler: LoadedAnalyzeHandler[Any, LoadedT]
|
|
455
|
+
) -> AnalyzeHandler[Any]:
|
|
456
|
+
"""``handler``, given what the load returned; ``unknown`` until then."""
|
|
457
|
+
|
|
458
|
+
async def analyze(
|
|
459
|
+
request: AnalyzeRequest, settings: PluginSettings
|
|
460
|
+
) -> AnalyzerVerdict:
|
|
461
|
+
state = self.state
|
|
462
|
+
if state.status != "healthy":
|
|
463
|
+
return AnalyzerVerdict.unknown(state.reason or state.status)
|
|
464
|
+
verdict: AnalyzerVerdict = await _run(
|
|
465
|
+
handler, request, settings, self._loaded
|
|
466
|
+
)
|
|
467
|
+
return verdict
|
|
468
|
+
|
|
469
|
+
return analyze
|
|
470
|
+
|
|
471
|
+
def _start(self) -> None:
|
|
472
|
+
threading.Thread(target=self._run, name="load", daemon=True).start()
|
|
473
|
+
|
|
474
|
+
def _run(self) -> None:
|
|
475
|
+
try:
|
|
476
|
+
self._loaded = self._load(self._settings)
|
|
477
|
+
except Exception as error:
|
|
478
|
+
_log_failure("load", error)
|
|
479
|
+
reason = f"the load failed ({type(error).__name__})"
|
|
480
|
+
self.state = PluginHealth(status="unhealthy", reason=reason)
|
|
481
|
+
else:
|
|
482
|
+
self.state = _HEALTHY
|
|
483
|
+
|
|
484
|
+
|
|
349
485
|
class _SyncRunner:
|
|
350
486
|
"""Runs the sync handler in the background, one run at a time."""
|
|
351
487
|
|
|
@@ -443,18 +579,20 @@ async def _parse(request: Request, model: type[ModelT]) -> ModelT:
|
|
|
443
579
|
|
|
444
580
|
|
|
445
581
|
async def _run(handler: Callable[..., Any], *args: Any) -> Any:
|
|
446
|
-
"""Await an async handler; run any other in a worker thread, in this context.
|
|
447
|
-
|
|
448
|
-
A handler is async when it is an ``async def`` function, or an object whose
|
|
449
|
-
``__call__`` is one, as FastAPI tells them apart.
|
|
450
|
-
"""
|
|
451
|
-
if inspect.iscoroutinefunction(handler) or inspect.iscoroutinefunction(
|
|
452
|
-
getattr(handler, "__call__", None)
|
|
453
|
-
):
|
|
582
|
+
"""Await an async handler; run any other in a worker thread, in this context."""
|
|
583
|
+
if _is_async(handler):
|
|
454
584
|
return await handler(*args)
|
|
455
585
|
return await run_in_threadpool(handler, *args)
|
|
456
586
|
|
|
457
587
|
|
|
588
|
+
def _is_async(handler: Callable[..., Any]) -> bool:
|
|
589
|
+
"""Whether ``handler`` is an ``async def`` function, or an object whose
|
|
590
|
+
``__call__`` is one, as FastAPI tells them apart."""
|
|
591
|
+
return inspect.iscoroutinefunction(handler) or inspect.iscoroutinefunction(
|
|
592
|
+
getattr(handler, "__call__", None)
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
|
|
458
596
|
@contextmanager
|
|
459
597
|
def _correlated(request: Request) -> Iterator[None]:
|
|
460
598
|
"""Make the call's ``X-Correlation-ID`` the one the clients send."""
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"""A model's files, from a local path or an archive downloaded once.
|
|
2
|
+
|
|
3
|
+
``fetch(url)`` turns what a setting names into a local path: a local path as
|
|
4
|
+
it is, or the directory an http(s) archive (``.tar.gz``, ``.tgz``, ``.tar`` or
|
|
5
|
+
``.zip``) is extracted into. An archive is downloaded once into a cache, one
|
|
6
|
+
directory per URL. It is downloaded and extracted in a temporary directory
|
|
7
|
+
beside the cache, then renamed into place: a stop half-way leaves nothing that
|
|
8
|
+
the next start would take for the model.
|
|
9
|
+
|
|
10
|
+
A URL's userinfo, query and fragment, where its credentials or a presigned
|
|
11
|
+
URL's signature are, never reach a log line or an error.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import hashlib
|
|
17
|
+
import re
|
|
18
|
+
import shutil
|
|
19
|
+
import stat
|
|
20
|
+
import tarfile
|
|
21
|
+
import tempfile
|
|
22
|
+
import urllib.error
|
|
23
|
+
import urllib.request
|
|
24
|
+
import zipfile
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import IO, Literal
|
|
27
|
+
from urllib.parse import urlsplit
|
|
28
|
+
|
|
29
|
+
from .logging import get_logger
|
|
30
|
+
from .urls import redact
|
|
31
|
+
|
|
32
|
+
logger = get_logger(__name__)
|
|
33
|
+
|
|
34
|
+
CACHE = Path.home() / ".cache" / "spot-sdk" / "artifacts"
|
|
35
|
+
"""Where ``fetch`` keeps archives by default, one directory per URL."""
|
|
36
|
+
|
|
37
|
+
# How an archive is opened, by the end of its URL's path: a tarfile mode, or zip
|
|
38
|
+
_TarMode = Literal["r:gz", "r:"]
|
|
39
|
+
_ArchiveFormat = _TarMode | Literal["zip"]
|
|
40
|
+
_FORMATS: dict[str, _ArchiveFormat] = {
|
|
41
|
+
".tar.gz": "r:gz",
|
|
42
|
+
".tgz": "r:gz",
|
|
43
|
+
".tar": "r:",
|
|
44
|
+
".zip": "zip",
|
|
45
|
+
}
|
|
46
|
+
# A URL's scheme (RFC 3986): a local path has none
|
|
47
|
+
_SCHEME = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*:")
|
|
48
|
+
# How long the server may stay silent during the download, in seconds
|
|
49
|
+
_TIMEOUT_S = 60
|
|
50
|
+
# The file types a zip member may have: none given, a file, a directory
|
|
51
|
+
_ZIP_KINDS = (0, stat.S_IFREG, stat.S_IFDIR)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class FetchError(Exception):
|
|
55
|
+
"""An archive could not be fetched. The message names the URL without its
|
|
56
|
+
userinfo, query and fragment, and why: of a failed download, the error's
|
|
57
|
+
type only, as its text can quote the whole URL."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def fetch(url: str, *, cache: Path = CACHE) -> Path:
|
|
61
|
+
"""A local path for ``url``: itself when it is no URL, else the
|
|
62
|
+
directory its archive is extracted into, under ``cache``.
|
|
63
|
+
|
|
64
|
+
A URL is anything with a scheme, or that starts with ``//``. It must be
|
|
65
|
+
one ``spot_sdk.urls.redact`` can show safely, be http or https, name a
|
|
66
|
+
host, hold no credentials (a presigned URL gives access), and its path
|
|
67
|
+
end with ``.tar.gz``, ``.tgz``, ``.tar`` or ``.zip``. The archive is
|
|
68
|
+
downloaded once: the next call with the same URL returns the same
|
|
69
|
+
directory at once, and so does a call that finds another process put it
|
|
70
|
+
in place meanwhile. Only files and directories are extracted: a member
|
|
71
|
+
that would land outside the directory, a link or a special file is
|
|
72
|
+
refused, and nothing is kept. ``FetchError`` says why, and is logged at
|
|
73
|
+
ERROR, as a plugin logs a failed load by its type only.
|
|
74
|
+
"""
|
|
75
|
+
if not (_SCHEME.match(url) or url.startswith("//")):
|
|
76
|
+
return Path(url)
|
|
77
|
+
try:
|
|
78
|
+
shown = redact(url)
|
|
79
|
+
except ValueError as error:
|
|
80
|
+
# Its text quotes nothing of the URL
|
|
81
|
+
raise _failure(f"fetch refuses the URL: {error}") from None
|
|
82
|
+
parts = urlsplit(url)
|
|
83
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
84
|
+
raise _failure(f"{shown}: not an http or https URL with a host")
|
|
85
|
+
if "@" in parts.netloc:
|
|
86
|
+
raise _failure(
|
|
87
|
+
f"{shown}: a URL's credentials are never sent: give access with a "
|
|
88
|
+
"presigned URL"
|
|
89
|
+
)
|
|
90
|
+
archive_format = _archive_format(parts.path)
|
|
91
|
+
if archive_format is None:
|
|
92
|
+
raise _failure(f"{shown}: not a .tar.gz, .tgz, .tar or .zip archive")
|
|
93
|
+
entry = cache / hashlib.sha256(url.encode()).hexdigest()
|
|
94
|
+
if entry.is_dir():
|
|
95
|
+
return entry
|
|
96
|
+
cache.mkdir(parents=True, exist_ok=True)
|
|
97
|
+
with tempfile.TemporaryDirectory(prefix=".", dir=cache) as temporary:
|
|
98
|
+
archive = Path(temporary) / "archive"
|
|
99
|
+
extracted = Path(temporary) / "extracted"
|
|
100
|
+
logger.info("Downloading %s", shown)
|
|
101
|
+
try:
|
|
102
|
+
_download(url, archive)
|
|
103
|
+
except Exception as error:
|
|
104
|
+
# The type only: the error's text can quote the URL, credentials
|
|
105
|
+
# included (http.client's InvalidURL does)
|
|
106
|
+
raise _failure(
|
|
107
|
+
f"{shown} could not be downloaded ({type(error).__name__})"
|
|
108
|
+
) from None
|
|
109
|
+
try:
|
|
110
|
+
_extract(archive, extracted, archive_format)
|
|
111
|
+
except Exception as error:
|
|
112
|
+
# A local file's error: its text holds nothing of the URL
|
|
113
|
+
raise _failure(f"{shown} could not be extracted: {error}") from error
|
|
114
|
+
try:
|
|
115
|
+
extracted.rename(entry)
|
|
116
|
+
except OSError as error:
|
|
117
|
+
# Another process put the same URL's archive in place meanwhile
|
|
118
|
+
if not entry.is_dir():
|
|
119
|
+
raise _failure(f"{shown} could not be cached: {error}") from error
|
|
120
|
+
logger.info("Extracted %s into %s", shown, entry)
|
|
121
|
+
return entry
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _failure(message: str) -> FetchError:
|
|
125
|
+
"""``FetchError(message)``, logged."""
|
|
126
|
+
logger.error("%s", message)
|
|
127
|
+
return FetchError(message)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _archive_format(path: str) -> _ArchiveFormat | None:
|
|
131
|
+
"""The archive's format, by the end of the URL's path; ``None`` for
|
|
132
|
+
another file."""
|
|
133
|
+
name = path.lower()
|
|
134
|
+
for suffix, archive_format in _FORMATS.items():
|
|
135
|
+
if name.endswith(suffix):
|
|
136
|
+
return archive_format
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _download(url: str, archive: Path) -> None:
|
|
141
|
+
try:
|
|
142
|
+
answer = urllib.request.urlopen(url, timeout=_TIMEOUT_S)
|
|
143
|
+
except urllib.error.HTTPError as error:
|
|
144
|
+
error.close() # it holds the server's answer, and its connection
|
|
145
|
+
raise
|
|
146
|
+
with answer, archive.open("wb") as file:
|
|
147
|
+
shutil.copyfileobj(answer, file)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _extract(archive: Path, target: Path, archive_format: _ArchiveFormat) -> None:
|
|
151
|
+
"""Extract the archive's files and directories into ``target``."""
|
|
152
|
+
target.mkdir()
|
|
153
|
+
if archive_format == "zip":
|
|
154
|
+
_extract_zip(archive, target)
|
|
155
|
+
else:
|
|
156
|
+
_extract_tar(archive, target, archive_format)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _extract_zip(archive: Path, target: Path) -> None:
|
|
160
|
+
with zipfile.ZipFile(archive) as zipped:
|
|
161
|
+
for info in zipped.infolist():
|
|
162
|
+
path = _destination(target, info.filename)
|
|
163
|
+
# The file type, as the member's external attributes give it: none
|
|
164
|
+
# from a zip made elsewhere than Unix, which holds files only
|
|
165
|
+
if stat.S_IFMT(info.external_attr >> 16) not in _ZIP_KINDS:
|
|
166
|
+
raise _not_a_file(info.filename)
|
|
167
|
+
if info.is_dir():
|
|
168
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
169
|
+
continue
|
|
170
|
+
with zipped.open(info) as data:
|
|
171
|
+
_write(path, data)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _extract_tar(archive: Path, target: Path, mode: _TarMode) -> None:
|
|
175
|
+
with tarfile.open(archive, mode) as tarred:
|
|
176
|
+
for member in tarred:
|
|
177
|
+
path = _destination(target, member.name)
|
|
178
|
+
if member.isdir():
|
|
179
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
180
|
+
continue
|
|
181
|
+
data = tarred.extractfile(member) if member.isfile() else None
|
|
182
|
+
if data is None:
|
|
183
|
+
raise _not_a_file(member.name)
|
|
184
|
+
with data:
|
|
185
|
+
_write(path, data)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _not_a_file(name: str) -> ValueError:
|
|
189
|
+
return ValueError(f"the member {name!r} is neither a file nor a directory")
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _destination(target: Path, name: str) -> Path:
|
|
193
|
+
"""Where the member ``name`` lands in ``target``; refused outside it."""
|
|
194
|
+
path = (target / name).resolve()
|
|
195
|
+
if not path.is_relative_to(target.resolve()):
|
|
196
|
+
raise ValueError(
|
|
197
|
+
f"the member {name!r} would land outside the archive's directory"
|
|
198
|
+
)
|
|
199
|
+
return path
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _write(path: Path, data: IO[bytes]) -> None:
|
|
203
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
204
|
+
with path.open("wb") as file:
|
|
205
|
+
shutil.copyfileobj(data, file)
|
|
@@ -14,6 +14,7 @@ content.
|
|
|
14
14
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
|
+
import asyncio
|
|
17
18
|
import base64
|
|
18
19
|
import json
|
|
19
20
|
from contextvars import ContextVar
|
|
@@ -29,6 +30,11 @@ from .orchestrator import OrchestrationResult
|
|
|
29
30
|
from .retriever import IngestionPriority, IngestionRequest, IngestionResult
|
|
30
31
|
from .settings import PluginSettings
|
|
31
32
|
|
|
33
|
+
# The bytes of a raw message encoded at a time: a few milliseconds of a thread
|
|
34
|
+
# holding the GIL, between which the event loop runs. A multiple of 3, so that
|
|
35
|
+
# the pieces' base64 texts join into the message's.
|
|
36
|
+
_INGEST_PIECE = 768 * 1024
|
|
37
|
+
|
|
32
38
|
ModelT = TypeVar("ModelT", bound=BaseModel)
|
|
33
39
|
|
|
34
40
|
correlation_id: ContextVar[str | None] = ContextVar("spot_correlation_id", default=None)
|
|
@@ -76,18 +82,22 @@ async def spot_request(
|
|
|
76
82
|
path: str,
|
|
77
83
|
*,
|
|
78
84
|
json: Any = None,
|
|
85
|
+
content: bytes | None = None,
|
|
79
86
|
timeout: float,
|
|
80
87
|
transport: httpx.AsyncBaseTransport | None = None,
|
|
81
88
|
) -> httpx.Response:
|
|
82
|
-
"""Call a SPOT service as a plugin; an error status raises ``SpotAPIError``.
|
|
89
|
+
"""Call a SPOT service as a plugin; an error status raises ``SpotAPIError``.
|
|
90
|
+
|
|
91
|
+
The body is ``json`` encoded by :func:`json_body`, or ``content``, a JSON
|
|
92
|
+
body the caller has already encoded."""
|
|
83
93
|
headers = {"Authorization": f"Bearer {token.get_secret_value()}"}
|
|
84
94
|
current = correlation_id.get()
|
|
85
95
|
if current is not None:
|
|
86
96
|
headers["X-Correlation-ID"] = current
|
|
87
|
-
content = None
|
|
88
97
|
if json is not None:
|
|
89
|
-
headers["Content-Type"] = "application/json"
|
|
90
98
|
content = json_body(json)
|
|
99
|
+
if content is not None:
|
|
100
|
+
headers["Content-Type"] = "application/json"
|
|
91
101
|
async with httpx.AsyncClient(timeout=timeout, transport=transport) as client:
|
|
92
102
|
response = await client.request(
|
|
93
103
|
method, base_url + path, content=content, headers=headers
|
|
@@ -118,6 +128,36 @@ def ingestion_request(
|
|
|
118
128
|
)
|
|
119
129
|
|
|
120
130
|
|
|
131
|
+
def ingestion_body(
|
|
132
|
+
raw: bytes,
|
|
133
|
+
*,
|
|
134
|
+
envelope: Envelope | None = None,
|
|
135
|
+
workflow_id: str | None = None,
|
|
136
|
+
priority: IngestionPriority = "normal",
|
|
137
|
+
wait_for_verdict: bool = False,
|
|
138
|
+
timeout_ms: int = 30_000,
|
|
139
|
+
) -> bytes:
|
|
140
|
+
"""The JSON bytes of ``ingestion_request(raw, ...)``, built so that a large
|
|
141
|
+
message holds the GIL for a few milliseconds at a time: the message is
|
|
142
|
+
encoded in pieces and spliced into the JSON, since the base64 alphabet
|
|
143
|
+
needs no escaping. The request is validated as ``ingestion_request``
|
|
144
|
+
does, on the first piece: an empty message is refused."""
|
|
145
|
+
pieces = [
|
|
146
|
+
base64.b64encode(raw[start : start + _INGEST_PIECE])
|
|
147
|
+
for start in range(0, len(raw), _INGEST_PIECE)
|
|
148
|
+
]
|
|
149
|
+
request = IngestionRequest(
|
|
150
|
+
raw=(pieces[0] if pieces else b"").decode("ascii"),
|
|
151
|
+
envelope=envelope,
|
|
152
|
+
workflow_id=workflow_id,
|
|
153
|
+
priority=priority,
|
|
154
|
+
wait_for_verdict=wait_for_verdict,
|
|
155
|
+
timeout_ms=timeout_ms,
|
|
156
|
+
)
|
|
157
|
+
others = json_body(request.model_dump(mode="json", exclude={"raw"}))
|
|
158
|
+
return b"".join([b'{"raw": "', *pieces, b'", ', others[1:]])
|
|
159
|
+
|
|
160
|
+
|
|
121
161
|
class SpotClient:
|
|
122
162
|
"""The api-gateway's plugin routes, called with the plugin's own token."""
|
|
123
163
|
|
|
@@ -161,8 +201,14 @@ class SpotClient:
|
|
|
161
201
|
With ``wait_for_verdict``, the gateway answers once the workflow
|
|
162
202
|
decides, or with status ``timeout`` after ``timeout_ms``; the call
|
|
163
203
|
then waits that long on top of the client's timeout.
|
|
204
|
+
|
|
205
|
+
The whole request body, base64 encoding and JSON included, is built in
|
|
206
|
+
a worker thread, in pieces of a few milliseconds, whatever the
|
|
207
|
+
message's size: a large message does not hold up the caller's event
|
|
208
|
+
loop. Only sending the bytes is left to it.
|
|
164
209
|
"""
|
|
165
|
-
|
|
210
|
+
content = await asyncio.to_thread(
|
|
211
|
+
ingestion_body,
|
|
166
212
|
raw,
|
|
167
213
|
envelope=envelope,
|
|
168
214
|
workflow_id=workflow_id,
|
|
@@ -174,7 +220,7 @@ class SpotClient:
|
|
|
174
220
|
response = await self._call(
|
|
175
221
|
"POST",
|
|
176
222
|
"/internal/ingest",
|
|
177
|
-
|
|
223
|
+
content=content,
|
|
178
224
|
timeout=timeout,
|
|
179
225
|
)
|
|
180
226
|
return read_answer(IngestionResult, response, "POST", "/internal/ingest")
|
|
@@ -198,14 +244,19 @@ class SpotClient:
|
|
|
198
244
|
return read_answer(OrchestrationResult, response, "GET", path)
|
|
199
245
|
|
|
200
246
|
async def _call(
|
|
201
|
-
self,
|
|
247
|
+
self,
|
|
248
|
+
method: str,
|
|
249
|
+
path: str,
|
|
250
|
+
*,
|
|
251
|
+
content: bytes | None = None,
|
|
252
|
+
timeout: float | None = None,
|
|
202
253
|
) -> httpx.Response:
|
|
203
254
|
return await spot_request(
|
|
204
255
|
self.base_url,
|
|
205
256
|
self.token,
|
|
206
257
|
method,
|
|
207
258
|
path,
|
|
208
|
-
|
|
259
|
+
content=content,
|
|
209
260
|
timeout=timeout or self.timeout,
|
|
210
261
|
transport=self._transport,
|
|
211
262
|
)
|
|
@@ -104,7 +104,8 @@ class Anomaly(BaseModel):
|
|
|
104
104
|
location: str | None = Field(
|
|
105
105
|
default=None,
|
|
106
106
|
description='"part:1.2", "header:Subject@1", "param:Content-Type.boundary@1.2",'
|
|
107
|
-
' ..., "envelope", "source": at most 200
|
|
107
|
+
' "url:3" (the view\'s URL index), ..., "envelope", "source": at most 200'
|
|
108
|
+
" bytes of JSON",
|
|
108
109
|
)
|
|
109
110
|
detail: str | None = Field(
|
|
110
111
|
default=None,
|