spot-sdk-python 2.0.0b6__tar.gz → 2.0.0b7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/PKG-INFO +2 -2
  2. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/README.md +1 -1
  3. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/pyproject.toml +1 -1
  4. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/app.py +155 -17
  5. spot_sdk_python-2.0.0b7/spot_sdk/artifacts.py +205 -0
  6. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/clients.py +58 -7
  7. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/email.py +2 -1
  8. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/manifest.py +91 -1
  9. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/mime.py +71 -8
  10. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/README.md +3 -1
  11. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/__init__.py +4 -1
  12. spot_sdk_python-2.0.0b7/spot_sdk/testing/contract.py +219 -0
  13. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/fake_knowledge_client.py +7 -3
  14. spot_sdk_python-2.0.0b7/spot_sdk/testing/settings.py +128 -0
  15. spot_sdk_python-2.0.0b7/spot_sdk/urls.py +36 -0
  16. spot_sdk_python-2.0.0b7/spot_sdk/verdict.py +127 -0
  17. spot_sdk_python-2.0.0b6/spot_sdk/testing/contract.py +0 -51
  18. spot_sdk_python-2.0.0b6/spot_sdk/verdict.py +0 -53
  19. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/__init__.py +0 -0
  20. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/_json.py +0 -0
  21. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/config.py +0 -0
  22. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/errors.py +0 -0
  23. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/knowledge.py +0 -0
  24. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/knowledge_tags.py +0 -0
  25. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/llm.py +0 -0
  26. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/logging.py +0 -0
  27. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/orchestrator.py +0 -0
  28. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/py.typed +0 -0
  29. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/retriever.py +0 -0
  30. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/settings.py +0 -0
  31. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/signals.py +0 -0
  32. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/fake_llm.py +0 -0
  33. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/fake_spot.py +0 -0
  34. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/testing/views.py +0 -0
  35. {spot_sdk_python-2.0.0b6 → spot_sdk_python-2.0.0b7}/spot_sdk/workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: spot-sdk-python
3
- Version: 2.0.0b6
3
+ Version: 2.0.0b7
4
4
  Summary: SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers
5
5
  License: Apache-2.0
6
6
  Author: SPOT Project
@@ -50,7 +50,7 @@ plugins share:
50
50
  ## Install
51
51
 
52
52
  ```bash
53
- pip install spot-sdk-python==2.0.0b6
53
+ pip install spot-sdk-python==2.0.0b7
54
54
  ```
55
55
 
56
56
  Python 3.11 to 3.15. Version 2 implements plugin contract 2, for
@@ -26,7 +26,7 @@ plugins share:
26
26
  ## Install
27
27
 
28
28
  ```bash
29
- pip install spot-sdk-python==2.0.0b6
29
+ pip install spot-sdk-python==2.0.0b7
30
30
  ```
31
31
 
32
32
  Python 3.11 to 3.15. Version 2 implements plugin contract 2, for
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
4
4
 
5
5
  [tool.poetry]
6
6
  name = "spot-sdk-python"
7
- version = "2.0.0b6"
7
+ version = "2.0.0b7"
8
8
  description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
9
9
  authors = ["SPOT Project <spot@sonn.lu>"]
10
10
  license = "Apache-2.0"
@@ -15,6 +15,10 @@ An ``async def`` handler, or an object whose ``__call__`` is ``async def``,
15
15
  runs on the event loop; any other callable runs in a worker thread, so it may
16
16
  block. Either way the clients send the call's ``X-Correlation-ID`` on to SPOT.
17
17
 
18
+ A plugin that loads a model gives ``load``: run in a daemon thread once the app
19
+ starts, it makes the plugin ``starting`` until it returns, and the analyzer
20
+ answers ``unknown`` meanwhile; then ``analyze`` gets what it returned.
21
+
18
22
  Neither the logs nor the error answers hold the request's content: a
19
23
  handler's exception is logged by its type and place only, and an invalid
20
24
  request is answered with the fields refused, never the values sent.
@@ -29,11 +33,12 @@ from __future__ import annotations
29
33
  import asyncio
30
34
  import hmac
31
35
  import inspect
36
+ import threading
32
37
  import traceback
33
- from collections.abc import Awaitable, Callable, Iterator
34
- from contextlib import contextmanager
38
+ from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
39
+ from contextlib import asynccontextmanager, contextmanager
35
40
  from datetime import UTC, datetime
36
- from typing import Any, Literal, TypeAlias, TypeVar
41
+ from typing import Any, Generic, Literal, TypeAlias, TypeVar, overload
37
42
 
38
43
  from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
39
44
  from fastapi.responses import JSONResponse
@@ -55,6 +60,7 @@ logger = get_logger(__name__)
55
60
 
56
61
  SettingsT = TypeVar("SettingsT", bound=PluginSettings)
57
62
  ModelT = TypeVar("ModelT", bound=BaseModel)
63
+ LoadedT = TypeVar("LoadedT")
58
64
 
59
65
  AnalyzeHandler: TypeAlias = Callable[
60
66
  [AnalyzeRequest, SettingsT], AnalyzerVerdict | Awaitable[AnalyzerVerdict]
@@ -74,6 +80,12 @@ action idempotent (a ticket keyed by the job id).
74
80
  """
75
81
  SyncHandler: TypeAlias = Callable[[SettingsT], int | Awaitable[int]]
76
82
  """``sync(settings) -> int``: the number of documents written."""
83
+ Loader: TypeAlias = Callable[[SettingsT], LoadedT]
84
+ """``load(settings) -> object``: a model, say, loaded once in a daemon thread."""
85
+ LoadedAnalyzeHandler: TypeAlias = Callable[
86
+ [AnalyzeRequest, SettingsT, LoadedT], AnalyzerVerdict | Awaitable[AnalyzerVerdict]
87
+ ]
88
+ """``analyze(request, settings, loaded) -> AnalyzerVerdict``, with what ``load`` returned."""
77
89
 
78
90
 
79
91
  class SyncState(BaseModel):
@@ -117,6 +129,13 @@ class PluginHealth(BaseModel):
117
129
  )
118
130
 
119
131
 
132
+ class HealthAnswer(PluginHealth):
133
+ """``GET /health``'s body: the plugin's state and the version of its
134
+ manifest. ``reason`` is left out when there is none."""
135
+
136
+ version: str = Field(description="The manifest's version")
137
+
138
+
120
139
  class ErrorAnswer(BaseModel):
121
140
  """An error the app answers itself: 401 without the plugin's own token, 500
122
141
  when the handler fails. It never holds the request's content."""
@@ -152,8 +171,10 @@ _SCHEMAS = "#/components/schemas/{model}"
152
171
  _HEALTH_CHECK_FAILED = PluginHealth(
153
172
  status="unhealthy", reason="the health check failed"
154
173
  )
174
+ _LOADING = PluginHealth(status="starting", reason="loading")
155
175
 
156
176
 
177
+ @overload
157
178
  def create_plugin_app(
158
179
  manifest: Manifest,
159
180
  settings_cls: type[SettingsT],
@@ -163,6 +184,34 @@ def create_plugin_app(
163
184
  sync: SyncHandler[SettingsT] | None = None,
164
185
  lifespan: Lifespan[FastAPI] | None = None,
165
186
  health: HealthCheck | None = None,
187
+ load: None = None,
188
+ ) -> FastAPI: ...
189
+
190
+
191
+ @overload
192
+ def create_plugin_app(
193
+ manifest: Manifest,
194
+ settings_cls: type[SettingsT],
195
+ *,
196
+ load: Loader[SettingsT, LoadedT],
197
+ analyze: LoadedAnalyzeHandler[SettingsT, LoadedT],
198
+ respond: RespondHandler[SettingsT] | None = None,
199
+ sync: SyncHandler[SettingsT] | None = None,
200
+ lifespan: Lifespan[FastAPI] | None = None,
201
+ health: HealthCheck | None = None,
202
+ ) -> FastAPI: ...
203
+
204
+
205
+ def create_plugin_app(
206
+ manifest: Manifest,
207
+ settings_cls: type[SettingsT],
208
+ *,
209
+ analyze: Callable[..., AnalyzerVerdict | Awaitable[AnalyzerVerdict]] | None = None,
210
+ respond: RespondHandler[SettingsT] | None = None,
211
+ sync: SyncHandler[SettingsT] | None = None,
212
+ lifespan: Lifespan[FastAPI] | None = None,
213
+ health: HealthCheck | None = None,
214
+ load: Loader[SettingsT, Any] | None = None,
166
215
  ) -> FastAPI:
167
216
  """Build the plugin's app; the settings are read once, here.
168
217
 
@@ -181,6 +230,16 @@ def create_plugin_app(
181
230
  ``PluginHealth``, makes the plugin ``unhealthy`` for a fixed reason, and
182
231
  is logged as the handlers are: never with the exception's message.
183
232
 
233
+ ``load(settings)``, a plain function, loads what the analyzer needs (a
234
+ model) once the app starts, after ``lifespan``'s startup, in a daemon
235
+ thread: the app answers meanwhile. Until it returns, the plugin is
236
+ ``starting`` (``loading``) and ``/internal/analyze`` answers ``unknown``;
237
+ when it raises, ``unhealthy`` for good, with the exception's type
238
+ (``the load failed (OSError)``), logged as the handlers' failures are.
239
+ Then ``analyze(request, settings, loaded)`` gets what it returned, and
240
+ ``health``, if given, reports the plugin's state. ``load`` needs
241
+ ``analyze``, and may not be async, or ``ValueError`` is raised.
242
+
184
243
  ``POST /internal/sync`` answers 202 at once and runs ``sync`` in the
185
244
  background. While a sync runs, another call starts nothing and answers
186
245
  202 with the running sync's state.
@@ -197,6 +256,17 @@ def create_plugin_app(
197
256
  "mounts it as /run/secrets/spot_plugin_token; outside a container, set "
198
257
  "SPOT_PLUGIN_TOKEN."
199
258
  )
259
+ if load is not None:
260
+ if analyze is None:
261
+ raise ValueError(
262
+ "load is given, but no analyze handler to get what it loads"
263
+ )
264
+ if _is_async(load):
265
+ raise ValueError("load runs in a thread: a plain function, not async")
266
+ loading = _BackgroundLoad(load, settings)
267
+ lifespan = loading.lifespan(lifespan)
268
+ health = loading.health(health)
269
+ analyze = loading.analyze(analyze)
200
270
  app = _PluginApp(
201
271
  title=manifest.id,
202
272
  version=manifest.version,
@@ -204,13 +274,10 @@ def create_plugin_app(
204
274
  lifespan=lifespan,
205
275
  )
206
276
 
207
- @app.get("/health")
208
- async def health_route() -> dict[str, str]:
277
+ @app.get("/health", response_model_exclude_none=True)
278
+ async def health_route() -> HealthAnswer:
209
279
  state = await _health(health)
210
- answer = {"status": state.status, "version": manifest.version}
211
- if state.reason is not None:
212
- answer["reason"] = state.reason
213
- return answer
280
+ return HealthAnswer(**state.model_dump(), version=manifest.version)
214
281
 
215
282
  @app.get("/manifest")
216
283
  async def get_manifest() -> Manifest:
@@ -346,6 +413,75 @@ def _check_kinds(manifest: Manifest, handlers: dict[str, object]) -> None:
346
413
  )
347
414
 
348
415
 
416
+ class _BackgroundLoad(Generic[LoadedT]):
417
+ """Runs ``load`` in a daemon thread once the app starts, and holds what it
418
+ returned; the plugin's state is ``starting`` until then."""
419
+
420
+ _loaded: LoadedT
421
+
422
+ def __init__(self, load: Loader[Any, LoadedT], settings: PluginSettings) -> None:
423
+ self._load = load
424
+ self._settings = settings
425
+ self.state = _LOADING
426
+
427
+ def lifespan(self, lifespan: Lifespan[FastAPI] | None) -> Lifespan[FastAPI]:
428
+ """``lifespan``, with the load started after its startup: once the
429
+ plugin has set its logging up, say."""
430
+
431
+ @asynccontextmanager
432
+ async def loading(app: FastAPI) -> AsyncIterator[Any]:
433
+ if lifespan is None:
434
+ self._start()
435
+ yield None
436
+ return
437
+ async with lifespan(app) as state:
438
+ self._start()
439
+ yield state
440
+
441
+ return loading
442
+
443
+ def health(self, check: HealthCheck | None) -> HealthCheck:
444
+ """The plugin's state: the load's until it succeeded, then ``check``'s."""
445
+
446
+ async def health() -> PluginHealth:
447
+ if self.state.status != "healthy":
448
+ return self.state
449
+ return await _health(check)
450
+
451
+ return health
452
+
453
+ def analyze(
454
+ self, handler: LoadedAnalyzeHandler[Any, LoadedT]
455
+ ) -> AnalyzeHandler[Any]:
456
+ """``handler``, given what the load returned; ``unknown`` until then."""
457
+
458
+ async def analyze(
459
+ request: AnalyzeRequest, settings: PluginSettings
460
+ ) -> AnalyzerVerdict:
461
+ state = self.state
462
+ if state.status != "healthy":
463
+ return AnalyzerVerdict.unknown(state.reason or state.status)
464
+ verdict: AnalyzerVerdict = await _run(
465
+ handler, request, settings, self._loaded
466
+ )
467
+ return verdict
468
+
469
+ return analyze
470
+
471
+ def _start(self) -> None:
472
+ threading.Thread(target=self._run, name="load", daemon=True).start()
473
+
474
+ def _run(self) -> None:
475
+ try:
476
+ self._loaded = self._load(self._settings)
477
+ except Exception as error:
478
+ _log_failure("load", error)
479
+ reason = f"the load failed ({type(error).__name__})"
480
+ self.state = PluginHealth(status="unhealthy", reason=reason)
481
+ else:
482
+ self.state = _HEALTHY
483
+
484
+
349
485
  class _SyncRunner:
350
486
  """Runs the sync handler in the background, one run at a time."""
351
487
 
@@ -443,18 +579,20 @@ async def _parse(request: Request, model: type[ModelT]) -> ModelT:
443
579
 
444
580
 
445
581
  async def _run(handler: Callable[..., Any], *args: Any) -> Any:
446
- """Await an async handler; run any other in a worker thread, in this context.
447
-
448
- A handler is async when it is an ``async def`` function, or an object whose
449
- ``__call__`` is one, as FastAPI tells them apart.
450
- """
451
- if inspect.iscoroutinefunction(handler) or inspect.iscoroutinefunction(
452
- getattr(handler, "__call__", None)
453
- ):
582
+ """Await an async handler; run any other in a worker thread, in this context."""
583
+ if _is_async(handler):
454
584
  return await handler(*args)
455
585
  return await run_in_threadpool(handler, *args)
456
586
 
457
587
 
588
+ def _is_async(handler: Callable[..., Any]) -> bool:
589
+ """Whether ``handler`` is an ``async def`` function, or an object whose
590
+ ``__call__`` is one, as FastAPI tells them apart."""
591
+ return inspect.iscoroutinefunction(handler) or inspect.iscoroutinefunction(
592
+ getattr(handler, "__call__", None)
593
+ )
594
+
595
+
458
596
  @contextmanager
459
597
  def _correlated(request: Request) -> Iterator[None]:
460
598
  """Make the call's ``X-Correlation-ID`` the one the clients send."""
@@ -0,0 +1,205 @@
1
+ """A model's files, from a local path or an archive downloaded once.
2
+
3
+ ``fetch(url)`` turns what a setting names into a local path: a local path as
4
+ it is, or the directory an http(s) archive (``.tar.gz``, ``.tgz``, ``.tar`` or
5
+ ``.zip``) is extracted into. An archive is downloaded once into a cache, one
6
+ directory per URL. It is downloaded and extracted in a temporary directory
7
+ beside the cache, then renamed into place: a stop half-way leaves nothing that
8
+ the next start would take for the model.
9
+
10
+ A URL's userinfo, query and fragment, where its credentials or a presigned
11
+ URL's signature are, never reach a log line or an error.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import hashlib
17
+ import re
18
+ import shutil
19
+ import stat
20
+ import tarfile
21
+ import tempfile
22
+ import urllib.error
23
+ import urllib.request
24
+ import zipfile
25
+ from pathlib import Path
26
+ from typing import IO, Literal
27
+ from urllib.parse import urlsplit
28
+
29
+ from .logging import get_logger
30
+ from .urls import redact
31
+
32
+ logger = get_logger(__name__)
33
+
34
+ CACHE = Path.home() / ".cache" / "spot-sdk" / "artifacts"
35
+ """Where ``fetch`` keeps archives by default, one directory per URL."""
36
+
37
+ # How an archive is opened, by the end of its URL's path: a tarfile mode, or zip
38
+ _TarMode = Literal["r:gz", "r:"]
39
+ _ArchiveFormat = _TarMode | Literal["zip"]
40
+ _FORMATS: dict[str, _ArchiveFormat] = {
41
+ ".tar.gz": "r:gz",
42
+ ".tgz": "r:gz",
43
+ ".tar": "r:",
44
+ ".zip": "zip",
45
+ }
46
+ # A URL's scheme (RFC 3986): a local path has none
47
+ _SCHEME = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*:")
48
+ # How long the server may stay silent during the download, in seconds
49
+ _TIMEOUT_S = 60
50
+ # The file types a zip member may have: none given, a file, a directory
51
+ _ZIP_KINDS = (0, stat.S_IFREG, stat.S_IFDIR)
52
+
53
+
54
+ class FetchError(Exception):
55
+ """An archive could not be fetched. The message names the URL without its
56
+ userinfo, query and fragment, and why: of a failed download, the error's
57
+ type only, as its text can quote the whole URL."""
58
+
59
+
60
+ def fetch(url: str, *, cache: Path = CACHE) -> Path:
61
+ """A local path for ``url``: itself when it is no URL, else the
62
+ directory its archive is extracted into, under ``cache``.
63
+
64
+ A URL is anything with a scheme, or that starts with ``//``. It must be
65
+ one ``spot_sdk.urls.redact`` can show safely, be http or https, name a
66
+ host, hold no credentials (a presigned URL gives access), and its path
67
+ end with ``.tar.gz``, ``.tgz``, ``.tar`` or ``.zip``. The archive is
68
+ downloaded once: the next call with the same URL returns the same
69
+ directory at once, and so does a call that finds another process put it
70
+ in place meanwhile. Only files and directories are extracted: a member
71
+ that would land outside the directory, a link or a special file is
72
+ refused, and nothing is kept. ``FetchError`` says why, and is logged at
73
+ ERROR, as a plugin logs a failed load by its type only.
74
+ """
75
+ if not (_SCHEME.match(url) or url.startswith("//")):
76
+ return Path(url)
77
+ try:
78
+ shown = redact(url)
79
+ except ValueError as error:
80
+ # Its text quotes nothing of the URL
81
+ raise _failure(f"fetch refuses the URL: {error}") from None
82
+ parts = urlsplit(url)
83
+ if parts.scheme not in ("http", "https") or not parts.hostname:
84
+ raise _failure(f"{shown}: not an http or https URL with a host")
85
+ if "@" in parts.netloc:
86
+ raise _failure(
87
+ f"{shown}: a URL's credentials are never sent: give access with a "
88
+ "presigned URL"
89
+ )
90
+ archive_format = _archive_format(parts.path)
91
+ if archive_format is None:
92
+ raise _failure(f"{shown}: not a .tar.gz, .tgz, .tar or .zip archive")
93
+ entry = cache / hashlib.sha256(url.encode()).hexdigest()
94
+ if entry.is_dir():
95
+ return entry
96
+ cache.mkdir(parents=True, exist_ok=True)
97
+ with tempfile.TemporaryDirectory(prefix=".", dir=cache) as temporary:
98
+ archive = Path(temporary) / "archive"
99
+ extracted = Path(temporary) / "extracted"
100
+ logger.info("Downloading %s", shown)
101
+ try:
102
+ _download(url, archive)
103
+ except Exception as error:
104
+ # The type only: the error's text can quote the URL, credentials
105
+ # included (http.client's InvalidURL does)
106
+ raise _failure(
107
+ f"{shown} could not be downloaded ({type(error).__name__})"
108
+ ) from None
109
+ try:
110
+ _extract(archive, extracted, archive_format)
111
+ except Exception as error:
112
+ # A local file's error: its text holds nothing of the URL
113
+ raise _failure(f"{shown} could not be extracted: {error}") from error
114
+ try:
115
+ extracted.rename(entry)
116
+ except OSError as error:
117
+ # Another process put the same URL's archive in place meanwhile
118
+ if not entry.is_dir():
119
+ raise _failure(f"{shown} could not be cached: {error}") from error
120
+ logger.info("Extracted %s into %s", shown, entry)
121
+ return entry
122
+
123
+
124
+ def _failure(message: str) -> FetchError:
125
+ """``FetchError(message)``, logged."""
126
+ logger.error("%s", message)
127
+ return FetchError(message)
128
+
129
+
130
+ def _archive_format(path: str) -> _ArchiveFormat | None:
131
+ """The archive's format, by the end of the URL's path; ``None`` for
132
+ another file."""
133
+ name = path.lower()
134
+ for suffix, archive_format in _FORMATS.items():
135
+ if name.endswith(suffix):
136
+ return archive_format
137
+ return None
138
+
139
+
140
+ def _download(url: str, archive: Path) -> None:
141
+ try:
142
+ answer = urllib.request.urlopen(url, timeout=_TIMEOUT_S)
143
+ except urllib.error.HTTPError as error:
144
+ error.close() # it holds the server's answer, and its connection
145
+ raise
146
+ with answer, archive.open("wb") as file:
147
+ shutil.copyfileobj(answer, file)
148
+
149
+
150
+ def _extract(archive: Path, target: Path, archive_format: _ArchiveFormat) -> None:
151
+ """Extract the archive's files and directories into ``target``."""
152
+ target.mkdir()
153
+ if archive_format == "zip":
154
+ _extract_zip(archive, target)
155
+ else:
156
+ _extract_tar(archive, target, archive_format)
157
+
158
+
159
+ def _extract_zip(archive: Path, target: Path) -> None:
160
+ with zipfile.ZipFile(archive) as zipped:
161
+ for info in zipped.infolist():
162
+ path = _destination(target, info.filename)
163
+ # The file type, as the member's external attributes give it: none
164
+ # from a zip made elsewhere than Unix, which holds files only
165
+ if stat.S_IFMT(info.external_attr >> 16) not in _ZIP_KINDS:
166
+ raise _not_a_file(info.filename)
167
+ if info.is_dir():
168
+ path.mkdir(parents=True, exist_ok=True)
169
+ continue
170
+ with zipped.open(info) as data:
171
+ _write(path, data)
172
+
173
+
174
+ def _extract_tar(archive: Path, target: Path, mode: _TarMode) -> None:
175
+ with tarfile.open(archive, mode) as tarred:
176
+ for member in tarred:
177
+ path = _destination(target, member.name)
178
+ if member.isdir():
179
+ path.mkdir(parents=True, exist_ok=True)
180
+ continue
181
+ data = tarred.extractfile(member) if member.isfile() else None
182
+ if data is None:
183
+ raise _not_a_file(member.name)
184
+ with data:
185
+ _write(path, data)
186
+
187
+
188
+ def _not_a_file(name: str) -> ValueError:
189
+ return ValueError(f"the member {name!r} is neither a file nor a directory")
190
+
191
+
192
+ def _destination(target: Path, name: str) -> Path:
193
+ """Where the member ``name`` lands in ``target``; refused outside it."""
194
+ path = (target / name).resolve()
195
+ if not path.is_relative_to(target.resolve()):
196
+ raise ValueError(
197
+ f"the member {name!r} would land outside the archive's directory"
198
+ )
199
+ return path
200
+
201
+
202
+ def _write(path: Path, data: IO[bytes]) -> None:
203
+ path.parent.mkdir(parents=True, exist_ok=True)
204
+ with path.open("wb") as file:
205
+ shutil.copyfileobj(data, file)
@@ -14,6 +14,7 @@ content.
14
14
 
15
15
  from __future__ import annotations
16
16
 
17
+ import asyncio
17
18
  import base64
18
19
  import json
19
20
  from contextvars import ContextVar
@@ -29,6 +30,11 @@ from .orchestrator import OrchestrationResult
29
30
  from .retriever import IngestionPriority, IngestionRequest, IngestionResult
30
31
  from .settings import PluginSettings
31
32
 
33
+ # The bytes of a raw message encoded at a time: a few milliseconds of a thread
34
+ # holding the GIL, between which the event loop runs. A multiple of 3, so that
35
+ # the pieces' base64 texts join into the message's.
36
+ _INGEST_PIECE = 768 * 1024
37
+
32
38
  ModelT = TypeVar("ModelT", bound=BaseModel)
33
39
 
34
40
  correlation_id: ContextVar[str | None] = ContextVar("spot_correlation_id", default=None)
@@ -76,18 +82,22 @@ async def spot_request(
76
82
  path: str,
77
83
  *,
78
84
  json: Any = None,
85
+ content: bytes | None = None,
79
86
  timeout: float,
80
87
  transport: httpx.AsyncBaseTransport | None = None,
81
88
  ) -> httpx.Response:
82
- """Call a SPOT service as a plugin; an error status raises ``SpotAPIError``."""
89
+ """Call a SPOT service as a plugin; an error status raises ``SpotAPIError``.
90
+
91
+ The body is ``json`` encoded by :func:`json_body`, or ``content``, a JSON
92
+ body the caller has already encoded."""
83
93
  headers = {"Authorization": f"Bearer {token.get_secret_value()}"}
84
94
  current = correlation_id.get()
85
95
  if current is not None:
86
96
  headers["X-Correlation-ID"] = current
87
- content = None
88
97
  if json is not None:
89
- headers["Content-Type"] = "application/json"
90
98
  content = json_body(json)
99
+ if content is not None:
100
+ headers["Content-Type"] = "application/json"
91
101
  async with httpx.AsyncClient(timeout=timeout, transport=transport) as client:
92
102
  response = await client.request(
93
103
  method, base_url + path, content=content, headers=headers
@@ -118,6 +128,36 @@ def ingestion_request(
118
128
  )
119
129
 
120
130
 
131
+ def ingestion_body(
132
+ raw: bytes,
133
+ *,
134
+ envelope: Envelope | None = None,
135
+ workflow_id: str | None = None,
136
+ priority: IngestionPriority = "normal",
137
+ wait_for_verdict: bool = False,
138
+ timeout_ms: int = 30_000,
139
+ ) -> bytes:
140
+ """The JSON bytes of ``ingestion_request(raw, ...)``, built so that a large
141
+ message holds the GIL for a few milliseconds at a time: the message is
142
+ encoded in pieces and spliced into the JSON, since the base64 alphabet
143
+ needs no escaping. The request is validated as ``ingestion_request``
144
+ does, on the first piece: an empty message is refused."""
145
+ pieces = [
146
+ base64.b64encode(raw[start : start + _INGEST_PIECE])
147
+ for start in range(0, len(raw), _INGEST_PIECE)
148
+ ]
149
+ request = IngestionRequest(
150
+ raw=(pieces[0] if pieces else b"").decode("ascii"),
151
+ envelope=envelope,
152
+ workflow_id=workflow_id,
153
+ priority=priority,
154
+ wait_for_verdict=wait_for_verdict,
155
+ timeout_ms=timeout_ms,
156
+ )
157
+ others = json_body(request.model_dump(mode="json", exclude={"raw"}))
158
+ return b"".join([b'{"raw": "', *pieces, b'", ', others[1:]])
159
+
160
+
121
161
  class SpotClient:
122
162
  """The api-gateway's plugin routes, called with the plugin's own token."""
123
163
 
@@ -161,8 +201,14 @@ class SpotClient:
161
201
  With ``wait_for_verdict``, the gateway answers once the workflow
162
202
  decides, or with status ``timeout`` after ``timeout_ms``; the call
163
203
  then waits that long on top of the client's timeout.
204
+
205
+ The whole request body, base64 encoding and JSON included, is built in
206
+ a worker thread, in pieces of a few milliseconds, whatever the
207
+ message's size: a large message does not hold up the caller's event
208
+ loop. Only sending the bytes is left to it.
164
209
  """
165
- request = ingestion_request(
210
+ content = await asyncio.to_thread(
211
+ ingestion_body,
166
212
  raw,
167
213
  envelope=envelope,
168
214
  workflow_id=workflow_id,
@@ -174,7 +220,7 @@ class SpotClient:
174
220
  response = await self._call(
175
221
  "POST",
176
222
  "/internal/ingest",
177
- json=request.model_dump(mode="json"),
223
+ content=content,
178
224
  timeout=timeout,
179
225
  )
180
226
  return read_answer(IngestionResult, response, "POST", "/internal/ingest")
@@ -198,14 +244,19 @@ class SpotClient:
198
244
  return read_answer(OrchestrationResult, response, "GET", path)
199
245
 
200
246
  async def _call(
201
- self, method: str, path: str, *, json: Any = None, timeout: float | None = None
247
+ self,
248
+ method: str,
249
+ path: str,
250
+ *,
251
+ content: bytes | None = None,
252
+ timeout: float | None = None,
202
253
  ) -> httpx.Response:
203
254
  return await spot_request(
204
255
  self.base_url,
205
256
  self.token,
206
257
  method,
207
258
  path,
208
- json=json,
259
+ content=content,
209
260
  timeout=timeout or self.timeout,
210
261
  transport=self._transport,
211
262
  )
@@ -104,7 +104,8 @@ class Anomaly(BaseModel):
104
104
  location: str | None = Field(
105
105
  default=None,
106
106
  description='"part:1.2", "header:Subject@1", "param:Content-Type.boundary@1.2",'
107
- ' ..., "envelope", "source": at most 200 bytes of JSON',
107
+ ' "url:3" (the view\'s URL index), ..., "envelope", "source": at most 200'
108
+ " bytes of JSON",
108
109
  )
109
110
  detail: str | None = Field(
110
111
  default=None,