spot-sdk-python 2.0.0b10__tar.gz → 2.0.0b11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/PKG-INFO +2 -2
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/README.md +1 -1
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/pyproject.toml +1 -1
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/__init__.py +6 -2
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/app.py +58 -31
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/clients.py +7 -11
- spot_sdk_python-2.0.0b11/spot_sdk/correlation.py +65 -0
- spot_sdk_python-2.0.0b11/spot_sdk/logging.py +284 -0
- spot_sdk_python-2.0.0b10/spot_sdk/logging.py +0 -133
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/_json.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/artifacts.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/config.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/email.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/errors.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/knowledge.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/knowledge_tags.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/llm.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/manifest.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/mime.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/model_input.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/orchestrator.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/py.typed +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/retriever.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/settings.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/signals.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/README.md +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/__init__.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/contract.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/fake_knowledge_client.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/fake_llm.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/fake_spot.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/settings.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/views.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/urls.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/verdict.py +0 -0
- {spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/workflow.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: spot-sdk-python
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.0b11
|
|
4
4
|
Summary: SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Author: SPOT Project
|
|
@@ -51,7 +51,7 @@ plugins share:
|
|
|
51
51
|
## Install
|
|
52
52
|
|
|
53
53
|
```bash
|
|
54
|
-
pip install spot-sdk-python==2.0.
|
|
54
|
+
pip install spot-sdk-python==2.0.0b11
|
|
55
55
|
```
|
|
56
56
|
|
|
57
57
|
Python 3.11 to 3.15. Version 2 implements plugin contract 2, for
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "spot-sdk-python"
|
|
7
|
-
version = "2.0.
|
|
7
|
+
version = "2.0.0b11"
|
|
8
8
|
description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
|
|
9
9
|
authors = ["SPOT Project <spot@sonn.lu>"]
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -42,7 +42,7 @@ from .workflow import (
|
|
|
42
42
|
)
|
|
43
43
|
|
|
44
44
|
if TYPE_CHECKING:
|
|
45
|
-
from .app import PluginHealth, SyncState, create_plugin_app
|
|
45
|
+
from .app import CorrelationMiddleware, PluginHealth, SyncState, create_plugin_app
|
|
46
46
|
from .clients import SpotAPIError, SpotClient
|
|
47
47
|
from .knowledge import KnowledgeClient, KnowledgeDocument, chunk_text, content_hash
|
|
48
48
|
from .llm import LLMClient, LLMOutputError, LLMToolCall, LLMToolCalls, LLMToolRound
|
|
@@ -50,7 +50,10 @@ if TYPE_CHECKING:
|
|
|
50
50
|
# The modules that import FastAPI, Starlette or httpx2 load on first use, so importing
|
|
51
51
|
# `spot_sdk.mime` or the models stays light (PEP 562).
|
|
52
52
|
_LAZY = {
|
|
53
|
-
**dict.fromkeys(
|
|
53
|
+
**dict.fromkeys(
|
|
54
|
+
("CorrelationMiddleware", "PluginHealth", "SyncState", "create_plugin_app"),
|
|
55
|
+
".app",
|
|
56
|
+
),
|
|
54
57
|
**dict.fromkeys(("SpotAPIError", "SpotClient"), ".clients"),
|
|
55
58
|
**dict.fromkeys(
|
|
56
59
|
("KnowledgeClient", "KnowledgeDocument", "chunk_text", "content_hash"),
|
|
@@ -123,6 +126,7 @@ __all__ = [
|
|
|
123
126
|
"SettingField",
|
|
124
127
|
"SettingType",
|
|
125
128
|
# Plugin runtime
|
|
129
|
+
"CorrelationMiddleware",
|
|
126
130
|
"PluginHealth",
|
|
127
131
|
"PluginSettings",
|
|
128
132
|
"SyncState",
|
|
@@ -13,7 +13,11 @@ class and its handlers:
|
|
|
13
13
|
|
|
14
14
|
An ``async def`` handler, or an object whose ``__call__`` is ``async def``,
|
|
15
15
|
runs on the event loop; any other callable runs in a worker thread, so it may
|
|
16
|
-
block.
|
|
16
|
+
block.
|
|
17
|
+
|
|
18
|
+
Each request runs under its ``X-Correlation-ID``, or a new one when it has
|
|
19
|
+
none or an invalid one (``CorrelationMiddleware``): the answer carries it,
|
|
20
|
+
the logs hold it, and the clients send it on to SPOT.
|
|
17
21
|
|
|
18
22
|
A plugin that loads a model gives ``load``: run in a daemon thread once the app
|
|
19
23
|
starts, it makes the plugin ``starting`` until it returns, and the analyzer
|
|
@@ -35,8 +39,8 @@ import hmac
|
|
|
35
39
|
import inspect
|
|
36
40
|
import threading
|
|
37
41
|
import traceback
|
|
38
|
-
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
39
|
-
from contextlib import asynccontextmanager
|
|
42
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
43
|
+
from contextlib import asynccontextmanager
|
|
40
44
|
from datetime import UTC, datetime
|
|
41
45
|
from typing import Any, Generic, Literal, TypeAlias, TypeVar, overload
|
|
42
46
|
|
|
@@ -47,9 +51,10 @@ from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
|
|
|
47
51
|
from pydantic.json_schema import models_json_schema
|
|
48
52
|
from pydantic_core import PydanticSerializationError
|
|
49
53
|
from starlette.concurrency import run_in_threadpool
|
|
50
|
-
from starlette.
|
|
54
|
+
from starlette.datastructures import Headers, MutableHeaders
|
|
55
|
+
from starlette.types import ASGIApp, Lifespan, Message, Receive, Scope, Send
|
|
51
56
|
|
|
52
|
-
from .
|
|
57
|
+
from .correlation import HEADER, adopt
|
|
53
58
|
from .logging import get_logger
|
|
54
59
|
from .manifest import Manifest, PluginKind
|
|
55
60
|
from .orchestrator import AnalyzeRequest, RespondRequest
|
|
@@ -303,12 +308,11 @@ def create_plugin_app(
|
|
|
303
308
|
)
|
|
304
309
|
async def analyze_route(request: Request) -> Response:
|
|
305
310
|
call = await _parse(request, AnalyzeRequest)
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
return JSONResponse({"detail": "analysis failed"}, status_code=500)
|
|
311
|
+
try:
|
|
312
|
+
answer = await _run(analyze, call, settings)
|
|
313
|
+
except Exception as error:
|
|
314
|
+
_log_failure(f"analysis of job {call.job_id}", error)
|
|
315
|
+
return JSONResponse({"detail": "analysis failed"}, status_code=500)
|
|
312
316
|
try:
|
|
313
317
|
verdict = AnalyzerVerdict.model_validate(answer)
|
|
314
318
|
except ValidationError as error:
|
|
@@ -332,21 +336,19 @@ def create_plugin_app(
|
|
|
332
336
|
)
|
|
333
337
|
async def respond_route(request: Request) -> Response:
|
|
334
338
|
call = await _parse(request, RespondRequest)
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
return JSONResponse({"detail": "response failed"}, status_code=500)
|
|
339
|
+
try:
|
|
340
|
+
await _run(respond, call, settings)
|
|
341
|
+
except Exception as error:
|
|
342
|
+
_log_failure(f"response to job {call.job_id}", error)
|
|
343
|
+
return JSONResponse({"detail": "response failed"}, status_code=500)
|
|
341
344
|
return Response(status_code=204)
|
|
342
345
|
|
|
343
346
|
if sync is not None:
|
|
344
347
|
syncs = _SyncRunner(sync, settings)
|
|
345
348
|
|
|
346
349
|
@internal.post("/sync", status_code=202)
|
|
347
|
-
async def sync_route(
|
|
348
|
-
|
|
349
|
-
return syncs.start()
|
|
350
|
+
async def sync_route() -> SyncState:
|
|
351
|
+
return syncs.start()
|
|
350
352
|
|
|
351
353
|
@internal.get("/state")
|
|
352
354
|
async def state_route() -> SyncState:
|
|
@@ -356,14 +358,49 @@ def create_plugin_app(
|
|
|
356
358
|
return app
|
|
357
359
|
|
|
358
360
|
|
|
361
|
+
class CorrelationMiddleware:
|
|
362
|
+
"""Gives each HTTP request its ID, the ``X-Correlation-ID`` sent when valid,
|
|
363
|
+
else a new one, and sends it back with the answer.
|
|
364
|
+
|
|
365
|
+
The ID stays current for the rest of the request's task: add this
|
|
366
|
+
middleware last, so that it is the app's outermost, and the other
|
|
367
|
+
middleware, the app's 500 handler and the server's error line all run
|
|
368
|
+
under it. The plugin app wraps its whole stack in it, Starlette's error
|
|
369
|
+
middleware included, so that a 500 answer carries the ID too.
|
|
370
|
+
"""
|
|
371
|
+
|
|
372
|
+
def __init__(self, app: ASGIApp) -> None:
|
|
373
|
+
self.app = app
|
|
374
|
+
|
|
375
|
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
376
|
+
if scope["type"] != "http":
|
|
377
|
+
await self.app(scope, receive, send)
|
|
378
|
+
return
|
|
379
|
+
cid = adopt(Headers(scope=scope).get(HEADER))
|
|
380
|
+
|
|
381
|
+
async def send_with_id(message: Message) -> None:
|
|
382
|
+
if message["type"] == "http.response.start":
|
|
383
|
+
MutableHeaders(scope=message)[HEADER] = cid
|
|
384
|
+
await send(message)
|
|
385
|
+
|
|
386
|
+
await self.app(scope, receive, send_with_id)
|
|
387
|
+
|
|
388
|
+
|
|
359
389
|
class _PluginApp(FastAPI):
|
|
360
390
|
"""The plugin's app. Its OpenAPI document also holds the schemas of the
|
|
361
|
-
request bodies its routes read themselves, which FastAPI leaves out
|
|
391
|
+
request bodies its routes read themselves, which FastAPI leaves out, and
|
|
392
|
+
``CorrelationMiddleware`` wraps its whole stack."""
|
|
362
393
|
|
|
363
394
|
def __init__(self, **options: Any) -> None:
|
|
364
395
|
super().__init__(**options)
|
|
365
396
|
self._read_bodies: list[type[BaseModel]] = []
|
|
366
397
|
|
|
398
|
+
def build_middleware_stack(self) -> ASGIApp:
|
|
399
|
+
"""The stack inside ``CorrelationMiddleware``: outside Starlette's error
|
|
400
|
+
middleware, whose 500 then carries the ID, and outside any middleware
|
|
401
|
+
the plugin adds, which Starlette would put outside it otherwise."""
|
|
402
|
+
return CorrelationMiddleware(super().build_middleware_stack())
|
|
403
|
+
|
|
367
404
|
def reads(self, model: type[BaseModel]) -> dict[str, Any]:
|
|
368
405
|
"""The ``openapi_extra`` of a route that reads ``model`` from its body
|
|
369
406
|
itself, validated (``_parse``)."""
|
|
@@ -593,16 +630,6 @@ def _is_async(handler: Callable[..., Any]) -> bool:
|
|
|
593
630
|
)
|
|
594
631
|
|
|
595
632
|
|
|
596
|
-
@contextmanager
|
|
597
|
-
def _correlated(request: Request) -> Iterator[None]:
|
|
598
|
-
"""Make the call's ``X-Correlation-ID`` the one the clients send."""
|
|
599
|
-
reset = correlation_id.set(request.headers.get("x-correlation-id"))
|
|
600
|
-
try:
|
|
601
|
-
yield
|
|
602
|
-
finally:
|
|
603
|
-
correlation_id.reset(reset)
|
|
604
|
-
|
|
605
|
-
|
|
606
633
|
def _log_failure(what: str, error: Exception) -> None:
|
|
607
634
|
"""Log a handler's failure by its type and place only: its message, or the
|
|
608
635
|
locations of a validation error, may quote the email, or a directory URL
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"""The gateway client: what plugins ask of SPOT.
|
|
2
2
|
|
|
3
3
|
Every call carries the plugin's own token (``Authorization: Bearer``) and
|
|
4
|
-
the
|
|
5
|
-
plugin's calls to the analysis
|
|
6
|
-
:class:`SpotAPIError`, which names
|
|
7
|
-
content.
|
|
4
|
+
the current correlation ID (``spot_sdk.correlation``) as
|
|
5
|
+
``X-Correlation-ID``, so SPOT's logs tie a plugin's calls to the analysis
|
|
6
|
+
that made them. An error status raises :class:`SpotAPIError`, which names
|
|
7
|
+
the route and the status, never the content.
|
|
8
8
|
|
|
9
9
|
- Mail retrievers push raw messages with :meth:`SpotClient.ingest`.
|
|
10
10
|
- Analyzers fetch the raw message or one decoded part with
|
|
@@ -17,7 +17,6 @@ from __future__ import annotations
|
|
|
17
17
|
import asyncio
|
|
18
18
|
import base64
|
|
19
19
|
import json
|
|
20
|
-
from contextvars import ContextVar
|
|
21
20
|
from typing import Any, TypeVar
|
|
22
21
|
from urllib.parse import quote
|
|
23
22
|
from uuid import UUID
|
|
@@ -25,6 +24,7 @@ from uuid import UUID
|
|
|
25
24
|
import httpx2
|
|
26
25
|
from pydantic import BaseModel, SecretStr, ValidationError
|
|
27
26
|
|
|
27
|
+
from . import correlation
|
|
28
28
|
from .email import Envelope
|
|
29
29
|
from .orchestrator import OrchestrationResult
|
|
30
30
|
from .retriever import (
|
|
@@ -42,9 +42,6 @@ _INGEST_PIECE = 768 * 1024
|
|
|
42
42
|
|
|
43
43
|
ModelT = TypeVar("ModelT", bound=BaseModel)
|
|
44
44
|
|
|
45
|
-
correlation_id: ContextVar[str | None] = ContextVar("spot_correlation_id", default=None)
|
|
46
|
-
"""The ``X-Correlation-ID`` of the call being handled; the clients send it on."""
|
|
47
|
-
|
|
48
45
|
|
|
49
46
|
class SpotAPIError(Exception):
|
|
50
47
|
"""SPOT answered an error status, or an answer that does not validate.
|
|
@@ -96,9 +93,8 @@ async def spot_request(
|
|
|
96
93
|
The body is ``json`` encoded by :func:`json_body`, or ``content``, a JSON
|
|
97
94
|
body the caller has already encoded."""
|
|
98
95
|
headers = {"Authorization": f"Bearer {token.get_secret_value()}"}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
headers["X-Correlation-ID"] = current
|
|
96
|
+
if (cid := correlation.current()) is not None:
|
|
97
|
+
headers[correlation.HEADER] = cid
|
|
102
98
|
if json is not None:
|
|
103
99
|
content = json_body(json)
|
|
104
100
|
if content is not None:
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""The correlation ID that ties together the work done for one request or message.
|
|
2
|
+
|
|
3
|
+
Core gives each request, message and scheduled sync an ID and sends it as
|
|
4
|
+
``X-Correlation-ID`` to the plugins it calls. ``create_plugin_app`` makes
|
|
5
|
+
each call's ID current (``spot_sdk.app.CorrelationMiddleware``), the SDK's
|
|
6
|
+
clients send it back with their calls to SPOT, and the SDK's logs carry it
|
|
7
|
+
(``spot_sdk.logging``).
|
|
8
|
+
|
|
9
|
+
An ID is 1 to 128 characters of letters, digits and ``.``, ``_``, ``:``,
|
|
10
|
+
``-``. Anything else is replaced with a new ID, so that a caller cannot write
|
|
11
|
+
into the logs through it.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
import uuid
|
|
18
|
+
from collections.abc import Iterator
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from contextvars import ContextVar
|
|
21
|
+
|
|
22
|
+
HEADER = "X-Correlation-ID"
|
|
23
|
+
_ID = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
|
24
|
+
_current: ContextVar[str | None] = ContextVar("spot_correlation_id", default=None)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def current() -> str | None:
|
|
28
|
+
"""The ID of the work being done; None outside any."""
|
|
29
|
+
return _current.get()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def new_id() -> str:
|
|
33
|
+
"""A new ID."""
|
|
34
|
+
return str(uuid.uuid4())
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def accepted(value: str | None) -> str:
|
|
38
|
+
"""``value`` when it is a valid ID, else a new one."""
|
|
39
|
+
return value if value is not None and _ID.fullmatch(value) else new_id()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@contextmanager
|
|
43
|
+
def correlated(value: str | None = None) -> Iterator[str]:
|
|
44
|
+
"""Run the block under ``value``, or a new ID when it is None or invalid."""
|
|
45
|
+
cid = accepted(value)
|
|
46
|
+
token = _current.set(cid)
|
|
47
|
+
try:
|
|
48
|
+
yield cid
|
|
49
|
+
finally:
|
|
50
|
+
_current.reset(token)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def adopt(value: str | None) -> str:
|
|
54
|
+
"""Make ``value``, or a new ID, current for the rest of the calling task.
|
|
55
|
+
|
|
56
|
+
Nothing restores the previous ID: use it in a task that does one piece of
|
|
57
|
+
work, as uvicorn runs each request, so that whatever runs after the code
|
|
58
|
+
that set it (an app's 500 handler, uvicorn's error line) still sees it. In a
|
|
59
|
+
task that serves several requests, the last one's ID stays set after it:
|
|
60
|
+
a test that calls an app through ``httpx2.ASGITransport`` from its own task
|
|
61
|
+
keeps that ID until the test ends.
|
|
62
|
+
"""
|
|
63
|
+
cid = accepted(value)
|
|
64
|
+
_current.set(cid)
|
|
65
|
+
return cid
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
"""The logging of SPOT's services and plugins: one JSON object per line.
|
|
2
|
+
|
|
3
|
+
``configure_logging`` writes every record to stdout (and ``LOG_FILE`` when
|
|
4
|
+
set): one JSON object on one line by default (``LOG_FORMAT=json``), or, for
|
|
5
|
+
development, a readable line, the traceback after it (``LOG_FORMAT=text``).
|
|
6
|
+
Each record holds the current correlation ID (``spot_sdk.correlation``).
|
|
7
|
+
uvicorn's lines go the same way, its access lines without their query string
|
|
8
|
+
and its unhandled exceptions, a lifespan's failure included, by type and
|
|
9
|
+
frames; so do Python's warnings.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
import logging.config
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import traceback
|
|
20
|
+
from datetime import UTC, datetime
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from .correlation import current
|
|
24
|
+
|
|
25
|
+
LOG_FORMATS = ("json", "text")
|
|
26
|
+
|
|
27
|
+
# A record's own attributes; anything else on it came from ``extra=``.
|
|
28
|
+
# ``color_message`` is uvicorn's coloured copy of the message.
|
|
29
|
+
_RECORD_FIELDS = frozenset(vars(logging.LogRecord("", 0, "", 0, "", None, None))) | {
|
|
30
|
+
"message",
|
|
31
|
+
"asctime",
|
|
32
|
+
"correlation_id",
|
|
33
|
+
"color_message",
|
|
34
|
+
}
|
|
35
|
+
# A traceback written as text: its header, a frame's line, an exception's type.
|
|
36
|
+
_TRACEBACK = "Traceback (most recent call last):"
|
|
37
|
+
_FRAME = re.compile(r'File "(.*)", line (\d+), in (.*)')
|
|
38
|
+
_TYPE = re.compile(r"[A-Za-z_][\w.]*")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class CorrelationFilter(logging.Filter):
|
|
42
|
+
"""Puts the current correlation ID on each record (``correlation_id``, None outside any work)."""
|
|
43
|
+
|
|
44
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
45
|
+
# Not a LogRecord attribute: mypy refuses the assignment.
|
|
46
|
+
setattr(record, "correlation_id", current())
|
|
47
|
+
return True
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class AccessQueryFilter(logging.Filter):
|
|
51
|
+
"""Cuts the query string off uvicorn's access lines: it can hold search terms and addresses.
|
|
52
|
+
|
|
53
|
+
uvicorn logs ``(client, method, path with query, http version, status)``.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
57
|
+
if isinstance(record.args, tuple) and len(record.args) == 5:
|
|
58
|
+
client, method, path, version, status = record.args
|
|
59
|
+
record.args = (client, method, str(path).split("?", 1)[0], version, status)
|
|
60
|
+
return True
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class ServerErrorFilter(logging.Filter):
|
|
64
|
+
"""Logs uvicorn's unhandled exceptions by type and frames: their text, and
|
|
65
|
+
the source lines of a traceback, can quote a request's content.
|
|
66
|
+
|
|
67
|
+
uvicorn logs a request's exception as ``exc_info``, and a lifespan's
|
|
68
|
+
failure as the traceback's text that Starlette sends it.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
72
|
+
error = record.exc_info[1] if record.exc_info else None
|
|
73
|
+
if error is not None:
|
|
74
|
+
frames = "\n".join(
|
|
75
|
+
_frame(frame.filename, frame.lineno, frame.name)
|
|
76
|
+
for frame in traceback.extract_tb(error.__traceback__)
|
|
77
|
+
)
|
|
78
|
+
record.args = (record.getMessage().strip(), type(error).__name__, frames)
|
|
79
|
+
record.msg = "%s: %s, raised at:\n%s"
|
|
80
|
+
record.exc_info = None
|
|
81
|
+
record.exc_text = None
|
|
82
|
+
elif (written := _written_traceback(record.getMessage())) is not None:
|
|
83
|
+
record.args = written
|
|
84
|
+
record.msg = "%s, raised at:\n%s"
|
|
85
|
+
return True
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _frame(filename: str, lineno: object, name: str) -> str:
|
|
89
|
+
"""A frame as a traceback writes it, without its source line."""
|
|
90
|
+
return f' File "{filename}", line {lineno}, in {name}'
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _written_traceback(text: str) -> tuple[str, str] | None:
|
|
94
|
+
"""The type and frames of the first traceback written in ``text``; None
|
|
95
|
+
when it holds none.
|
|
96
|
+
|
|
97
|
+
The first: an exception group is named by its own type and frames, a
|
|
98
|
+
chain by its root cause, and a traceback quoted in a message or a note is
|
|
99
|
+
never read. The type is read from the line that names it, the first after
|
|
100
|
+
the frames that is less indented than them (an exception group's ``|``
|
|
101
|
+
counts as indent), never from the lines after it: the message, which can
|
|
102
|
+
run over several lines, and the notes.
|
|
103
|
+
"""
|
|
104
|
+
start = text.find(_TRACEBACK)
|
|
105
|
+
if start < 0:
|
|
106
|
+
return None
|
|
107
|
+
kind, depth = "unknown", 0
|
|
108
|
+
frames: list[str] = []
|
|
109
|
+
for line in text[start + len(_TRACEBACK) :].splitlines():
|
|
110
|
+
content = line.lstrip(" |")
|
|
111
|
+
indent = len(line) - len(content)
|
|
112
|
+
if frame := _FRAME.fullmatch(content):
|
|
113
|
+
frames.append(_frame(frame[1], frame[2], frame[3]))
|
|
114
|
+
depth = indent
|
|
115
|
+
elif frames and content and indent < depth:
|
|
116
|
+
name = content.split(":", 1)[0]
|
|
117
|
+
kind = name if _TYPE.fullmatch(name) else kind
|
|
118
|
+
break
|
|
119
|
+
return kind, "\n".join(frames)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class JsonFormatter(logging.Formatter):
|
|
123
|
+
"""One JSON object per record, on one line."""
|
|
124
|
+
|
|
125
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
126
|
+
entry: dict[str, Any] = {
|
|
127
|
+
"timestamp": datetime.fromtimestamp(record.created, UTC)
|
|
128
|
+
.isoformat(timespec="milliseconds")
|
|
129
|
+
.replace("+00:00", "Z"),
|
|
130
|
+
"level": record.levelname,
|
|
131
|
+
"logger": record.name,
|
|
132
|
+
"message": record.getMessage(),
|
|
133
|
+
}
|
|
134
|
+
cid = getattr(record, "correlation_id", None)
|
|
135
|
+
if cid:
|
|
136
|
+
entry["correlation_id"] = cid
|
|
137
|
+
if record.exc_info:
|
|
138
|
+
entry["exception"] = self.formatException(record.exc_info)
|
|
139
|
+
if record.stack_info:
|
|
140
|
+
entry["stack"] = self.formatStack(record.stack_info)
|
|
141
|
+
extra = {
|
|
142
|
+
key: value
|
|
143
|
+
for key, value in vars(record).items()
|
|
144
|
+
if key not in _RECORD_FIELDS and key not in entry
|
|
145
|
+
}
|
|
146
|
+
try:
|
|
147
|
+
return json.dumps({**entry, **extra}, default=str, allow_nan=False)
|
|
148
|
+
except (TypeError, ValueError):
|
|
149
|
+
# A NaN, or a key JSON cannot hold, in an extra field: all as text.
|
|
150
|
+
text = {key: str(value) for key, value in extra.items()}
|
|
151
|
+
return json.dumps({**entry, **text})
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class TextFormatter(logging.Formatter):
|
|
155
|
+
"""A readable line per record, the traceback after it: time, level, logger,
|
|
156
|
+
correlation ID, message."""
|
|
157
|
+
|
|
158
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
159
|
+
cid = getattr(record, "correlation_id", None) or "-"
|
|
160
|
+
line = f"{self.formatTime(record)} {record.levelname} {record.name} [{cid}] {record.getMessage()}"
|
|
161
|
+
if record.exc_info:
|
|
162
|
+
line += "\n" + self.formatException(record.exc_info)
|
|
163
|
+
return line
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def get_log_level() -> str:
|
|
167
|
+
"""The log level from ``LOG_LEVEL``, default ``INFO``."""
|
|
168
|
+
return os.getenv("LOG_LEVEL", "INFO").upper()
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def get_log_format() -> str:
|
|
172
|
+
"""The log format from ``LOG_FORMAT``, default ``json``."""
|
|
173
|
+
return os.getenv("LOG_FORMAT", "json").lower()
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def get_logging_config(
|
|
177
|
+
log_level: str | None = None, log_format: str | None = None
|
|
178
|
+
) -> dict[str, Any]:
|
|
179
|
+
"""The ``dictConfig`` of ``configure_logging``; ``ValueError`` for a format
|
|
180
|
+
other than ``json`` and ``text``.
|
|
181
|
+
|
|
182
|
+
The level and the format default to ``LOG_LEVEL`` and ``LOG_FORMAT``;
|
|
183
|
+
``LOG_FILE``, when set, gets the same lines as stdout.
|
|
184
|
+
"""
|
|
185
|
+
level = (log_level or get_log_level()).upper()
|
|
186
|
+
fmt = (log_format or get_log_format()).lower()
|
|
187
|
+
if fmt not in LOG_FORMATS:
|
|
188
|
+
raise ValueError(
|
|
189
|
+
f"LOG_FORMAT must be one of {', '.join(LOG_FORMATS)}, not {fmt!r}"
|
|
190
|
+
)
|
|
191
|
+
handler = {
|
|
192
|
+
"class": "logging.StreamHandler",
|
|
193
|
+
"formatter": "default",
|
|
194
|
+
"filters": ["correlation"],
|
|
195
|
+
"stream": "ext://sys.stdout",
|
|
196
|
+
}
|
|
197
|
+
config: dict[str, Any] = {
|
|
198
|
+
"version": 1,
|
|
199
|
+
"disable_existing_loggers": False,
|
|
200
|
+
"filters": {
|
|
201
|
+
"correlation": {"()": CorrelationFilter},
|
|
202
|
+
"access_query": {"()": AccessQueryFilter},
|
|
203
|
+
"server_error": {"()": ServerErrorFilter},
|
|
204
|
+
},
|
|
205
|
+
"formatters": {
|
|
206
|
+
"default": {"()": JsonFormatter if fmt == "json" else TextFormatter}
|
|
207
|
+
},
|
|
208
|
+
"handlers": {"console": handler},
|
|
209
|
+
"loggers": {
|
|
210
|
+
# uvicorn configures these before it imports the app: they keep
|
|
211
|
+
# their level and write through the root handler instead.
|
|
212
|
+
"uvicorn": {"handlers": [], "propagate": True},
|
|
213
|
+
"uvicorn.error": {"filters": ["server_error"]},
|
|
214
|
+
"uvicorn.access": {
|
|
215
|
+
"handlers": [],
|
|
216
|
+
"filters": ["access_query"],
|
|
217
|
+
"propagate": True,
|
|
218
|
+
},
|
|
219
|
+
# At INFO, httpx and httpx2 write each request's URL, and at DEBUG
|
|
220
|
+
# httpcore and httpcore2 (their sub-loggers inherit the level) the
|
|
221
|
+
# raw headers: any of them may hold a credential.
|
|
222
|
+
"httpx": {"level": "WARNING"},
|
|
223
|
+
"httpx2": {"level": "WARNING"},
|
|
224
|
+
"httpcore": {"level": "WARNING"},
|
|
225
|
+
"httpcore2": {"level": "WARNING"},
|
|
226
|
+
},
|
|
227
|
+
"root": {"level": level, "handlers": ["console"]},
|
|
228
|
+
}
|
|
229
|
+
log_file = os.getenv("LOG_FILE")
|
|
230
|
+
if log_file:
|
|
231
|
+
config["handlers"]["file"] = {
|
|
232
|
+
"class": "logging.FileHandler",
|
|
233
|
+
"formatter": "default",
|
|
234
|
+
"filters": ["correlation"],
|
|
235
|
+
"filename": log_file,
|
|
236
|
+
"mode": "a",
|
|
237
|
+
}
|
|
238
|
+
config["root"]["handlers"].append("file")
|
|
239
|
+
return config
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def configure_logging(
|
|
243
|
+
log_level: str | None = None, log_format: str | None = None
|
|
244
|
+
) -> None:
|
|
245
|
+
"""Configure logging once, at import, before the server writes its first lines.
|
|
246
|
+
|
|
247
|
+
The level and the format default to ``LOG_LEVEL`` (``INFO``) and
|
|
248
|
+
``LOG_FORMAT`` (``json``); ``LOG_FILE``, when set, gets the same lines
|
|
249
|
+
as stdout. Python's warnings become log lines. A second call replaces
|
|
250
|
+
the first.
|
|
251
|
+
"""
|
|
252
|
+
config = get_logging_config(log_level, log_format)
|
|
253
|
+
# dictConfig replaces a logger's handlers but adds to its filters.
|
|
254
|
+
for name, options in config["loggers"].items():
|
|
255
|
+
if "filters" in options:
|
|
256
|
+
logging.getLogger(name).filters.clear()
|
|
257
|
+
logging.config.dictConfig(config)
|
|
258
|
+
# warnings.warn writes to stderr otherwise: one more line that is not JSON.
|
|
259
|
+
logging.captureWarnings(True)
|
|
260
|
+
log_file = os.getenv("LOG_FILE")
|
|
261
|
+
logging.getLogger("spot.logging").info(
|
|
262
|
+
"Logging configured: level %s, format %s, output %s",
|
|
263
|
+
(log_level or get_log_level()).upper(),
|
|
264
|
+
(log_format or get_log_format()).lower(),
|
|
265
|
+
f"console and file {log_file}" if log_file else "console",
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def get_logger(name: str) -> logging.Logger:
|
|
270
|
+
"""
|
|
271
|
+
Get a logger instance for the given name.
|
|
272
|
+
|
|
273
|
+
Args:
|
|
274
|
+
name: Logger name, typically __name__
|
|
275
|
+
|
|
276
|
+
Returns:
|
|
277
|
+
Standard Python logger
|
|
278
|
+
|
|
279
|
+
Example:
|
|
280
|
+
logger = get_logger(__name__)
|
|
281
|
+
logger.info("Service started")
|
|
282
|
+
logger.debug("Processing email", extra={"email_id": "123"})
|
|
283
|
+
"""
|
|
284
|
+
return logging.getLogger(name)
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Standard logging configuration for SPOT platform.
|
|
3
|
-
|
|
4
|
-
Uses Python's built-in logging with dictConfig.
|
|
5
|
-
Simple, configurable via environment variables.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
from __future__ import annotations
|
|
9
|
-
|
|
10
|
-
import logging
|
|
11
|
-
import logging.config
|
|
12
|
-
import os
|
|
13
|
-
from typing import Any, Optional
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def get_log_level() -> str:
|
|
17
|
-
"""Get log level from environment, default to INFO."""
|
|
18
|
-
return os.getenv("LOG_LEVEL", "INFO").upper()
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def get_log_format() -> str:
|
|
22
|
-
"""Get log format from environment, default to simple."""
|
|
23
|
-
return os.getenv("LOG_FORMAT", "simple").lower()
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def get_logging_config(
|
|
27
|
-
log_level: Optional[str] = None, log_format: Optional[str] = None
|
|
28
|
-
) -> dict[str, Any]:
|
|
29
|
-
"""
|
|
30
|
-
Get logging configuration dictionary for dictConfig.
|
|
31
|
-
|
|
32
|
-
Args:
|
|
33
|
-
log_level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
|
|
34
|
-
log_format: Format style (simple, detailed, json)
|
|
35
|
-
|
|
36
|
-
Returns:
|
|
37
|
-
Dictionary suitable for logging.config.dictConfig
|
|
38
|
-
"""
|
|
39
|
-
level = log_level or get_log_level()
|
|
40
|
-
fmt = log_format or get_log_format()
|
|
41
|
-
|
|
42
|
-
# Choose format string
|
|
43
|
-
if fmt == "json":
|
|
44
|
-
format_str = '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "logger": "%(name)s", "message": "%(message)s"}'
|
|
45
|
-
elif fmt == "detailed":
|
|
46
|
-
format_str = "%(asctime)s - %(name)-20s - %(levelname)-8s - [%(filename)s:%(lineno)d] - %(message)s"
|
|
47
|
-
else: # simple
|
|
48
|
-
format_str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
49
|
-
|
|
50
|
-
config = {
|
|
51
|
-
"version": 1,
|
|
52
|
-
"disable_existing_loggers": False,
|
|
53
|
-
"formatters": {
|
|
54
|
-
"default": {
|
|
55
|
-
"format": format_str,
|
|
56
|
-
"datefmt": "%Y-%m-%d %H:%M:%S" if fmt == "detailed" else "%H:%M:%S",
|
|
57
|
-
}
|
|
58
|
-
},
|
|
59
|
-
"handlers": {
|
|
60
|
-
"console": {
|
|
61
|
-
"class": "logging.StreamHandler",
|
|
62
|
-
"formatter": "default",
|
|
63
|
-
"stream": "ext://sys.stdout",
|
|
64
|
-
}
|
|
65
|
-
},
|
|
66
|
-
"root": {"level": level, "handlers": ["console"]},
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
# Add file handler if LOG_FILE is specified
|
|
70
|
-
log_file = os.getenv("LOG_FILE")
|
|
71
|
-
if log_file:
|
|
72
|
-
config["handlers"]["file"] = { # type: ignore[index]
|
|
73
|
-
"class": "logging.FileHandler",
|
|
74
|
-
"formatter": "default",
|
|
75
|
-
"filename": log_file,
|
|
76
|
-
"mode": "a",
|
|
77
|
-
}
|
|
78
|
-
config["root"]["handlers"].append("file") # type: ignore[index]
|
|
79
|
-
|
|
80
|
-
return config
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def configure_logging(
|
|
84
|
-
log_level: Optional[str] = None, log_format: Optional[str] = None
|
|
85
|
-
) -> None:
|
|
86
|
-
"""
|
|
87
|
-
Configure logging for the application.
|
|
88
|
-
|
|
89
|
-
Call this once during application startup.
|
|
90
|
-
|
|
91
|
-
Args:
|
|
92
|
-
log_level: Log level (default: from LOG_LEVEL env var or INFO)
|
|
93
|
-
log_format: Format style (default: from LOG_FORMAT env var or simple)
|
|
94
|
-
|
|
95
|
-
Environment Variables:
|
|
96
|
-
LOG_LEVEL: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO)
|
|
97
|
-
LOG_FORMAT: json, simple, detailed (default: simple)
|
|
98
|
-
LOG_FILE: Path to log file (optional, logs to console if not set)
|
|
99
|
-
"""
|
|
100
|
-
config = get_logging_config(log_level, log_format)
|
|
101
|
-
logging.config.dictConfig(config)
|
|
102
|
-
|
|
103
|
-
# Log configuration
|
|
104
|
-
logger = logging.getLogger("spot.logging")
|
|
105
|
-
level = log_level or get_log_level()
|
|
106
|
-
fmt = log_format or get_log_format()
|
|
107
|
-
log_file = os.getenv("LOG_FILE")
|
|
108
|
-
|
|
109
|
-
log_targets = "console"
|
|
110
|
-
if log_file:
|
|
111
|
-
log_targets += f" and file ({log_file})"
|
|
112
|
-
|
|
113
|
-
logger.info(
|
|
114
|
-
f"Logging configured - Level: {level}, Format: {fmt}, Output: {log_targets}"
|
|
115
|
-
)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def get_logger(name: str) -> logging.Logger:
|
|
119
|
-
"""
|
|
120
|
-
Get a logger instance for the given name.
|
|
121
|
-
|
|
122
|
-
Args:
|
|
123
|
-
name: Logger name, typically __name__
|
|
124
|
-
|
|
125
|
-
Returns:
|
|
126
|
-
Standard Python logger
|
|
127
|
-
|
|
128
|
-
Example:
|
|
129
|
-
logger = get_logger(__name__)
|
|
130
|
-
logger.info("Service started")
|
|
131
|
-
logger.debug("Processing email", extra={"email_id": "123"})
|
|
132
|
-
"""
|
|
133
|
-
return logging.getLogger(name)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/fake_knowledge_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|