spot-sdk-python 2.0.0b9__tar.gz → 2.0.0b11__tar.gz

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spot-sdk-python
-Version: 2.0.0b9
+Version: 2.0.0b11
 Summary: SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers
 License: Apache-2.0
 Author: SPOT Project
@@ -51,7 +51,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b9
+pip install spot-sdk-python==2.0.0b11
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/README.md

@@ -26,7 +26,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b9
+pip install spot-sdk-python==2.0.0b11
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "spot-sdk-python"
-version = "2.0.0b9"
+version = "2.0.0b11"
 description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
 authors = ["SPOT Project <spot@sonn.lu>"]
 license = "Apache-2.0"

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/spot_sdk/__init__.py

@@ -25,7 +25,7 @@ from .orchestrator import (
     RespondRequest,
     StageOutcome,
 )
-from .retriever import IngestionRequest, IngestionResult, IngestionStatus
+from .retriever import ActionReport, IngestionRequest, IngestionResult, IngestionStatus
 from .settings import PluginSettings
 from .verdict import AnalyzerVerdict, Score, Signal, VerdictClass
 from .workflow import (
@@ -42,7 +42,7 @@ from .workflow import (
 )
 
 if TYPE_CHECKING:
-    from .app import PluginHealth, SyncState, create_plugin_app
+    from .app import CorrelationMiddleware, PluginHealth, SyncState, create_plugin_app
     from .clients import SpotAPIError, SpotClient
     from .knowledge import KnowledgeClient, KnowledgeDocument, chunk_text, content_hash
     from .llm import LLMClient, LLMOutputError, LLMToolCall, LLMToolCalls, LLMToolRound
@@ -50,7 +50,10 @@ if TYPE_CHECKING:
 # The modules that import FastAPI, Starlette or httpx2 load on first use, so importing
 # `spot_sdk.mime` or the models stays light (PEP 562).
 _LAZY = {
-    **dict.fromkeys(("PluginHealth", "SyncState", "create_plugin_app"), ".app"),
+    **dict.fromkeys(
+        ("CorrelationMiddleware", "PluginHealth", "SyncState", "create_plugin_app"),
+        ".app",
+    ),
     **dict.fromkeys(("SpotAPIError", "SpotClient"), ".clients"),
     **dict.fromkeys(
         ("KnowledgeClient", "KnowledgeDocument", "chunk_text", "content_hash"),
@@ -110,6 +113,7 @@ __all__ = [
     "RespondRequest",
     "StageOutcome",
     # Ingestion
+    "ActionReport",
     "IngestionRequest",
     "IngestionResult",
     "IngestionStatus",
@@ -122,6 +126,7 @@ __all__ = [
     "SettingField",
     "SettingType",
     # Plugin runtime
+    "CorrelationMiddleware",
     "PluginHealth",
     "PluginSettings",
     "SyncState",

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/spot_sdk/app.py

@@ -13,7 +13,11 @@ class and its handlers:
 
 An ``async def`` handler, or an object whose ``__call__`` is ``async def``,
 runs on the event loop; any other callable runs in a worker thread, so it may
-block. Either way the clients send the call's ``X-Correlation-ID`` on to SPOT.
+block.
+
+Each request runs under its ``X-Correlation-ID``, or a new one when it has
+none or an invalid one (``CorrelationMiddleware``): the answer carries it,
+the logs hold it, and the clients send it on to SPOT.
 
 A plugin that loads a model gives ``load``: run in a daemon thread once the app
 starts, it makes the plugin ``starting`` until it returns, and the analyzer
@@ -35,8 +39,8 @@ import hmac
 import inspect
 import threading
 import traceback
-from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
-from contextlib import asynccontextmanager, contextmanager
+from collections.abc import AsyncIterator, Awaitable, Callable
+from contextlib import asynccontextmanager
 from datetime import UTC, datetime
 from typing import Any, Generic, Literal, TypeAlias, TypeVar, overload
 
@@ -47,9 +51,10 @@ from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
 from pydantic.json_schema import models_json_schema
 from pydantic_core import PydanticSerializationError
 from starlette.concurrency import run_in_threadpool
-from starlette.types import Lifespan
+from starlette.datastructures import Headers, MutableHeaders
+from starlette.types import ASGIApp, Lifespan, Message, Receive, Scope, Send
 
-from .clients import correlation_id
+from .correlation import HEADER, adopt
 from .logging import get_logger
 from .manifest import Manifest, PluginKind
 from .orchestrator import AnalyzeRequest, RespondRequest
@@ -303,12 +308,11 @@ def create_plugin_app(
         )
         async def analyze_route(request: Request) -> Response:
             call = await _parse(request, AnalyzeRequest)
-            with _correlated(request):
-                try:
-                    answer = await _run(analyze, call, settings)
-                except Exception as error:
-                    _log_failure(f"analysis of job {call.job_id}", error)
-                    return JSONResponse({"detail": "analysis failed"}, status_code=500)
+            try:
+                answer = await _run(analyze, call, settings)
+            except Exception as error:
+                _log_failure(f"analysis of job {call.job_id}", error)
+                return JSONResponse({"detail": "analysis failed"}, status_code=500)
             try:
                 verdict = AnalyzerVerdict.model_validate(answer)
             except ValidationError as error:
@@ -332,21 +336,19 @@ def create_plugin_app(
         )
         async def respond_route(request: Request) -> Response:
             call = await _parse(request, RespondRequest)
-            with _correlated(request):
-                try:
-                    await _run(respond, call, settings)
-                except Exception as error:
-                    _log_failure(f"response to job {call.job_id}", error)
-                    return JSONResponse({"detail": "response failed"}, status_code=500)
+            try:
+                await _run(respond, call, settings)
+            except Exception as error:
+                _log_failure(f"response to job {call.job_id}", error)
+                return JSONResponse({"detail": "response failed"}, status_code=500)
             return Response(status_code=204)
 
     if sync is not None:
         syncs = _SyncRunner(sync, settings)
 
         @internal.post("/sync", status_code=202)
-        async def sync_route(request: Request) -> SyncState:
-            with _correlated(request):
-                return syncs.start()
+        async def sync_route() -> SyncState:
+            return syncs.start()
 
         @internal.get("/state")
         async def state_route() -> SyncState:
@@ -356,14 +358,49 @@ def create_plugin_app(
     return app
 
 
+class CorrelationMiddleware:
+    """Gives each HTTP request its ID, the ``X-Correlation-ID`` sent when valid,
+    else a new one, and sends it back with the answer.
+
+    The ID stays current for the rest of the request's task: add this
+    middleware last, so that it is the app's outermost, and the other
+    middleware, the app's 500 handler and the server's error line all run
+    under it. The plugin app wraps its whole stack in it, Starlette's error
+    middleware included, so that a 500 answer carries the ID too.
+    """
+
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+        cid = adopt(Headers(scope=scope).get(HEADER))
+
+        async def send_with_id(message: Message) -> None:
+            if message["type"] == "http.response.start":
+                MutableHeaders(scope=message)[HEADER] = cid
+            await send(message)
+
+        await self.app(scope, receive, send_with_id)
+
+
 class _PluginApp(FastAPI):
     """The plugin's app. Its OpenAPI document also holds the schemas of the
-    request bodies its routes read themselves, which FastAPI leaves out."""
+    request bodies its routes read themselves, which FastAPI leaves out, and
+    ``CorrelationMiddleware`` wraps its whole stack."""
 
     def __init__(self, **options: Any) -> None:
         super().__init__(**options)
         self._read_bodies: list[type[BaseModel]] = []
 
+    def build_middleware_stack(self) -> ASGIApp:
+        """The stack inside ``CorrelationMiddleware``: outside Starlette's error
+        middleware, whose 500 then carries the ID, and outside any middleware
+        the plugin adds, which Starlette would put outside it otherwise."""
+        return CorrelationMiddleware(super().build_middleware_stack())
+
     def reads(self, model: type[BaseModel]) -> dict[str, Any]:
         """The ``openapi_extra`` of a route that reads ``model`` from its body
         itself, validated (``_parse``)."""
@@ -593,16 +630,6 @@ def _is_async(handler: Callable[..., Any]) -> bool:
     )
 
 
-@contextmanager
-def _correlated(request: Request) -> Iterator[None]:
-    """Make the call's ``X-Correlation-ID`` the one the clients send."""
-    reset = correlation_id.set(request.headers.get("x-correlation-id"))
-    try:
-        yield
-    finally:
-        correlation_id.reset(reset)
-
-
 def _log_failure(what: str, error: Exception) -> None:
     """Log a handler's failure by its type and place only: its message, or the
     locations of a validation error, may quote the email, or a directory URL

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/spot_sdk/clients.py

@@ -1,10 +1,10 @@
 """The gateway client: what plugins ask of SPOT.
 
 Every call carries the plugin's own token (``Authorization: Bearer``) and
-the ``X-Correlation-ID`` of the call being handled, so SPOT's logs tie a
-plugin's calls to the analysis that made them. An error status raises
-:class:`SpotAPIError`, which names the route and the status, never the
-content.
+the current correlation ID (``spot_sdk.correlation``) as
+``X-Correlation-ID``, so SPOT's logs tie a plugin's calls to the analysis
+that made them. An error status raises :class:`SpotAPIError`, which names
+the route and the status, never the content.
 
 - Mail retrievers push raw messages with :meth:`SpotClient.ingest`.
 - Analyzers fetch the raw message or one decoded part with
@@ -17,7 +17,6 @@ from __future__ import annotations
 import asyncio
 import base64
 import json
-from contextvars import ContextVar
 from typing import Any, TypeVar
 from urllib.parse import quote
 from uuid import UUID
@@ -25,9 +24,15 @@ from uuid import UUID
 import httpx2
 from pydantic import BaseModel, SecretStr, ValidationError
 
+from . import correlation
 from .email import Envelope
 from .orchestrator import OrchestrationResult
-from .retriever import IngestionPriority, IngestionRequest, IngestionResult
+from .retriever import (
+    ActionReport,
+    IngestionPriority,
+    IngestionRequest,
+    IngestionResult,
+)
 from .settings import PluginSettings
 
 # The bytes of a raw message encoded at a time: a few milliseconds of a thread
@@ -37,9 +42,6 @@ _INGEST_PIECE = 768 * 1024
 
 ModelT = TypeVar("ModelT", bound=BaseModel)
 
-correlation_id: ContextVar[str | None] = ContextVar("spot_correlation_id", default=None)
-"""The ``X-Correlation-ID`` of the call being handled; the clients send it on."""
-
 
 class SpotAPIError(Exception):
     """SPOT answered an error status, or an answer that does not validate.
@@ -91,9 +93,8 @@ async def spot_request(
     The body is ``json`` encoded by :func:`json_body`, or ``content``, a JSON
     body the caller has already encoded."""
     headers = {"Authorization": f"Bearer {token.get_secret_value()}"}
-    current = correlation_id.get()
-    if current is not None:
-        headers["X-Correlation-ID"] = current
+    if (cid := correlation.current()) is not None:
+        headers[correlation.HEADER] = cid
     if json is not None:
         content = json_body(json)
     if content is not None:
@@ -219,6 +220,16 @@ class SpotClient:
         )
         return read_answer(IngestionResult, response, "POST", "/internal/ingest")
 
+    async def report_action(self, job_id: str, action: str, *, applied: bool) -> None:
+        """Tell SPOT what the retriever did with a message: ``action``, and whether
+        it was ``applied`` (in observe, it is not) (scope ``ingest``)."""
+        report = ActionReport(action=action, applied=applied)
+        await self._call(
+            "POST",
+            f"/internal/jobs/{quote(job_id, safe='')}/action",
+            content=json_body(report.model_dump(mode="json")),
+        )
+
     async def raw(self, email_id: UUID) -> bytes:
         """The stored message, as received (scope ``content:read``)."""
         response = await self._call("GET", f"/internal/emails/{email_id}/raw")

spot_sdk_python-2.0.0b11/spot_sdk/correlation.py

@@ -0,0 +1,65 @@
+"""The correlation ID that ties together the work done for one request or message.
+
+Core gives each request, message and scheduled sync an ID and sends it as
+``X-Correlation-ID`` to the plugins it calls. ``create_plugin_app`` makes
+each call's ID current (``spot_sdk.app.CorrelationMiddleware``), the SDK's
+clients send it back with their calls to SPOT, and the SDK's logs carry it
+(``spot_sdk.logging``).
+
+An ID is 1 to 128 characters of letters, digits and ``.``, ``_``, ``:``,
+``-``. Anything else is replaced with a new ID, so that a caller cannot write
+into the logs through it.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+
+HEADER = "X-Correlation-ID"
+_ID = re.compile(r"[A-Za-z0-9._:-]{1,128}")
+_current: ContextVar[str | None] = ContextVar("spot_correlation_id", default=None)
+
+
+def current() -> str | None:
+    """The ID of the work being done; None outside any."""
+    return _current.get()
+
+
+def new_id() -> str:
+    """A new ID."""
+    return str(uuid.uuid4())
+
+
+def accepted(value: str | None) -> str:
+    """``value`` when it is a valid ID, else a new one."""
+    return value if value is not None and _ID.fullmatch(value) else new_id()
+
+
+@contextmanager
+def correlated(value: str | None = None) -> Iterator[str]:
+    """Run the block under ``value``, or a new ID when it is None or invalid."""
+    cid = accepted(value)
+    token = _current.set(cid)
+    try:
+        yield cid
+    finally:
+        _current.reset(token)
+
+
+def adopt(value: str | None) -> str:
+    """Make ``value``, or a new ID, current for the rest of the calling task.
+
+    Nothing restores the previous ID: use it in a task that does one piece of
+    work, as uvicorn runs each request, so that whatever runs after the code
+    that set it (an app's 500 handler, uvicorn's error line) still sees it. In a
+    task that serves several requests, the last one's ID stays set after it:
+    a test that calls an app through ``httpx2.ASGITransport`` from its own task
+    keeps that ID until the test ends.
+    """
+    cid = accepted(value)
+    _current.set(cid)
+    return cid

spot_sdk_python-2.0.0b11/spot_sdk/logging.py

@@ -0,0 +1,284 @@
+"""The logging of SPOT's services and plugins: one JSON object per line.
+
+``configure_logging`` writes every record to stdout (and ``LOG_FILE`` when
+set): one JSON object on one line by default (``LOG_FORMAT=json``), or, for
+development, a readable line, the traceback after it (``LOG_FORMAT=text``).
+Each record holds the current correlation ID (``spot_sdk.correlation``).
+uvicorn's lines go the same way, its access lines without their query string
+and its unhandled exceptions, a lifespan's failure included, by type and
+frames; so do Python's warnings.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import logging.config
+import os
+import re
+import traceback
+from datetime import UTC, datetime
+from typing import Any
+
+from .correlation import current
+
+LOG_FORMATS = ("json", "text")
+
+# A record's own attributes; anything else on it came from ``extra=``.
+# ``color_message`` is uvicorn's coloured copy of the message.
+_RECORD_FIELDS = frozenset(vars(logging.LogRecord("", 0, "", 0, "", None, None))) | {
+    "message",
+    "asctime",
+    "correlation_id",
+    "color_message",
+}
+# A traceback written as text: its header, a frame's line, an exception's type.
+_TRACEBACK = "Traceback (most recent call last):"
+_FRAME = re.compile(r'File "(.*)", line (\d+), in (.*)')
+_TYPE = re.compile(r"[A-Za-z_][\w.]*")
+
+
+class CorrelationFilter(logging.Filter):
+    """Puts the current correlation ID on each record (``correlation_id``, None outside any work)."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        # Not a LogRecord attribute: mypy refuses the assignment.
+        setattr(record, "correlation_id", current())
+        return True
+
+
+class AccessQueryFilter(logging.Filter):
+    """Cuts the query string off uvicorn's access lines: it can hold search terms and addresses.
+
+    uvicorn logs ``(client, method, path with query, http version, status)``.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if isinstance(record.args, tuple) and len(record.args) == 5:
+            client, method, path, version, status = record.args
+            record.args = (client, method, str(path).split("?", 1)[0], version, status)
+        return True
+
+
+class ServerErrorFilter(logging.Filter):
+    """Logs uvicorn's unhandled exceptions by type and frames: their text, and
+    the source lines of a traceback, can quote a request's content.
+
+    uvicorn logs a request's exception as ``exc_info``, and a lifespan's
+    failure as the traceback's text that Starlette sends it.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        error = record.exc_info[1] if record.exc_info else None
+        if error is not None:
+            frames = "\n".join(
+                _frame(frame.filename, frame.lineno, frame.name)
+                for frame in traceback.extract_tb(error.__traceback__)
+            )
+            record.args = (record.getMessage().strip(), type(error).__name__, frames)
+            record.msg = "%s: %s, raised at:\n%s"
+            record.exc_info = None
+            record.exc_text = None
+        elif (written := _written_traceback(record.getMessage())) is not None:
+            record.args = written
+            record.msg = "%s, raised at:\n%s"
+        return True
+
+
+def _frame(filename: str, lineno: object, name: str) -> str:
+    """A frame as a traceback writes it, without its source line."""
+    return f'  File "{filename}", line {lineno}, in {name}'
+
+
+def _written_traceback(text: str) -> tuple[str, str] | None:
+    """The type and frames of the first traceback written in ``text``; None
+    when it holds none.
+
+    The first: an exception group is named by its own type and frames, a
+    chain by its root cause, and a traceback quoted in a message or a note is
+    never read. The type is read from the line that names it, the first after
+    the frames that is less indented than them (an exception group's ``|``
+    counts as indent), never from the lines after it: the message, which can
+    run over several lines, and the notes.
+    """
+    start = text.find(_TRACEBACK)
+    if start < 0:
+        return None
+    kind, depth = "unknown", 0
+    frames: list[str] = []
+    for line in text[start + len(_TRACEBACK) :].splitlines():
+        content = line.lstrip(" |")
+        indent = len(line) - len(content)
+        if frame := _FRAME.fullmatch(content):
+            frames.append(_frame(frame[1], frame[2], frame[3]))
+            depth = indent
+        elif frames and content and indent < depth:
+            name = content.split(":", 1)[0]
+            kind = name if _TYPE.fullmatch(name) else kind
+            break
+    return kind, "\n".join(frames)
+
+
+class JsonFormatter(logging.Formatter):
+    """One JSON object per record, on one line."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        entry: dict[str, Any] = {
+            "timestamp": datetime.fromtimestamp(record.created, UTC)
+            .isoformat(timespec="milliseconds")
+            .replace("+00:00", "Z"),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        cid = getattr(record, "correlation_id", None)
+        if cid:
+            entry["correlation_id"] = cid
+        if record.exc_info:
+            entry["exception"] = self.formatException(record.exc_info)
+        if record.stack_info:
+            entry["stack"] = self.formatStack(record.stack_info)
+        extra = {
+            key: value
+            for key, value in vars(record).items()
+            if key not in _RECORD_FIELDS and key not in entry
+        }
+        try:
+            return json.dumps({**entry, **extra}, default=str, allow_nan=False)
+        except (TypeError, ValueError):
+            # A NaN, or a key JSON cannot hold, in an extra field: all as text.
+            text = {key: str(value) for key, value in extra.items()}
+            return json.dumps({**entry, **text})
+
+
+class TextFormatter(logging.Formatter):
+    """A readable line per record, the traceback after it: time, level, logger,
+    correlation ID, message."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        cid = getattr(record, "correlation_id", None) or "-"
+        line = f"{self.formatTime(record)} {record.levelname} {record.name} [{cid}] {record.getMessage()}"
+        if record.exc_info:
+            line += "\n" + self.formatException(record.exc_info)
+        return line
+
+
+def get_log_level() -> str:
+    """The log level from ``LOG_LEVEL``, default ``INFO``."""
+    return os.getenv("LOG_LEVEL", "INFO").upper()
+
+
+def get_log_format() -> str:
+    """The log format from ``LOG_FORMAT``, default ``json``."""
+    return os.getenv("LOG_FORMAT", "json").lower()
+
+
+def get_logging_config(
+    log_level: str | None = None, log_format: str | None = None
+) -> dict[str, Any]:
+    """The ``dictConfig`` of ``configure_logging``; ``ValueError`` for a format
+    other than ``json`` and ``text``.
+
+    The level and the format default to ``LOG_LEVEL`` and ``LOG_FORMAT``;
+    ``LOG_FILE``, when set, gets the same lines as stdout.
+    """
+    level = (log_level or get_log_level()).upper()
+    fmt = (log_format or get_log_format()).lower()
+    if fmt not in LOG_FORMATS:
+        raise ValueError(
+            f"LOG_FORMAT must be one of {', '.join(LOG_FORMATS)}, not {fmt!r}"
+        )
+    handler = {
+        "class": "logging.StreamHandler",
+        "formatter": "default",
+        "filters": ["correlation"],
+        "stream": "ext://sys.stdout",
+    }
+    config: dict[str, Any] = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "filters": {
+            "correlation": {"()": CorrelationFilter},
+            "access_query": {"()": AccessQueryFilter},
+            "server_error": {"()": ServerErrorFilter},
+        },
+        "formatters": {
+            "default": {"()": JsonFormatter if fmt == "json" else TextFormatter}
+        },
+        "handlers": {"console": handler},
+        "loggers": {
+            # uvicorn configures these before it imports the app: they keep
+            # their level and write through the root handler instead.
+            "uvicorn": {"handlers": [], "propagate": True},
+            "uvicorn.error": {"filters": ["server_error"]},
+            "uvicorn.access": {
+                "handlers": [],
+                "filters": ["access_query"],
+                "propagate": True,
+            },
+            # At INFO, httpx and httpx2 write each request's URL, and at DEBUG
+            # httpcore and httpcore2 (their sub-loggers inherit the level) the
+            # raw headers: any of them may hold a credential.
+            "httpx": {"level": "WARNING"},
+            "httpx2": {"level": "WARNING"},
+            "httpcore": {"level": "WARNING"},
+            "httpcore2": {"level": "WARNING"},
+        },
+        "root": {"level": level, "handlers": ["console"]},
+    }
+    log_file = os.getenv("LOG_FILE")
+    if log_file:
+        config["handlers"]["file"] = {
+            "class": "logging.FileHandler",
+            "formatter": "default",
+            "filters": ["correlation"],
+            "filename": log_file,
+            "mode": "a",
+        }
+        config["root"]["handlers"].append("file")
+    return config
+
+
+def configure_logging(
+    log_level: str | None = None, log_format: str | None = None
+) -> None:
+    """Configure logging once, at import, before the server writes its first lines.
+
+    The level and the format default to ``LOG_LEVEL`` (``INFO``) and
+    ``LOG_FORMAT`` (``json``); ``LOG_FILE``, when set, gets the same lines
+    as stdout. Python's warnings become log lines. A second call replaces
+    the first.
+    """
+    config = get_logging_config(log_level, log_format)
+    # dictConfig replaces a logger's handlers but adds to its filters.
+    for name, options in config["loggers"].items():
+        if "filters" in options:
+            logging.getLogger(name).filters.clear()
+    logging.config.dictConfig(config)
+    # warnings.warn writes to stderr otherwise: one more line that is not JSON.
+    logging.captureWarnings(True)
+    log_file = os.getenv("LOG_FILE")
+    logging.getLogger("spot.logging").info(
+        "Logging configured: level %s, format %s, output %s",
+        (log_level or get_log_level()).upper(),
+        (log_format or get_log_format()).lower(),
+        f"console and file {log_file}" if log_file else "console",
+    )
+
+
+def get_logger(name: str) -> logging.Logger:
+    """
+    Get a logger instance for the given name.
+
+    Args:
+        name: Logger name, typically __name__
+
+    Returns:
+        Standard Python logger
+
+    Example:
+        logger = get_logger(__name__)
+        logger.info("Service started")
+        logger.debug("Processing email", extra={"email_id": "123"})
+    """
+    return logging.getLogger(name)

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/spot_sdk/orchestrator.py

@@ -88,5 +88,5 @@ class RespondRequest(BaseModel):
     decided_by: str
     signals: list[AttributedSignal]
     mode: ResponseMode = Field(
-        description="The workflow's response_mode; a responder acts only in enforce"
+        description="The job's mode (its connection's and its route's); a responder acts only in enforce"
     )

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/spot_sdk/retriever.py

@@ -35,6 +35,7 @@ from pydantic import (
 
 from .email import Envelope
 from .verdict import Score, VerdictClass
+from .workflow import ResponseMode
 
 
 def _decode_message(value: object) -> bytes:
@@ -115,3 +116,18 @@ class IngestionResult(BaseModel):
     error: str | None = Field(
         default=None, description="Why there is no verdict, when the status is timeout"
     )
+    mode: ResponseMode = Field(
+        default="observe",
+        description=(
+            "The job's mode: enforce only when its connection and its route both "
+            "are; a retriever applies its actions only in enforce"
+        ),
+    )
+
+
+class ActionReport(BaseModel):
+    """The body of ``POST /internal/jobs/{job_id}/action``: what a retriever
+    did with a message (or, in observe, would have done)."""
+
+    action: str = Field(pattern=r"^[a-z][a-z0-9_]{0,31}$")
+    applied: bool

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/spot_sdk/testing/fake_spot.py

@@ -21,24 +21,33 @@ from ..email import Envelope
 from ..mime import PartNotFoundError, part_bytes
 from ..orchestrator import OrchestrationResult
 from ..retriever import (
+    ActionReport,
     IngestionPriority,
     IngestionRequest,
     IngestionResult,
     IngestionStatus,
 )
 from ..verdict import VerdictClass
+from ..workflow import ResponseMode
 
 
 class FakeSpot:
     """In-memory gateway with the methods of ``SpotClient``."""
 
     def __init__(
-        self, *, verdict: VerdictClass = VerdictClass.BENIGN, score: int = 0
+        self,
+        *,
+        verdict: VerdictClass = VerdictClass.BENIGN,
+        score: int = 0,
+        mode: ResponseMode = "observe",
     ) -> None:
-        """``verdict`` and ``score`` answer an ingest that waits for the verdict."""
+        """``verdict`` and ``score`` answer an ingest that waits for the verdict;
+        ``mode`` is the mode of every job."""
         self.verdict = verdict
         self.score = score
+        self.mode = mode
         self.ingested: list[IngestionRequest] = []
+        self.actions: list[tuple[str, ActionReport]] = []
         self._emails: dict[UUID, bytes] = {}
         self._results: dict[str, OrchestrationResult] = {}
 
@@ -73,7 +82,10 @@ class FakeSpot:
         email_id = self.add_email(request.raw)
         if not wait_for_verdict:
             return IngestionResult(
-                job_id=job_id, email_id=email_id, status=IngestionStatus.ACCEPTED
+                job_id=job_id,
+                email_id=email_id,
+                status=IngestionStatus.ACCEPTED,
+                mode=self.mode,
             )
         return IngestionResult(
             job_id=job_id,
@@ -81,8 +93,12 @@ class FakeSpot:
             status=IngestionStatus.ANALYZED,
             verdict=self.verdict,
             score=self.score,
+            mode=self.mode,
         )
 
+    async def report_action(self, job_id: str, action: str, *, applied: bool) -> None:
+        self.actions.append((job_id, ActionReport(action=action, applied=applied)))
+
     async def raw(self, email_id: UUID) -> bytes:
         return self._stored(email_id, f"/internal/emails/{email_id}/raw")
 

{spot_sdk_python-2.0.0b9 → spot_sdk_python-2.0.0b11}/spot_sdk/workflow.py

@@ -10,7 +10,7 @@ from pydantic import BaseModel, Field, model_validator
 from .verdict import VerdictClass
 
 ResponseMode = Literal["observe", "enforce"]
-"""Whether responders act on verdicts (``enforce``) or only record them (``observe``)."""
+"""Whether responders and retrievers act on verdicts (``enforce``) or only record them (``observe``): the mode of a job, from its connection and its route."""
 
 DECISIVE_SIGNAL_PATTERN = r"^[a-z0-9_]+(\.[a-z0-9_]+)*\.([a-z0-9_]+|\*)$"
 """A signal name, or a prefix ending in ``.*``."""
@@ -181,10 +181,6 @@ class Workflow(BaseModel):
     on_verdict: dict[VerdictClass, list[str]] = Field(
         default_factory=dict, description="Responder ids to call, by verdict"
     )
-    response_mode: ResponseMode = Field(
-        default="observe",
-        description="Sent to responders as mode; a responder acts only in enforce",
-    )
 
     @model_validator(mode="after")
     def _decision_stages_exist(self) -> Self:

spot_sdk_python-2.0.0b9/spot_sdk/logging.py

@@ -1,133 +0,0 @@
-"""
-Standard logging configuration for SPOT platform.
-
-Uses Python's built-in logging with dictConfig.
-Simple, configurable via environment variables.
-"""
-
-from __future__ import annotations
-
-import logging
-import logging.config
-import os
-from typing import Any, Optional
-
-
-def get_log_level() -> str:
-    """Get log level from environment, default to INFO."""
-    return os.getenv("LOG_LEVEL", "INFO").upper()
-
-
-def get_log_format() -> str:
-    """Get log format from environment, default to simple."""
-    return os.getenv("LOG_FORMAT", "simple").lower()
-
-
-def get_logging_config(
-    log_level: Optional[str] = None, log_format: Optional[str] = None
-) -> dict[str, Any]:
-    """
-    Get logging configuration dictionary for dictConfig.
-
-    Args:
-        log_level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
-        log_format: Format style (simple, detailed, json)
-
-    Returns:
-        Dictionary suitable for logging.config.dictConfig
-    """
-    level = log_level or get_log_level()
-    fmt = log_format or get_log_format()
-
-    # Choose format string
-    if fmt == "json":
-        format_str = '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "logger": "%(name)s", "message": "%(message)s"}'
-    elif fmt == "detailed":
-        format_str = "%(asctime)s - %(name)-20s - %(levelname)-8s - [%(filename)s:%(lineno)d] - %(message)s"
-    else:  # simple
-        format_str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-
-    config = {
-        "version": 1,
-        "disable_existing_loggers": False,
-        "formatters": {
-            "default": {
-                "format": format_str,
-                "datefmt": "%Y-%m-%d %H:%M:%S" if fmt == "detailed" else "%H:%M:%S",
-            }
-        },
-        "handlers": {
-            "console": {
-                "class": "logging.StreamHandler",
-                "formatter": "default",
-                "stream": "ext://sys.stdout",
-            }
-        },
-        "root": {"level": level, "handlers": ["console"]},
-    }
-
-    # Add file handler if LOG_FILE is specified
-    log_file = os.getenv("LOG_FILE")
-    if log_file:
-        config["handlers"]["file"] = {  # type: ignore[index]
-            "class": "logging.FileHandler",
-            "formatter": "default",
-            "filename": log_file,
-            "mode": "a",
-        }
-        config["root"]["handlers"].append("file")  # type: ignore[index]
-
-    return config
-
-
-def configure_logging(
-    log_level: Optional[str] = None, log_format: Optional[str] = None
-) -> None:
-    """
-    Configure logging for the application.
-
-    Call this once during application startup.
-
-    Args:
-        log_level: Log level (default: from LOG_LEVEL env var or INFO)
-        log_format: Format style (default: from LOG_FORMAT env var or simple)
-
-    Environment Variables:
-        LOG_LEVEL: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO)
-        LOG_FORMAT: json, simple, detailed (default: simple)
-        LOG_FILE: Path to log file (optional, logs to console if not set)
-    """
-    config = get_logging_config(log_level, log_format)
-    logging.config.dictConfig(config)
-
-    # Log configuration
-    logger = logging.getLogger("spot.logging")
-    level = log_level or get_log_level()
-    fmt = log_format or get_log_format()
-    log_file = os.getenv("LOG_FILE")
-
-    log_targets = "console"
-    if log_file:
-        log_targets += f" and file ({log_file})"
-
-    logger.info(
-        f"Logging configured - Level: {level}, Format: {fmt}, Output: {log_targets}"
-    )
-
-
-def get_logger(name: str) -> logging.Logger:
-    """
-    Get a logger instance for the given name.
-
-    Args:
-        name: Logger name, typically __name__
-
-    Returns:
-        Standard Python logger
-
-    Example:
-        logger = get_logger(__name__)
-        logger.info("Service started")
-        logger.debug("Processing email", extra={"email_id": "123"})
-    """
-    return logging.getLogger(name)