spot-sdk-python 2.0.0b10__tar.gz → 2.0.0b12__tar.gz

{spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spot-sdk-python
-Version: 2.0.0b10
+Version: 2.0.0b12
 Summary: SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers
 License: Apache-2.0
 Author: SPOT Project
@@ -51,7 +51,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b10
+pip install spot-sdk-python==2.0.0b12
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for

{spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b12}/README.md

@@ -26,7 +26,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b10
+pip install spot-sdk-python==2.0.0b12
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for

{spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b12}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "spot-sdk-python"
-version = "2.0.0b10"
+version = "2.0.0b12"
 description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
 authors = ["SPOT Project <spot@sonn.lu>"]
 license = "Apache-2.0"

{spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b12}/spot_sdk/__init__.py

@@ -25,6 +25,7 @@ from .orchestrator import (
     RespondRequest,
     StageOutcome,
 )
+from .proxy import PROXY_SECRET, apply_proxy_credentials
 from .retriever import ActionReport, IngestionRequest, IngestionResult, IngestionStatus
 from .settings import PluginSettings
 from .verdict import AnalyzerVerdict, Score, Signal, VerdictClass
@@ -42,7 +43,7 @@ from .workflow import (
 )
 
 if TYPE_CHECKING:
-    from .app import PluginHealth, SyncState, create_plugin_app
+    from .app import CorrelationMiddleware, PluginHealth, SyncState, create_plugin_app
     from .clients import SpotAPIError, SpotClient
     from .knowledge import KnowledgeClient, KnowledgeDocument, chunk_text, content_hash
     from .llm import LLMClient, LLMOutputError, LLMToolCall, LLMToolCalls, LLMToolRound
@@ -50,7 +51,10 @@ if TYPE_CHECKING:
 # The modules that import FastAPI, Starlette or httpx2 load on first use, so importing
 # `spot_sdk.mime` or the models stays light (PEP 562).
 _LAZY = {
-    **dict.fromkeys(("PluginHealth", "SyncState", "create_plugin_app"), ".app"),
+    **dict.fromkeys(
+        ("CorrelationMiddleware", "PluginHealth", "SyncState", "create_plugin_app"),
+        ".app",
+    ),
     **dict.fromkeys(("SpotAPIError", "SpotClient"), ".clients"),
     **dict.fromkeys(
         ("KnowledgeClient", "KnowledgeDocument", "chunk_text", "content_hash"),
@@ -123,9 +127,12 @@ __all__ = [
     "SettingField",
     "SettingType",
     # Plugin runtime
+    "CorrelationMiddleware",
+    "PROXY_SECRET",
     "PluginHealth",
     "PluginSettings",
     "SyncState",
+    "apply_proxy_credentials",
     "create_plugin_app",
     # Clients
     "LLMClient",

{spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b12}/spot_sdk/app.py

@@ -13,7 +13,11 @@ class and its handlers:
 
 An ``async def`` handler, or an object whose ``__call__`` is ``async def``,
 runs on the event loop; any other callable runs in a worker thread, so it may
-block. Either way the clients send the call's ``X-Correlation-ID`` on to SPOT.
+block.
+
+Each request runs under its ``X-Correlation-ID``, or a new one when it has
+none or an invalid one (``CorrelationMiddleware``): the answer carries it,
+the logs hold it, and the clients send it on to SPOT.
 
 A plugin that loads a model gives ``load``: run in a daemon thread once the app
 starts, it makes the plugin ``starting`` until it returns, and the analyzer
@@ -35,8 +39,8 @@ import hmac
 import inspect
 import threading
 import traceback
-from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
-from contextlib import asynccontextmanager, contextmanager
+from collections.abc import AsyncIterator, Awaitable, Callable
+from contextlib import asynccontextmanager
 from datetime import UTC, datetime
 from typing import Any, Generic, Literal, TypeAlias, TypeVar, overload
 
@@ -47,12 +51,14 @@ from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
 from pydantic.json_schema import models_json_schema
 from pydantic_core import PydanticSerializationError
 from starlette.concurrency import run_in_threadpool
-from starlette.types import Lifespan
+from starlette.datastructures import Headers, MutableHeaders
+from starlette.types import ASGIApp, Lifespan, Message, Receive, Scope, Send
 
-from .clients import correlation_id
+from .correlation import HEADER, adopt
 from .logging import get_logger
 from .manifest import Manifest, PluginKind
 from .orchestrator import AnalyzeRequest, RespondRequest
+from .proxy import apply_proxy_credentials
 from .settings import PluginSettings
 from .verdict import AnalyzerVerdict
 
@@ -215,6 +221,10 @@ def create_plugin_app(
 ) -> FastAPI:
     """Build the plugin's app; the settings are read once, here.
 
+    First, the egress proxy's credentials, which core mounts as a secret
+    file, go back into the proxy variables (``apply_proxy_credentials``), so
+    the plugin's HTTP clients reach an authenticated proxy.
+
     The manifest's kind needs its handler (analyzer: ``analyze``, responder:
     ``respond``, context provider: ``sync``; a mail retriever has none), and
     each handler its kind, or ``ValueError`` is raised.
@@ -248,6 +258,7 @@ def create_plugin_app(
     components of that document alone, so read it from this app: another
     FastAPI app that includes its routes leaves their ``$ref`` dangling.
     """
+    apply_proxy_credentials()
     _check_kind(manifest, {"analyze": analyze, "respond": respond, "sync": sync})
     settings = settings_cls()
     if not settings.spot_plugin_token.get_secret_value():
@@ -303,12 +314,11 @@ def create_plugin_app(
         )
         async def analyze_route(request: Request) -> Response:
             call = await _parse(request, AnalyzeRequest)
-            with _correlated(request):
-                try:
-                    answer = await _run(analyze, call, settings)
-                except Exception as error:
-                    _log_failure(f"analysis of job {call.job_id}", error)
-                    return JSONResponse({"detail": "analysis failed"}, status_code=500)
+            try:
+                answer = await _run(analyze, call, settings)
+            except Exception as error:
+                _log_failure(f"analysis of job {call.job_id}", error)
+                return JSONResponse({"detail": "analysis failed"}, status_code=500)
             try:
                 verdict = AnalyzerVerdict.model_validate(answer)
             except ValidationError as error:
@@ -332,21 +342,19 @@ def create_plugin_app(
         )
         async def respond_route(request: Request) -> Response:
             call = await _parse(request, RespondRequest)
-            with _correlated(request):
-                try:
-                    await _run(respond, call, settings)
-                except Exception as error:
-                    _log_failure(f"response to job {call.job_id}", error)
-                    return JSONResponse({"detail": "response failed"}, status_code=500)
+            try:
+                await _run(respond, call, settings)
+            except Exception as error:
+                _log_failure(f"response to job {call.job_id}", error)
+                return JSONResponse({"detail": "response failed"}, status_code=500)
             return Response(status_code=204)
 
     if sync is not None:
         syncs = _SyncRunner(sync, settings)
 
         @internal.post("/sync", status_code=202)
-        async def sync_route(request: Request) -> SyncState:
-            with _correlated(request):
-                return syncs.start()
+        async def sync_route() -> SyncState:
+            return syncs.start()
 
         @internal.get("/state")
         async def state_route() -> SyncState:
@@ -356,14 +364,49 @@ def create_plugin_app(
     return app
 
 
+class CorrelationMiddleware:
+    """Gives each HTTP request its ID, the ``X-Correlation-ID`` sent when valid,
+    else a new one, and sends it back with the answer.
+
+    The ID stays current for the rest of the request's task: add this
+    middleware last, so that it is the app's outermost, and the other
+    middleware, the app's 500 handler and the server's error line all run
+    under it. The plugin app wraps its whole stack in it, Starlette's error
+    middleware included, so that a 500 answer carries the ID too.
+    """
+
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+        cid = adopt(Headers(scope=scope).get(HEADER))
+
+        async def send_with_id(message: Message) -> None:
+            if message["type"] == "http.response.start":
+                MutableHeaders(scope=message)[HEADER] = cid
+            await send(message)
+
+        await self.app(scope, receive, send_with_id)
+
+
 class _PluginApp(FastAPI):
     """The plugin's app. Its OpenAPI document also holds the schemas of the
-    request bodies its routes read themselves, which FastAPI leaves out."""
+    request bodies its routes read themselves, which FastAPI leaves out, and
+    ``CorrelationMiddleware`` wraps its whole stack."""
 
     def __init__(self, **options: Any) -> None:
         super().__init__(**options)
         self._read_bodies: list[type[BaseModel]] = []
 
+    def build_middleware_stack(self) -> ASGIApp:
+        """The stack inside ``CorrelationMiddleware``: outside Starlette's error
+        middleware, whose 500 then carries the ID, and outside any middleware
+        the plugin adds, which Starlette would put outside it otherwise."""
+        return CorrelationMiddleware(super().build_middleware_stack())
+
     def reads(self, model: type[BaseModel]) -> dict[str, Any]:
         """The ``openapi_extra`` of a route that reads ``model`` from its body
         itself, validated (``_parse``)."""
@@ -593,16 +636,6 @@ def _is_async(handler: Callable[..., Any]) -> bool:
     )
 
 
-@contextmanager
-def _correlated(request: Request) -> Iterator[None]:
-    """Make the call's ``X-Correlation-ID`` the one the clients send."""
-    reset = correlation_id.set(request.headers.get("x-correlation-id"))
-    try:
-        yield
-    finally:
-        correlation_id.reset(reset)
-
-
 def _log_failure(what: str, error: Exception) -> None:
     """Log a handler's failure by its type and place only: its message, or the
     locations of a validation error, may quote the email, or a directory URL

{spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b12}/spot_sdk/clients.py

@@ -1,10 +1,10 @@
 """The gateway client: what plugins ask of SPOT.
 
 Every call carries the plugin's own token (``Authorization: Bearer``) and
-the ``X-Correlation-ID`` of the call being handled, so SPOT's logs tie a
-plugin's calls to the analysis that made them. An error status raises
-:class:`SpotAPIError`, which names the route and the status, never the
-content.
+the current correlation ID (``spot_sdk.correlation``) as
+``X-Correlation-ID``, so SPOT's logs tie a plugin's calls to the analysis
+that made them. An error status raises :class:`SpotAPIError`, which names
+the route and the status, never the content.
 
 - Mail retrievers push raw messages with :meth:`SpotClient.ingest`.
 - Analyzers fetch the raw message or one decoded part with
@@ -17,7 +17,6 @@ from __future__ import annotations
 import asyncio
 import base64
 import json
-from contextvars import ContextVar
 from typing import Any, TypeVar
 from urllib.parse import quote
 from uuid import UUID
@@ -25,6 +24,7 @@ from uuid import UUID
 import httpx2
 from pydantic import BaseModel, SecretStr, ValidationError
 
+from . import correlation
 from .email import Envelope
 from .orchestrator import OrchestrationResult
 from .retriever import (
@@ -42,9 +42,6 @@ _INGEST_PIECE = 768 * 1024
 
 ModelT = TypeVar("ModelT", bound=BaseModel)
 
-correlation_id: ContextVar[str | None] = ContextVar("spot_correlation_id", default=None)
-"""The ``X-Correlation-ID`` of the call being handled; the clients send it on."""
-
 
 class SpotAPIError(Exception):
     """SPOT answered an error status, or an answer that does not validate.
@@ -96,9 +93,8 @@ async def spot_request(
     The body is ``json`` encoded by :func:`json_body`, or ``content``, a JSON
     body the caller has already encoded."""
     headers = {"Authorization": f"Bearer {token.get_secret_value()}"}
-    current = correlation_id.get()
-    if current is not None:
-        headers["X-Correlation-ID"] = current
+    if (cid := correlation.current()) is not None:
+        headers[correlation.HEADER] = cid
     if json is not None:
         content = json_body(json)
     if content is not None:

spot_sdk_python-2.0.0b12/spot_sdk/correlation.py

@@ -0,0 +1,65 @@
+"""The correlation ID that ties together the work done for one request or message.
+
+Core gives each request, message and scheduled sync an ID and sends it as
+``X-Correlation-ID`` to the plugins it calls. ``create_plugin_app`` makes
+each call's ID current (``spot_sdk.app.CorrelationMiddleware``), the SDK's
+clients send it back with their calls to SPOT, and the SDK's logs carry it
+(``spot_sdk.logging``).
+
+An ID is 1 to 128 characters of letters, digits and ``.``, ``_``, ``:``,
+``-``. Anything else is replaced with a new ID, so that a caller cannot write
+into the logs through it.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+
+HEADER = "X-Correlation-ID"
+_ID = re.compile(r"[A-Za-z0-9._:-]{1,128}")
+_current: ContextVar[str | None] = ContextVar("spot_correlation_id", default=None)
+
+
+def current() -> str | None:
+    """The ID of the work being done; None outside any."""
+    return _current.get()
+
+
+def new_id() -> str:
+    """A new ID."""
+    return str(uuid.uuid4())
+
+
+def accepted(value: str | None) -> str:
+    """``value`` when it is a valid ID, else a new one."""
+    return value if value is not None and _ID.fullmatch(value) else new_id()
+
+
+@contextmanager
+def correlated(value: str | None = None) -> Iterator[str]:
+    """Run the block under ``value``, or a new ID when it is None or invalid."""
+    cid = accepted(value)
+    token = _current.set(cid)
+    try:
+        yield cid
+    finally:
+        _current.reset(token)
+
+
+def adopt(value: str | None) -> str:
+    """Make ``value``, or a new ID, current for the rest of the calling task.
+
+    Nothing restores the previous ID: use it in a task that does one piece of
+    work, as uvicorn runs each request, so that whatever runs after the code
+    that set it (an app's 500 handler, uvicorn's error line) still sees it. In a
+    task that serves several requests, the last one's ID stays set after it:
+    a test that calls an app through ``httpx2.ASGITransport`` from its own task
+    keeps that ID until the test ends.
+    """
+    cid = accepted(value)
+    _current.set(cid)
+    return cid

{spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b12}/spot_sdk/knowledge.py

@@ -64,8 +64,8 @@ class KnowledgeDocument(BaseModel):
     id: str = Field(
         ...,
         description=(
-            "Stable unique id (e.g. 'employee:alice@co.com', "
-            "'wiki:page-123#chunk-4'). Upsert is idempotent on this key."
+            "Stable id, unique within its source (e.g. 'employee:alice@co.com'). "
+            "Upsert is idempotent on (source, id)."
         ),
     )
     content: str = Field(
@@ -88,7 +88,10 @@ class KnowledgeDocument(BaseModel):
     )
     source: str = Field(
         default="",
-        description="Provider that produced the document (audit + future ownership).",
+        description=(
+            "The provider that wrote it: its plugin id, set by the store for a "
+            "plugin's write."
+        ),
     )
     updated_at: datetime | None = None
     expires_at: datetime | None = Field(
@@ -103,6 +106,16 @@ class KnowledgeDocument(BaseModel):
     )
 
 
+def _documents_path(source: str) -> str:
+    """The store's path of a source's documents, the source quoted."""
+    return f"/sources/{quote(source, safe='')}/documents"
+
+
+def _document_path(source: str, doc_id: str) -> str:
+    """The store's path of one of a source's documents, both parts quoted."""
+    return f"{_documents_path(source)}/{quote(doc_id, safe='')}"
+
+
 class _QueryAnswer(BaseModel):
     """The store's answer to ``POST /query``."""
 
@@ -256,8 +269,10 @@ class KnowledgeClient:
         await self._call("POST", "/bulk-upsert", json=payload)
 
     async def delete(self, doc_id: str) -> None:
-        """Delete a document by id."""
-        await self._call("DELETE", f"/documents/{quote(doc_id, safe='')}")
+        """Delete one of the plugin's own documents, by its id within the plugin's source."""
+        if not self.source:
+            raise ValueError("delete needs the plugin's id: set SPOT_PLUGIN_ID")
+        await self._call("DELETE", _document_path(self.source, doc_id))
 
     async def replace_source(self, docs: list[KnowledgeDocument]) -> None:
         """Make ``docs`` the plugin's whole set of documents.
@@ -269,8 +284,7 @@ class KnowledgeClient:
         if not self.source:
             raise ValueError("replace_source needs the plugin's id: set SPOT_PLUGIN_ID")
         payload = {"documents": [d.model_dump(mode="json") for d in docs]}
-        path = f"/sources/{quote(self.source, safe='')}/documents"
-        await self._call("PUT", path, json=payload)
+        await self._call("PUT", _documents_path(self.source), json=payload)
 
     # ------------------------------------------------------------------ #
     # Read side: analyzers

spot_sdk_python-2.0.0b12/spot_sdk/logging.py

@@ -0,0 +1,284 @@
+"""The logging of SPOT's services and plugins: one JSON object per line.
+
+``configure_logging`` writes every record to stdout (and ``LOG_FILE`` when
+set): one JSON object on one line by default (``LOG_FORMAT=json``), or, for
+development, a readable line, the traceback after it (``LOG_FORMAT=text``).
+Each record holds the current correlation ID (``spot_sdk.correlation``).
+uvicorn's lines go the same way, its access lines without their query string
+and its unhandled exceptions, a lifespan's failure included, by type and
+frames; so do Python's warnings.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import logging.config
+import os
+import re
+import traceback
+from datetime import UTC, datetime
+from typing import Any
+
+from .correlation import current
+
+LOG_FORMATS = ("json", "text")
+
+# A record's own attributes; anything else on it came from ``extra=``.
+# ``color_message`` is uvicorn's coloured copy of the message.
+_RECORD_FIELDS = frozenset(vars(logging.LogRecord("", 0, "", 0, "", None, None))) | {
+    "message",
+    "asctime",
+    "correlation_id",
+    "color_message",
+}
+# A traceback written as text: its header, a frame's line, an exception's type.
+_TRACEBACK = "Traceback (most recent call last):"
+_FRAME = re.compile(r'File "(.*)", line (\d+), in (.*)')
+_TYPE = re.compile(r"[A-Za-z_][\w.]*")
+
+
+class CorrelationFilter(logging.Filter):
+    """Puts the current correlation ID on each record (``correlation_id``, None outside any work)."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        # Not a LogRecord attribute: mypy refuses the assignment.
+        setattr(record, "correlation_id", current())
+        return True
+
+
+class AccessQueryFilter(logging.Filter):
+    """Cuts the query string off uvicorn's access lines: it can hold search terms and addresses.
+
+    uvicorn logs ``(client, method, path with query, http version, status)``.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if isinstance(record.args, tuple) and len(record.args) == 5:
+            client, method, path, version, status = record.args
+            record.args = (client, method, str(path).split("?", 1)[0], version, status)
+        return True
+
+
+class ServerErrorFilter(logging.Filter):
+    """Logs uvicorn's unhandled exceptions by type and frames: their text, and
+    the source lines of a traceback, can quote a request's content.
+
+    uvicorn logs a request's exception as ``exc_info``, and a lifespan's
+    failure as the traceback's text that Starlette sends it.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        error = record.exc_info[1] if record.exc_info else None
+        if error is not None:
+            frames = "\n".join(
+                _frame(frame.filename, frame.lineno, frame.name)
+                for frame in traceback.extract_tb(error.__traceback__)
+            )
+            record.args = (record.getMessage().strip(), type(error).__name__, frames)
+            record.msg = "%s: %s, raised at:\n%s"
+            record.exc_info = None
+            record.exc_text = None
+        elif (written := _written_traceback(record.getMessage())) is not None:
+            record.args = written
+            record.msg = "%s, raised at:\n%s"
+        return True
+
+
+def _frame(filename: str, lineno: object, name: str) -> str:
+    """A frame as a traceback writes it, without its source line."""
+    return f'  File "{filename}", line {lineno}, in {name}'
+
+
+def _written_traceback(text: str) -> tuple[str, str] | None:
+    """The type and frames of the first traceback written in ``text``; None
+    when it holds none.
+
+    The first: an exception group is named by its own type and frames, a
+    chain by its root cause, and a traceback quoted in a message or a note is
+    never read. The type is read from the line that names it, the first after
+    the frames that is less indented than them (an exception group's ``|``
+    counts as indent), never from the lines after it: the message, which can
+    run over several lines, and the notes.
+    """
+    start = text.find(_TRACEBACK)
+    if start < 0:
+        return None
+    kind, depth = "unknown", 0
+    frames: list[str] = []
+    for line in text[start + len(_TRACEBACK) :].splitlines():
+        content = line.lstrip(" |")
+        indent = len(line) - len(content)
+        if frame := _FRAME.fullmatch(content):
+            frames.append(_frame(frame[1], frame[2], frame[3]))
+            depth = indent
+        elif frames and content and indent < depth:
+            name = content.split(":", 1)[0]
+            kind = name if _TYPE.fullmatch(name) else kind
+            break
+    return kind, "\n".join(frames)
+
+
+class JsonFormatter(logging.Formatter):
+    """One JSON object per record, on one line."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        entry: dict[str, Any] = {
+            "timestamp": datetime.fromtimestamp(record.created, UTC)
+            .isoformat(timespec="milliseconds")
+            .replace("+00:00", "Z"),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        cid = getattr(record, "correlation_id", None)
+        if cid:
+            entry["correlation_id"] = cid
+        if record.exc_info:
+            entry["exception"] = self.formatException(record.exc_info)
+        if record.stack_info:
+            entry["stack"] = self.formatStack(record.stack_info)
+        extra = {
+            key: value
+            for key, value in vars(record).items()
+            if key not in _RECORD_FIELDS and key not in entry
+        }
+        try:
+            return json.dumps({**entry, **extra}, default=str, allow_nan=False)
+        except (TypeError, ValueError):
+            # A NaN, or a key JSON cannot hold, in an extra field: all as text.
+            text = {key: str(value) for key, value in extra.items()}
+            return json.dumps({**entry, **text})
+
+
+class TextFormatter(logging.Formatter):
+    """A readable line per record, the traceback after it: time, level, logger,
+    correlation ID, message."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        cid = getattr(record, "correlation_id", None) or "-"
+        line = f"{self.formatTime(record)} {record.levelname} {record.name} [{cid}] {record.getMessage()}"
+        if record.exc_info:
+            line += "\n" + self.formatException(record.exc_info)
+        return line
+
+
+def get_log_level() -> str:
+    """The log level from ``LOG_LEVEL``, default ``INFO``."""
+    return os.getenv("LOG_LEVEL", "INFO").upper()
+
+
+def get_log_format() -> str:
+    """The log format from ``LOG_FORMAT``, default ``json``."""
+    return os.getenv("LOG_FORMAT", "json").lower()
+
+
+def get_logging_config(
+    log_level: str | None = None, log_format: str | None = None
+) -> dict[str, Any]:
+    """The ``dictConfig`` of ``configure_logging``; ``ValueError`` for a format
+    other than ``json`` and ``text``.
+
+    The level and the format default to ``LOG_LEVEL`` and ``LOG_FORMAT``;
+    ``LOG_FILE``, when set, gets the same lines as stdout.
+    """
+    level = (log_level or get_log_level()).upper()
+    fmt = (log_format or get_log_format()).lower()
+    if fmt not in LOG_FORMATS:
+        raise ValueError(
+            f"LOG_FORMAT must be one of {', '.join(LOG_FORMATS)}, not {fmt!r}"
+        )
+    handler = {
+        "class": "logging.StreamHandler",
+        "formatter": "default",
+        "filters": ["correlation"],
+        "stream": "ext://sys.stdout",
+    }
+    config: dict[str, Any] = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "filters": {
+            "correlation": {"()": CorrelationFilter},
+            "access_query": {"()": AccessQueryFilter},
+            "server_error": {"()": ServerErrorFilter},
+        },
+        "formatters": {
+            "default": {"()": JsonFormatter if fmt == "json" else TextFormatter}
+        },
+        "handlers": {"console": handler},
+        "loggers": {
+            # uvicorn configures these before it imports the app: they keep
+            # their level and write through the root handler instead.
+            "uvicorn": {"handlers": [], "propagate": True},
+            "uvicorn.error": {"filters": ["server_error"]},
+            "uvicorn.access": {
+                "handlers": [],
+                "filters": ["access_query"],
+                "propagate": True,
+            },
+            # At INFO, httpx and httpx2 write each request's URL, and at DEBUG
+            # httpcore and httpcore2 (their sub-loggers inherit the level) the
+            # raw headers: any of them may hold a credential.
+            "httpx": {"level": "WARNING"},
+            "httpx2": {"level": "WARNING"},
+            "httpcore": {"level": "WARNING"},
+            "httpcore2": {"level": "WARNING"},
+        },
+        "root": {"level": level, "handlers": ["console"]},
+    }
+    log_file = os.getenv("LOG_FILE")
+    if log_file:
+        config["handlers"]["file"] = {
+            "class": "logging.FileHandler",
+            "formatter": "default",
+            "filters": ["correlation"],
+            "filename": log_file,
+            "mode": "a",
+        }
+        config["root"]["handlers"].append("file")
+    return config
+
+
+def configure_logging(
+    log_level: str | None = None, log_format: str | None = None
+) -> None:
+    """Configure logging once, at import, before the server writes its first lines.
+
+    The level and the format default to ``LOG_LEVEL`` (``INFO``) and
+    ``LOG_FORMAT`` (``json``); ``LOG_FILE``, when set, gets the same lines
+    as stdout. Python's warnings become log lines. A second call replaces
+    the first.
+    """
+    config = get_logging_config(log_level, log_format)
+    # dictConfig replaces a logger's handlers but adds to its filters.
+    for name, options in config["loggers"].items():
+        if "filters" in options:
+            logging.getLogger(name).filters.clear()
+    logging.config.dictConfig(config)
+    # warnings.warn writes to stderr otherwise: one more line that is not JSON.
+    logging.captureWarnings(True)
+    log_file = os.getenv("LOG_FILE")
+    logging.getLogger("spot.logging").info(
+        "Logging configured: level %s, format %s, output %s",
+        (log_level or get_log_level()).upper(),
+        (log_format or get_log_format()).lower(),
+        f"console and file {log_file}" if log_file else "console",
+    )
+
+
+def get_logger(name: str) -> logging.Logger:
+    """
+    Get a logger instance for the given name.
+
+    Args:
+        name: Logger name, typically __name__
+
+    Returns:
+        Standard Python logger
+
+    Example:
+        logger = get_logger(__name__)
+        logger.info("Service started")
+        logger.debug("Processing email", extra={"email_id": "123"})
+    """
+    return logging.getLogger(name)

spot_sdk_python-2.0.0b12/spot_sdk/proxy.py

@@ -0,0 +1,94 @@
+"""The egress proxy's credentials, from a secret file.
+
+Core never puts the proxy's user and password in a plugin's environment: it
+forwards each proxy variable without them, and mounts them as the file
+``/run/secrets/spot_proxy_credentials``, ``user:password`` as a URL carries
+them (percent-encoded). :func:`apply_proxy_credentials`, which
+:func:`spot_sdk.create_plugin_app` calls first, puts them back into the
+process's proxy variables, so its HTTP clients reach the proxy
+authenticated. They are never logged.
+"""
+
+from __future__ import annotations
+
+import os
+import unicodedata
+from collections.abc import MutableMapping
+from pathlib import Path
+from urllib.parse import urlsplit, urlunsplit
+
+from .settings import SECRETS_DIR
+
+PROXY_SECRET = "spot_proxy_credentials"
+"""The file, in the secrets directory, that holds the proxy's ``user:password``."""
+
+PROXY_VARIABLES = (
+    "HTTP_PROXY",
+    "HTTPS_PROXY",
+    "ALL_PROXY",
+    "http_proxy",
+    "https_proxy",
+    "all_proxy",
+)
+"""The variables of the proxy URLs that HTTP clients read."""
+
+# What a URL's user and password cannot hold as written: they end it.
+_DELIMITERS = frozenset("/?#[]")
+
+
+def apply_proxy_credentials(
+    secrets_dir: Path | None = None, environ: MutableMapping[str, str] | None = None
+) -> bool:
+    """Put the proxy's credentials into each proxy variable whose URL names no user.
+
+    The credentials are the first line of ``secrets_dir / PROXY_SECRET``
+    (by default in ``/run/secrets``), and they go into the proxy variables
+    of ``environ`` (by default the process's). Core writes the file only
+    when every proxy variable it forwards carried these same credentials,
+    so each gets them. A URL that already names a user keeps it. A value
+    without a scheme (``proxy.example:3128``) is left as it is: a client
+    that reads it reaches the proxy without them. Every variable is left as
+    it is when the file is empty.
+
+    Returns:
+        Whether the file exists.
+
+    Raises:
+        ValueError: the file is not UTF-8, or its first line holds ``/``,
+            ``?``, ``#``, ``[``, ``]`` or a control character, which a URL
+            carries percent-encoded; the error names the file, never what it
+            holds.
+    """
+    path = (SECRETS_DIR if secrets_dir is None else secrets_dir) / PROXY_SECRET
+    if not path.is_file():
+        return False
+    credentials = _read_credentials(path)
+    if credentials is None:
+        # Raised here, where no local holds the file's content.
+        raise ValueError(f"{path} does not hold user:password as a URL carries them")
+    if not credentials:
+        return True
+    variables = os.environ if environ is None else environ
+    for name in PROXY_VARIABLES:
+        parts = urlsplit(variables.get(name, ""))
+        if parts.netloc and "@" not in parts.netloc:
+            netloc = f"{credentials}@{parts.netloc}"
+            variables[name] = urlunsplit(parts._replace(netloc=netloc))
+    return True
+
+
+def _read_credentials(path: Path) -> str | None:
+    """The file's first line, stripped; None when it is not UTF-8, or holds
+    ``/``, ``?``, ``#``, ``[``, ``]`` or a control character, which a URL
+    carries percent-encoded."""
+    try:
+        lines = path.read_bytes().decode("utf-8").splitlines()
+    except UnicodeDecodeError:
+        return None
+    credentials = lines[0].strip() if lines else ""
+    if any(
+        char in _DELIMITERS or unicodedata.category(char) == "Cc"
+        for char in credentials
+    ):
+        return None
+    return credentials

{spot_sdk_python-2.0.0b10 → spot_sdk_python-2.0.0b12}/spot_sdk/testing/fake_knowledge_client.py

@@ -1,7 +1,8 @@
 """In-memory fake of :class:`spot_sdk.knowledge.KnowledgeClient` for tests.
 
-No HTTP, no embedding, no vector math — just a dict keyed by ``id``
-that supports the same methods. ``fetch()`` does naive substring
+No HTTP, no embedding, no vector math — just a dict keyed by
+``(source, id)``, as a document's id is unique within its source, that
+supports the same methods. ``fetch()`` does naive substring
 matching over content + tag-expression filtering, which is enough for
 plugin unit tests to assert that an analyzer reads documents out of the
 store.
@@ -23,7 +24,8 @@ from __future__ import annotations
 import re
 from typing import TYPE_CHECKING
 
-from ..knowledge import KnowledgeDocument
+from ..clients import SpotAPIError
+from ..knowledge import KnowledgeDocument, _document_path
 from ..workflow import RetrievalLimits
 
 if TYPE_CHECKING:
@@ -36,7 +38,7 @@ class FakeKnowledgeClient:
     def __init__(
         self, retrieval_limits: RetrievalLimits | None = None, *, source: str = ""
     ) -> None:
-        self._docs: dict[str, KnowledgeDocument] = {}
+        self._docs: dict[tuple[str, str], KnowledgeDocument] = {}
         self.retrieval_limits = retrieval_limits or RetrievalLimits()
         self.source = source
 
@@ -49,13 +51,34 @@ class FakeKnowledgeClient:
     # --- write side ---------------------------------------------------- #
 
     async def upsert(self, doc: KnowledgeDocument) -> None:
-        self.add(doc)
+        await self.bulk_upsert([doc])
 
     async def bulk_upsert(self, docs: list[KnowledgeDocument]) -> None:
+        """Store the documents, as the store does for a plugin's write.
+
+        With a source, a batch holding another source's document is refused
+        whole, before anything is stored, and a document without one takes
+        the client's. Without a source (a core-like caller), the documents
+        are stored as given.
+        """
+        if self.source:
+            self._refuse_other_sources(docs)
+            docs = [
+                d if d.source else d.model_copy(update={"source": self.source})
+                for d in docs
+            ]
         self.add(*docs)
 
     async def delete(self, doc_id: str) -> None:
-        self._docs.pop(doc_id, None)
+        """Delete the client's own document of this id; it needs a source, as the client does.
+
+        A document the source does not hold raises ``SpotAPIError`` 404, as
+        the store answers.
+        """
+        if not self.source:
+            raise ValueError("delete needs a source: FakeKnowledgeClient(source=...)")
+        if self._docs.pop((self.source, doc_id), None) is None:
+            raise SpotAPIError("DELETE", _document_path(self.source, doc_id), 404)
 
     async def replace_source(self, docs: list[KnowledgeDocument]) -> None:
         """Keep ``docs`` as the source's documents, as the store does: an empty
@@ -64,13 +87,16 @@ class FakeKnowledgeClient:
             raise ValueError(
                 "replace_source needs a source: FakeKnowledgeClient(source=...)"
             )
+        self._refuse_other_sources(docs)
+        for key in [key for key in self._docs if key[0] == self.source]:
+            del self._docs[key]
+        self.add(*(d.model_copy(update={"source": self.source}) for d in docs))
+
+    def _refuse_other_sources(self, docs: list[KnowledgeDocument]) -> None:
+        """Refuse documents naming another source than the client's: the store answers 403."""
         foreign = sorted({d.source for d in docs} - {"", self.source})
         if foreign:
             raise ValueError(f"documents of another source: {foreign}")
-        for doc_id in [i for i, d in self._docs.items() if d.source == self.source]:
-            del self._docs[doc_id]
-        for d in docs:
-            self._docs[d.id] = d.model_copy(update={"source": self.source})
 
     # --- read side ----------------------------------------------------- #
 
@@ -107,9 +133,9 @@ class FakeKnowledgeClient:
     # --- test helpers -------------------------------------------------- #
 
     def add(self, *docs: KnowledgeDocument) -> None:
-        """Store documents as they are, without awaiting (test helper)."""
+        """Store documents as they are, by source and id, without awaiting (test helper)."""
         for d in docs:
-            self._docs[d.id] = d
+            self._docs[d.source, d.id] = d
 
     def all(self) -> list[KnowledgeDocument]:
         """Return all stored documents (test helper)."""

spot_sdk_python-2.0.0b10/spot_sdk/logging.py

@@ -1,133 +0,0 @@
-"""
-Standard logging configuration for SPOT platform.
-
-Uses Python's built-in logging with dictConfig.
-Simple, configurable via environment variables.
-"""
-
-from __future__ import annotations
-
-import logging
-import logging.config
-import os
-from typing import Any, Optional
-
-
-def get_log_level() -> str:
-    """Get log level from environment, default to INFO."""
-    return os.getenv("LOG_LEVEL", "INFO").upper()
-
-
-def get_log_format() -> str:
-    """Get log format from environment, default to simple."""
-    return os.getenv("LOG_FORMAT", "simple").lower()
-
-
-def get_logging_config(
-    log_level: Optional[str] = None, log_format: Optional[str] = None
-) -> dict[str, Any]:
-    """
-    Get logging configuration dictionary for dictConfig.
-
-    Args:
-        log_level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
-        log_format: Format style (simple, detailed, json)
-
-    Returns:
-        Dictionary suitable for logging.config.dictConfig
-    """
-    level = log_level or get_log_level()
-    fmt = log_format or get_log_format()
-
-    # Choose format string
-    if fmt == "json":
-        format_str = '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "logger": "%(name)s", "message": "%(message)s"}'
-    elif fmt == "detailed":
-        format_str = "%(asctime)s - %(name)-20s - %(levelname)-8s - [%(filename)s:%(lineno)d] - %(message)s"
-    else:  # simple
-        format_str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-
-    config = {
-        "version": 1,
-        "disable_existing_loggers": False,
-        "formatters": {
-            "default": {
-                "format": format_str,
-                "datefmt": "%Y-%m-%d %H:%M:%S" if fmt == "detailed" else "%H:%M:%S",
-            }
-        },
-        "handlers": {
-            "console": {
-                "class": "logging.StreamHandler",
-                "formatter": "default",
-                "stream": "ext://sys.stdout",
-            }
-        },
-        "root": {"level": level, "handlers": ["console"]},
-    }
-
-    # Add file handler if LOG_FILE is specified
-    log_file = os.getenv("LOG_FILE")
-    if log_file:
-        config["handlers"]["file"] = {  # type: ignore[index]
-            "class": "logging.FileHandler",
-            "formatter": "default",
-            "filename": log_file,
-            "mode": "a",
-        }
-        config["root"]["handlers"].append("file")  # type: ignore[index]
-
-    return config
-
-
-def configure_logging(
-    log_level: Optional[str] = None, log_format: Optional[str] = None
-) -> None:
-    """
-    Configure logging for the application.
-
-    Call this once during application startup.
-
-    Args:
-        log_level: Log level (default: from LOG_LEVEL env var or INFO)
-        log_format: Format style (default: from LOG_FORMAT env var or simple)
-
-    Environment Variables:
-        LOG_LEVEL: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO)
-        LOG_FORMAT: json, simple, detailed (default: simple)
-        LOG_FILE: Path to log file (optional, logs to console if not set)
-    """
-    config = get_logging_config(log_level, log_format)
-    logging.config.dictConfig(config)
-
-    # Log configuration
-    logger = logging.getLogger("spot.logging")
-    level = log_level or get_log_level()
-    fmt = log_format or get_log_format()
-    log_file = os.getenv("LOG_FILE")
-
-    log_targets = "console"
-    if log_file:
-        log_targets += f" and file ({log_file})"
-
-    logger.info(
-        f"Logging configured - Level: {level}, Format: {fmt}, Output: {log_targets}"
-    )
-
-
-def get_logger(name: str) -> logging.Logger:
-    """
-    Get a logger instance for the given name.
-
-    Args:
-        name: Logger name, typically __name__
-
-    Returns:
-        Standard Python logger
-
-    Example:
-        logger = get_logger(__name__)
-        logger.info("Service started")
-        logger.debug("Processing email", extra={"email_id": "123"})
-    """
-    return logging.getLogger(name)