spot-sdk-python 1.2.1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

spot_sdk/__init__.py

@@ -1,99 +1,152 @@
-"""SPOT API interfaces and data models."""
+"""SPOT SDK: the plugin contract v2, the plugin runtime and its clients."""
 
-from .analysis_context import AnalysisContextReader
-from .analyzer import AnalyzerCapability, AnalyzerInterface, ThreatLevel
-from .config import ConfigOption, ConfigReloadResult, ConfigStatus, ConfigUpdateRequest
-from .config_client import ConfigClient
-from .config_helpers import merge_settings
-from .email import Attachment, Email, EmailHeader
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+from .config import ConfigOption, ConfigStatus, ConfigUpdateRequest
+from .email import Address, Anomaly, AuthResult, EmailView, Envelope, Header, Part, Url
 from .errors import ErrorResponse
-from .knowledge import KnowledgeClient, KnowledgeDocument, chunk_text, content_hash
 from .knowledge_tags import KnowledgeTag
 from .logging import configure_logging, get_logger, get_logging_config
-from .ollama import LLMResponse, OllamaClientProtocol
-from .orchestrator import AnalyzerResult, OrchestrationResult, WorkflowStageResult
-from .plugin import PluginKind
-from .results import (
-    AnalysisIndicator,
-    AnalysisMetadata,
-    AnalysisResult,
-    IndicatorType,
+from .manifest import (
+    CONTRACT_VERSION,
+    Egress,
+    Manifest,
+    PluginKind,
+    Resources,
+    SettingField,
+    SettingType,
 )
-from .retriever import (
-    IngestionRequest,
-    IngestionResult,
-    IngestionStatus,
-    MailRetrieverClient,
+from .orchestrator import (
+    AnalyzeRequest,
+    AnalyzerOutcome,
+    AttributedSignal,
+    OrchestrationResult,
+    RespondRequest,
+    StageOutcome,
 )
-from .settings_schema import register_settings_schema
-from .threat_levels import confidence_to_threat_level
+from .retriever import IngestionRequest, IngestionResult, IngestionStatus
+from .settings import PluginSettings
+from .verdict import AnalyzerVerdict, Score, Signal, VerdictClass
 from .workflow import (
+    AggregationMethod,
     AnalyzerConfig,
+    DecisiveSignal,
     FailureStrategy,
+    ResponseMode,
     RetrievalLimits,
     RetryConfig,
+    VerdictPolicy,
     Workflow,
     WorkflowStage,
 )
 
+if TYPE_CHECKING:
+    from .app import SyncState, create_plugin_app
+    from .clients import SpotAPIError, SpotClient
+    from .knowledge import KnowledgeClient, KnowledgeDocument, chunk_text, content_hash
+    from .llm import LLMClient, LLMOutputError, LLMToolCall, LLMToolCalls, LLMToolRound
+
+# The modules that import FastAPI, Starlette or httpx load on first use, so importing
+# `spot_sdk.mime` or the models stays light (PEP 562).
+_LAZY = {
+    **dict.fromkeys(("SyncState", "create_plugin_app"), ".app"),
+    **dict.fromkeys(("SpotAPIError", "SpotClient"), ".clients"),
+    **dict.fromkeys(
+        ("KnowledgeClient", "KnowledgeDocument", "chunk_text", "content_hash"),
+        ".knowledge",
+    ),
+    **dict.fromkeys(
+        ("LLMClient", "LLMOutputError", "LLMToolCall", "LLMToolCalls", "LLMToolRound"),
+        ".llm",
+    ),
+}
+
+
+def __getattr__(name: str) -> Any:
+    module = _LAZY.get(name)
+    if module is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    value = getattr(import_module(module, __name__), name)
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> list[str]:
+    return sorted({*globals(), *_LAZY})
+
+
 __all__ = [
-    # Analyzer interface
-    "AnalyzerInterface",
-    "AnalyzerCapability",
-    "ThreatLevel",
+    # Email view
+    "Address",
+    "Anomaly",
+    "AuthResult",
+    "EmailView",
+    "Envelope",
+    "Header",
+    "Part",
+    "Url",
+    # Verdicts
+    "AnalyzerVerdict",
+    "Score",
+    "Signal",
+    "VerdictClass",
+    # Workflow
+    "AggregationMethod",
+    "AnalyzerConfig",
+    "DecisiveSignal",
+    "FailureStrategy",
+    "ResponseMode",
+    "RetrievalLimits",
+    "RetryConfig",
+    "VerdictPolicy",
+    "Workflow",
+    "WorkflowStage",
+    # Analyzer and responder calls, results
+    "AnalyzeRequest",
+    "AnalyzerOutcome",
+    "AttributedSignal",
+    "OrchestrationResult",
+    "RespondRequest",
+    "StageOutcome",
+    # Ingestion
+    "IngestionRequest",
+    "IngestionResult",
+    "IngestionStatus",
+    # Plugin manifest
+    "CONTRACT_VERSION",
+    "Egress",
+    "Manifest",
+    "PluginKind",
+    "Resources",
+    "SettingField",
+    "SettingType",
+    # Plugin runtime
+    "PluginSettings",
+    "SyncState",
+    "create_plugin_app",
+    # Clients
+    "LLMClient",
+    "LLMOutputError",
+    "LLMToolCall",
+    "LLMToolCalls",
+    "LLMToolRound",
+    "SpotAPIError",
+    "SpotClient",
+    # Knowledge Store
+    "KnowledgeClient",
+    "KnowledgeDocument",
+    "KnowledgeTag",
+    "chunk_text",
+    "content_hash",
     # Config models
     "ConfigOption",
     "ConfigStatus",
     "ConfigUpdateRequest",
-    "ConfigReloadResult",
-    "ConfigClient",
     # Error response
     "ErrorResponse",
-    # Email models
-    "Email",
-    "EmailHeader",
-    "Attachment",
-    "AnalysisContextReader",
-    # Analysis results
-    "AnalysisResult",
-    "AnalysisIndicator",
-    "AnalysisMetadata",
-    "IndicatorType",
     # Logging
     "configure_logging",
     "get_logger",
     "get_logging_config",
-    # Workflow
-    "Workflow",
-    "WorkflowStage",
-    "AnalyzerConfig",
-    "RetryConfig",
-    "RetrievalLimits",
-    "FailureStrategy",
-    # Knowledge Store
-    "KnowledgeDocument",
-    "KnowledgeClient",
-    "KnowledgeTag",
-    "chunk_text",
-    "content_hash",
-    # Plugin vocabulary
-    "PluginKind",
-    # Mail-retriever ingest contract
-    "IngestionRequest",
-    "IngestionResult",
-    "IngestionStatus",
-    "MailRetrieverClient",
-    # Orchestration
-    "OrchestrationResult",
-    "AnalyzerResult",
-    "WorkflowStageResult",
-    # Ollama protocol
-    "OllamaClientProtocol",
-    "LLMResponse",
-    # Settings schema
-    "register_settings_schema",
-    # Config helpers
-    "merge_settings",
-    # Threat levels
-    "confidence_to_threat_level",
 ]

spot_sdk/_json.py

@@ -0,0 +1,46 @@
+"""Strings measured and cut by the size of their JSON.
+
+The view and the LLM client's rendering are sent as JSON, where escapes make
+a string larger than its text: a control character takes six bytes
+(``\\u0001``). ``json.dumps`` with ``ensure_ascii=False`` and pydantic's
+``model_dump_json`` escape strings alike, so sizes here are theirs.
+"""
+
+from __future__ import annotations
+
+import json
+
+_MAX_CHARACTER_BYTES = 6
+"""The most bytes one character takes: a control character's escape."""
+
+
+def json_size(text: str) -> int:
+    """The UTF-8 bytes of ``text``'s JSON string between its quotes, escapes
+    included; a lone surrogate (possible in a string built in Python) counts
+    as three."""
+    escaped = json.dumps(text, ensure_ascii=False)[1:-1]
+    return len(escaped.encode("utf-8", "surrogatepass"))
+
+
+def cut_json(text: str, max_bytes: int) -> tuple[str, bool]:
+    """The longest start of ``text`` whose JSON string takes at most
+    ``max_bytes`` bytes between its quotes (``json_size``), and whether that
+    cut anything.
+
+    A character is kept or dropped whole, so neither an escape nor a
+    character's UTF-8 bytes are split. The text is measured in chunks that
+    surely fit, a sixth of the room left each, then character by character:
+    the work follows ``max_bytes``, whatever the text's length.
+    """
+    if len(text) * _MAX_CHARACTER_BYTES <= max_bytes:
+        return text, False  # it surely fits
+    kept = 0
+    room = max_bytes
+    while kept < len(text):
+        chunk = text[kept : kept + max(room // _MAX_CHARACTER_BYTES, 1)]
+        used = json_size(chunk)
+        if used > room:  # one character, which does not fit
+            break
+        kept += len(chunk)
+        room -= used
+    return text[:kept], kept < len(text)

spot_sdk/app.py

@@ -0,0 +1,332 @@
+"""The plugin runtime: one FastAPI app with the routes core calls.
+
+``create_plugin_app`` builds it from the plugin's manifest, its settings
+class and its handlers:
+
+- ``GET /health`` and ``GET /manifest`` are open;
+- ``POST /internal/analyze``, ``POST /internal/respond``, and
+  ``POST /internal/sync`` with ``GET /internal/state``, exist for the
+  handlers given. Each checks ``Authorization: Bearer <the plugin's own
+  token>`` before it reads the request, and answers 401 otherwise; an empty
+  configured token refuses every call.
+
+An ``async def`` handler, or an object whose ``__call__`` is ``async def``,
+runs on the event loop; any other callable runs in a worker thread, so it may
+block. Either way the clients send the call's ``X-Correlation-ID`` on to SPOT.
+
+Neither the logs nor the error answers hold the request's content: a
+handler's exception is logged by its type and place only, and an invalid
+request is answered with the fields refused, never the values sent.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hmac
+import inspect
+import traceback
+from collections.abc import Awaitable, Callable, Iterator
+from contextlib import contextmanager
+from datetime import UTC, datetime
+from typing import Any, TypeAlias, TypeVar
+
+from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field, SecretStr, ValidationError
+from pydantic_core import PydanticSerializationError
+from starlette.concurrency import run_in_threadpool
+from starlette.types import Lifespan
+
+from .clients import correlation_id
+from .logging import get_logger
+from .manifest import Manifest, PluginKind
+from .orchestrator import AnalyzeRequest, RespondRequest
+from .settings import PluginSettings
+from .verdict import AnalyzerVerdict
+
+logger = get_logger(__name__)
+
+SettingsT = TypeVar("SettingsT", bound=PluginSettings)
+ModelT = TypeVar("ModelT", bound=BaseModel)
+
+AnalyzeHandler: TypeAlias = Callable[
+    [AnalyzeRequest, SettingsT], AnalyzerVerdict | Awaitable[AnalyzerVerdict]
+]
+"""``analyze(request, settings) -> AnalyzerVerdict``; an exception is answered 500."""
+RespondHandler: TypeAlias = Callable[
+    [RespondRequest, SettingsT], None | Awaitable[None]
+]
+"""``respond(request, settings) -> None``; it acts only when ``request.mode`` is enforce."""
+SyncHandler: TypeAlias = Callable[[SettingsT], int | Awaitable[int]]
+"""``sync(settings) -> int``: the number of documents written."""
+
+
+class SyncState(BaseModel):
+    """A context provider's syncs, as ``/internal/sync`` and ``/internal/state`` answer."""
+
+    in_progress: bool = False
+    last_sync_at: datetime | None = Field(
+        default=None, description="When the last sync started"
+    )
+    last_finished_at: datetime | None = Field(
+        default=None, description="When the last sync ended"
+    )
+    last_sync_count: int = Field(
+        default=0, description="Documents written by the last sync that succeeded"
+    )
+    last_error: str | None = Field(
+        default=None,
+        description="The exception's type when the last sync failed; None when it succeeded",
+    )
+
+
+def create_plugin_app(
+    manifest: Manifest,
+    settings_cls: type[SettingsT],
+    *,
+    analyze: AnalyzeHandler[SettingsT] | None = None,
+    respond: RespondHandler[SettingsT] | None = None,
+    sync: SyncHandler[SettingsT] | None = None,
+    lifespan: Lifespan[FastAPI] | None = None,
+) -> FastAPI:
+    """Build the plugin's app; the settings are read once, here.
+
+    Each kind the manifest declares needs its handler (analyzer: ``analyze``,
+    responder: ``respond``, context provider: ``sync``; a mail retriever has
+    none), and each handler its kind, or ``ValueError`` is raised.
+
+    ``lifespan`` is passed to FastAPI as is: an async context manager run
+    around the app's life, to load a model at startup, say.
+
+    ``POST /internal/sync`` answers 202 at once and runs ``sync`` in the
+    background. While a sync runs, another call starts nothing and answers
+    202 with the running sync's state.
+    """
+    _check_kinds(manifest, {"analyze": analyze, "respond": respond, "sync": sync})
+    settings = settings_cls()
+    if not settings.spot_plugin_token.get_secret_value():
+        logger.warning(
+            "The plugin has no token: every /internal call will be refused. Core "
+            "mounts it as /run/secrets/spot_plugin_token; outside a container, set "
+            "SPOT_PLUGIN_TOKEN."
+        )
+    app = FastAPI(
+        title=manifest.id,
+        version=manifest.version,
+        openapi_url=None,
+        lifespan=lifespan,
+    )
+
+    @app.get("/health")
+    async def health() -> dict[str, str]:
+        return {"status": "healthy", "version": manifest.version}
+
+    @app.get("/manifest")
+    async def get_manifest() -> Manifest:
+        return manifest
+
+    internal = APIRouter(
+        prefix="/internal",
+        dependencies=[Depends(_token_check(settings.spot_plugin_token))],
+    )
+
+    if analyze is not None:
+
+        @internal.post("/analyze")
+        async def analyze_route(request: Request) -> Response:
+            call = await _parse(request, AnalyzeRequest)
+            with _correlated(request):
+                try:
+                    answer = await _run(analyze, call, settings)
+                except Exception as error:
+                    _log_failure(f"analysis of job {call.job_id}", error)
+                    return JSONResponse({"detail": "analysis failed"}, status_code=500)
+            try:
+                verdict = AnalyzerVerdict.model_validate(answer)
+            except ValidationError as error:
+                _log_invalid_answer(f"analysis of job {call.job_id}", error)
+                return JSONResponse({"detail": "analysis failed"}, status_code=500)
+            try:
+                body = verdict.model_dump_json()
+            except PydanticSerializationError as error:
+                _log_failure(f"analysis of job {call.job_id}", error)
+                return JSONResponse({"detail": "analysis failed"}, status_code=500)
+            return Response(body, media_type="application/json")
+
+    if respond is not None:
+
+        @internal.post("/respond")
+        async def respond_route(request: Request) -> Response:
+            call = await _parse(request, RespondRequest)
+            with _correlated(request):
+                try:
+                    await _run(respond, call, settings)
+                except Exception as error:
+                    _log_failure(f"response to job {call.job_id}", error)
+                    return JSONResponse({"detail": "response failed"}, status_code=500)
+            return Response(status_code=204)
+
+    if sync is not None:
+        syncs = _SyncRunner(sync, settings)
+
+        @internal.post("/sync", status_code=202)
+        async def sync_route(request: Request) -> SyncState:
+            with _correlated(request):
+                return syncs.start()
+
+        @internal.get("/state")
+        async def state_route() -> SyncState:
+            return syncs.state.model_copy()
+
+    app.include_router(internal)
+    return app
+
+
+_HANDLER_KINDS = {
+    "analyze": PluginKind.ANALYZER,
+    "respond": PluginKind.RESPONDER,
+    "sync": PluginKind.CONTEXT_PROVIDER,
+}
+"""The kind each handler serves; a mail retriever has no handler."""
+
+
+def _check_kinds(manifest: Manifest, handlers: dict[str, object]) -> None:
+    """Each kind the manifest declares has its handler, and each handler its kind."""
+    for name, kind in _HANDLER_KINDS.items():
+        declared = kind in manifest.kinds
+        given = handlers[name] is not None
+        if declared and not given:
+            raise ValueError(
+                f"the manifest declares {kind}, but no {name} handler is given"
+            )
+        if given and not declared:
+            raise ValueError(
+                f"the {name} handler is given, but the manifest does not declare {kind}"
+            )
+
+
+class _SyncRunner:
+    """Runs the sync handler in the background, one run at a time."""
+
+    def __init__(self, handler: SyncHandler[Any], settings: PluginSettings) -> None:
+        self._handler = handler
+        self._settings = settings
+        self._task: asyncio.Task[None] | None = None
+        self.state = SyncState()
+
+    def start(self) -> SyncState:
+        """Start a sync unless one runs; the state either way."""
+        if not self.state.in_progress:
+            self.state.in_progress = True
+            self.state.last_sync_at = datetime.now(UTC)
+            self.state.last_error = None
+            self._task = asyncio.create_task(self._run())
+        return self.state.model_copy()
+
+    async def _run(self) -> None:
+        try:
+            count = await _run(self._handler, self._settings)
+            if not isinstance(count, int):
+                raise TypeError(
+                    f"the sync handler returned {type(count).__name__}, not int"
+                )
+            self.state.last_sync_count = count
+        except Exception as error:
+            self.state.last_error = type(error).__name__
+            _log_failure("sync", error)
+        finally:
+            self.state.last_finished_at = datetime.now(UTC)
+            self.state.in_progress = False
+
+
+def _token_check(token: SecretStr) -> Callable[[Request], Awaitable[None]]:
+    expected = token.get_secret_value().encode()
+
+    async def check_token(request: Request) -> None:
+        if not _is_bearer(request.headers.get("authorization"), expected):
+            raise HTTPException(
+                status_code=401,
+                detail="unauthorized",
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+
+    return check_token
+
+
+def _is_bearer(header: str | None, token: bytes) -> bool:
+    """Whether ``header`` is ``Bearer <token>``; an empty token matches nothing."""
+    if not token or header is None:
+        return False
+    scheme, _, presented = header.partition(" ")
+    return scheme.lower() == "bearer" and hmac.compare_digest(presented.encode(), token)
+
+
+async def _parse(request: Request, model: type[ModelT]) -> ModelT:
+    """The request body as ``model``; a 422 names the fields refused, never the values sent.
+
+    The 422 is raised after the handler, so the validation error, which holds
+    the request's content, is not attached as its context.
+    """
+    try:
+        return model.model_validate_json(await request.body())
+    except ValidationError as error:
+        detail = error.errors(
+            include_url=False, include_context=False, include_input=False
+        )
+    raise HTTPException(status_code=422, detail=detail)
+
+
+async def _run(handler: Callable[..., Any], *args: Any) -> Any:
+    """Await an async handler; run any other in a worker thread, in this context.
+
+    A handler is async when it is an ``async def`` function, or an object whose
+    ``__call__`` is one, as FastAPI tells them apart.
+    """
+    if inspect.iscoroutinefunction(handler) or inspect.iscoroutinefunction(
+        getattr(handler, "__call__", None)
+    ):
+        return await handler(*args)
+    return await run_in_threadpool(handler, *args)
+
+
+@contextmanager
+def _correlated(request: Request) -> Iterator[None]:
+    """Make the call's ``X-Correlation-ID`` the one the clients send."""
+    reset = correlation_id.set(request.headers.get("x-correlation-id"))
+    try:
+        yield
+    finally:
+        correlation_id.reset(reset)
+
+
+def _log_failure(what: str, error: Exception) -> None:
+    """Log a handler's failure by its type and place only: its message, or the
+    locations of a validation error, may quote the email, or a directory URL
+    with its credentials."""
+    frame = traceback.extract_tb(error.__traceback__)[-1]
+    logger.error(
+        "%s failed: %s at %s:%s in %s",
+        what,
+        type(error).__name__,
+        frame.filename,
+        frame.lineno,
+        frame.name,
+    )
+
+
+def _log_invalid_answer(what: str, error: ValidationError) -> None:
+    """Log a handler's answer that does not validate by the fields refused and
+    why (``loc`` and ``type``), never their values. The locations are the
+    answer model's own fields, so they hold none of the email."""
+    fields = ", ".join(
+        f"{'.'.join(str(part) for part in detail['loc'])} ({detail['type']})"
+        for detail in error.errors(
+            include_url=False, include_context=False, include_input=False
+        )
+    )
+    logger.error(
+        "%s failed: the handler's answer is not a valid %s; invalid: %s",
+        what,
+        error.title,
+        fields,
+    )