spot-sdk-python 1.0.0__py3-none-any.whl

spot_sdk/knowledge.py

@@ -0,0 +1,341 @@
+"""Knowledge Store SDK: document model, client, chunk helper.
+
+The Knowledge Store is a SPOT platform service where context providers
+deposit tagged company documents (employees, wikis, policies, ...) and
+analyzers fetch them on demand by semantic similarity + tag expression.
+
+Neither side knows the other exists. Plugins only see this SDK; the
+store owns embedding and vector search.
+
+Quick start (ingestion provider)::
+
+    from spot_sdk.knowledge import KnowledgeClient, KnowledgeDocument
+    from spot_sdk.knowledge_tags import KnowledgeTag
+
+    kb = KnowledgeClient(
+        url=os.environ["SPOT_KNOWLEDGE_URL"],
+        api_key=os.environ.get("SPOT_INTERNAL_API_KEY"),
+    )
+    await kb.bulk_upsert([
+        KnowledgeDocument(
+            id=f"employee:{e.email}",
+            content=f"{e.name}, {e.title}, {e.department}",
+            tags=[KnowledgeTag.EMPLOYEE, KnowledgeTag.EXECUTIVE],
+            metadata={"email": e.email, "title": e.title},
+            source="provider-employee-dir",
+        )
+        for e in employees
+    ])
+
+Quick start (analyzer)::
+
+    from spot_sdk.knowledge import KnowledgeClient
+
+    @app.post("/internal/analyze")
+    async def analyze(email: Email) -> AnalysisResult:
+        kb = KnowledgeClient.for_analysis(email)
+        execs = await kb.fetch(tags="employee+executive", top_k=3)
+        ...
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, cast
+
+import httpx
+from pydantic import BaseModel, Field
+
+from .logging import get_logger
+
+if TYPE_CHECKING:
+    from .email import Email
+
+logger = get_logger(__name__)
+
+
+# ----------------------------------------------------------------------- #
+# Models
+# ----------------------------------------------------------------------- #
+
+
+class KnowledgeDocument(BaseModel):
+    """A document stored in (or returned from) the SPOT Knowledge Store.
+
+    There is no separate ``type`` field — ``tags`` is the single
+    categorisation axis. Use the constants on
+    :class:`spot_sdk.knowledge_tags.KnowledgeTag` for interoperable
+    common labels.
+    """
+
+    id: str = Field(
+        ...,
+        description=(
+            "Stable unique id (e.g. 'employee:alice@co.com', "
+            "'wiki:page-123#chunk-4'). Upsert is idempotent on this key."
+        ),
+    )
+    content: str = Field(
+        ...,
+        description=(
+            "Text payload an LLM or analyzer reads. Should be chunk-sized "
+            "for long documents; use spot_sdk.knowledge.chunk_text() to split."
+        ),
+    )
+    tags: list[str] = Field(
+        default_factory=list,
+        description="Tag set; AND/OR-filterable at query time via tag expressions.",
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description=(
+            "Structured fields (role, email, url, ...). "
+            "Reserved key 'parent_id' links chunks back to their source doc."
+        ),
+    )
+    source: str = Field(
+        default="",
+        description="Provider that produced the document (audit + future ownership).",
+    )
+    updated_at: datetime | None = None
+    expires_at: datetime | None = Field(
+        default=None,
+        description="Optional TTL; the cleanup job deletes rows where expires_at < now().",
+    )
+    score: float | None = Field(
+        default=None,
+        ge=0.0,
+        le=1.0,
+        description="Populated on query results, 0.0 – 1.0 (cosine similarity).",
+    )
+
+
+# ----------------------------------------------------------------------- #
+# Chunking helper
+# ----------------------------------------------------------------------- #
+
+
+def chunk_text(
+    text: str,
+    max_chars: int = 2000,
+    overlap: int = 200,
+) -> list[str]:
+    """Split a long text into overlapping character-bounded chunks.
+
+    Naive but predictable: walk the string in steps of (max_chars - overlap),
+    cut at the next whitespace within ``max_chars`` to avoid mid-word splits.
+
+    Args:
+        text: The text to split.
+        max_chars: Maximum chunk size in characters.
+        overlap: Characters of overlap between consecutive chunks.
+
+    Returns:
+        List of chunk strings. Empty input returns ``[]``.
+    """
+    if not text:
+        return []
+    if max_chars <= 0:
+        raise ValueError("max_chars must be positive")
+    if overlap < 0 or overlap >= max_chars:
+        raise ValueError("overlap must be in [0, max_chars)")
+
+    text = text.strip()
+    if len(text) <= max_chars:
+        return [text]
+
+    chunks: list[str] = []
+    step = max_chars - overlap
+    start = 0
+    n = len(text)
+
+    while start < n:
+        end = min(start + max_chars, n)
+        # Try to cut on whitespace if we're not already at the end
+        if end < n:
+            cut = text.rfind(" ", start, end)
+            if cut > start + (max_chars // 2):
+                end = cut
+        chunks.append(text[start:end].strip())
+        if end >= n:
+            break
+        start = max(start + step, end - overlap)
+
+    return [c for c in chunks if c]
+
+
+# ----------------------------------------------------------------------- #
+# Client
+# ----------------------------------------------------------------------- #
+
+
+class KnowledgeClient:
+    """HTTP client for the SPOT Knowledge Store.
+
+    Same class for both write side (providers: ``upsert``,
+    ``bulk_upsert``, ``delete``) and read side (analyzers: ``fetch``).
+    All HTTP / embedding / vector / DB mechanics happen on the server;
+    callers only see typed Python objects.
+
+    Use :meth:`for_analysis` from analyzer code — it builds a client
+    pre-configured with the URL from env and the workflow-imposed
+    retrieval limits passed in by the orchestrator.
+    """
+
+    DEFAULT_TIMEOUT = 15.0
+
+    def __init__(
+        self,
+        url: str,
+        api_key: str | None = None,
+        timeout: float = DEFAULT_TIMEOUT,
+        retrieval_limits: dict[str, Any] | None = None,
+    ) -> None:
+        """Initialize the client.
+
+        Args:
+            url: Base URL of the Knowledge Store API
+                (e.g. ``http://knowledge:8000/api/v1/knowledge``).
+            api_key: Optional ``SPOT_INTERNAL_API_KEY`` for write endpoints.
+            timeout: Request timeout in seconds.
+            retrieval_limits: Optional caps applied to ``fetch()`` —
+                ``{"max_top_k": int, "min_score_floor": float}``. Only
+                shrinks what the caller asked for; never expands.
+        """
+        self.url = url.rstrip("/")
+        self.api_key = api_key
+        self.timeout = timeout
+        self._limits = dict(retrieval_limits or {})
+
+    # ------------------------------------------------------------------ #
+    # Factories
+    # ------------------------------------------------------------------ #
+
+    @classmethod
+    def for_analysis(cls, email: "Email") -> KnowledgeClient:
+        """Build a client pre-configured for the current analysis call.
+
+        Reads the URL from the ``SPOT_KNOWLEDGE_URL`` env var (injected
+        by the SPOT installer at plugin install time) and any retrieval
+        caps from ``email.retrieval_limits`` (set by the orchestrator
+        from the workflow stage's ``retrieval_limits``).
+        """
+        url = os.environ.get("SPOT_KNOWLEDGE_URL")
+        if not url:
+            raise RuntimeError(
+                "SPOT_KNOWLEDGE_URL is not set. The SPOT installer normally "
+                "injects this when the plugin is installed; in tests, set it "
+                "manually or use FakeKnowledgeClient."
+            )
+        return cls(
+            url=url,
+            api_key=os.environ.get("SPOT_INTERNAL_API_KEY"),
+            retrieval_limits=getattr(email, "retrieval_limits", None) or {},
+        )
+
+    # ------------------------------------------------------------------ #
+    # Write side: providers
+    # ------------------------------------------------------------------ #
+
+    async def upsert(self, doc: KnowledgeDocument) -> None:
+        """Insert or replace a single document."""
+        await self._post("/upsert", json=doc.model_dump(mode="json"), auth=True)
+
+    async def bulk_upsert(self, docs: list[KnowledgeDocument]) -> None:
+        """Insert or replace many documents in one round-trip."""
+        if not docs:
+            return
+        payload = {"documents": [d.model_dump(mode="json") for d in docs]}
+        await self._post("/bulk-upsert", json=payload, auth=True)
+
+    async def delete(self, doc_id: str) -> None:
+        """Delete a document by id."""
+        await self._delete(f"/documents/{doc_id}", auth=True)
+
+    # ------------------------------------------------------------------ #
+    # Read side: analyzers
+    # ------------------------------------------------------------------ #
+
+    async def fetch(
+        self,
+        tags: str | None = None,
+        text: str | None = None,
+        top_k: int = 5,
+        min_score: float = 0.0,
+        strategy: str = "similarity",
+    ) -> list[KnowledgeDocument]:
+        """Semantic vector search with tag-expression filter.
+
+        The store embeds ``text`` once (cached by content hash), runs the
+        tag filter, returns top-K by cosine similarity (or re-ranked for
+        diversity if ``strategy='diverse'``).
+
+        Args:
+            tags: Tag expression — ``"a"``, ``"a+b"`` (AND),
+                ``"a|b"`` (OR), ``"a+b|c"`` (precedence: AND > OR).
+                ``None`` / ``""`` means no tag filter.
+            text: Free-text query. Required.
+            top_k: Number of documents to return.
+            min_score: Minimum cosine similarity (0.0 – 1.0).
+            strategy: ``"similarity"`` (default) or ``"diverse"``.
+
+        Returns:
+            Documents sorted by ``score`` descending.
+        """
+        if text is None or not text.strip():
+            raise ValueError("fetch() requires a non-empty 'text' argument")
+
+        # Apply operator-policy caps (caps only — never raise the request).
+        max_top_k = self._limits.get("max_top_k")
+        if max_top_k is not None and top_k > max_top_k:
+            top_k = max_top_k
+        floor = self._limits.get("min_score_floor")
+        if floor is not None and min_score < floor:
+            min_score = floor
+
+        payload = {
+            "text": text,
+            "tags": tags or "",
+            "top_k": top_k,
+            "min_score": min_score,
+            "strategy": strategy,
+        }
+        data = await self._post("/query", json=payload, auth=False)
+        docs_raw = data.get("documents", [])
+        return [KnowledgeDocument(**d) for d in docs_raw]
+
+    # ------------------------------------------------------------------ #
+    # HTTP helpers
+    # ------------------------------------------------------------------ #
+
+    def _headers(self, auth: bool) -> dict[str, str]:
+        h: dict[str, str] = {"Content-Type": "application/json"}
+        if auth and self.api_key:
+            h["X-Internal-API-Key"] = self.api_key
+        return h
+
+    async def _post(
+        self, path: str, *, json: dict[str, Any], auth: bool
+    ) -> dict[str, Any]:
+        async with httpx.AsyncClient(timeout=self.timeout) as c:
+            r = await c.post(self.url + path, json=json, headers=self._headers(auth))
+            r.raise_for_status()
+            if r.status_code == 204 or not r.content:
+                return {}
+            return cast("dict[str, Any]", r.json())
+
+    async def _delete(self, path: str, *, auth: bool) -> None:
+        async with httpx.AsyncClient(timeout=self.timeout) as c:
+            r = await c.delete(self.url + path, headers=self._headers(auth))
+            r.raise_for_status()
+
+
+def content_hash(text: str) -> str:
+    """Stable content hash, used as a cache key for embeddings.
+
+    Exposed for tests and for providers that want to compute IDs from
+    content when no natural key exists.
+    """
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()

spot_sdk/knowledge_tags.py

@@ -0,0 +1,31 @@
+"""Well-known shared-vocabulary tag constants for the Knowledge Store.
+
+Tags are plain strings on ``KnowledgeDocument.tags``; these constants
+encode the recommended common vocabulary so providers and analyzers
+agree on the same labels. External developers may introduce private
+tags (e.g. ``"acme:jira_issue"``) without changing the SDK.
+"""
+
+from __future__ import annotations
+
+
+class KnowledgeTag:
+    """Common tag constants. Use these for interoperability across plugins."""
+
+    # ----- "Kind" tags: what is this document? -----
+    EMPLOYEE = "employee"
+    WIKI_PAGE = "wiki_page"
+    POLICY = "policy"
+    ORG_CHART = "org_chart"
+    INCIDENT_REPORT = "incident_report"
+    TICKET = "ticket"
+    THREAT_REPORT = "threat_report"
+    SENDER_HISTORY = "sender_history"
+
+    # ----- "Subset" tags: who/what is this about? -----
+    EXECUTIVE = "executive"
+    FINANCE = "finance"
+    ENGINEERING = "engineering"
+    HR = "hr"
+    PUBLIC = "public"
+    INTERNAL = "internal"

spot_sdk/logging.py

@@ -0,0 +1,133 @@
+"""
+Standard logging configuration for SPOT platform.
+
+Uses Python's built-in logging with dictConfig.
+Simple, configurable via environment variables.
+"""
+
+from __future__ import annotations
+
+import logging
+import logging.config
+import os
+from typing import Any, Optional
+
+
+def get_log_level() -> str:
+    """Get log level from environment, default to INFO."""
+    return os.getenv("LOG_LEVEL", "INFO").upper()
+
+
+def get_log_format() -> str:
+    """Get log format from environment, default to simple."""
+    return os.getenv("LOG_FORMAT", "simple").lower()
+
+
+def get_logging_config(
+    log_level: Optional[str] = None, log_format: Optional[str] = None
+) -> dict[str, Any]:
+    """
+    Get logging configuration dictionary for dictConfig.
+
+    Args:
+        log_level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+        log_format: Format style (simple, detailed, json)
+
+    Returns:
+        Dictionary suitable for logging.config.dictConfig
+    """
+    level = log_level or get_log_level()
+    fmt = log_format or get_log_format()
+
+    # Choose format string
+    if fmt == "json":
+        format_str = '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "logger": "%(name)s", "message": "%(message)s"}'
+    elif fmt == "detailed":
+        format_str = "%(asctime)s - %(name)-20s - %(levelname)-8s - [%(filename)s:%(lineno)d] - %(message)s"
+    else:  # simple
+        format_str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+    config = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "default": {
+                "format": format_str,
+                "datefmt": "%Y-%m-%d %H:%M:%S" if fmt == "detailed" else "%H:%M:%S",
+            }
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "formatter": "default",
+                "stream": "ext://sys.stdout",
+            }
+        },
+        "root": {"level": level, "handlers": ["console"]},
+    }
+
+    # Add file handler if LOG_FILE is specified
+    log_file = os.getenv("LOG_FILE")
+    if log_file:
+        config["handlers"]["file"] = {  # type: ignore[index]
+            "class": "logging.FileHandler",
+            "formatter": "default",
+            "filename": log_file,
+            "mode": "a",
+        }
+        config["root"]["handlers"].append("file")  # type: ignore[index]
+
+    return config
+
+
+def configure_logging(
+    log_level: Optional[str] = None, log_format: Optional[str] = None
+) -> None:
+    """
+    Configure logging for the application.
+
+    Call this once during application startup.
+
+    Args:
+        log_level: Log level (default: from LOG_LEVEL env var or INFO)
+        log_format: Format style (default: from LOG_FORMAT env var or simple)
+
+    Environment Variables:
+        LOG_LEVEL: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO)
+        LOG_FORMAT: json, simple, detailed (default: simple)
+        LOG_FILE: Path to log file (optional, logs to console if not set)
+    """
+    config = get_logging_config(log_level, log_format)
+    logging.config.dictConfig(config)
+
+    # Log configuration
+    logger = logging.getLogger("spot.logging")
+    level = log_level or get_log_level()
+    fmt = log_format or get_log_format()
+    log_file = os.getenv("LOG_FILE")
+
+    log_targets = "console"
+    if log_file:
+        log_targets += f" and file ({log_file})"
+
+    logger.info(
+        f"Logging configured - Level: {level}, Format: {fmt}, Output: {log_targets}"
+    )
+
+
+def get_logger(name: str) -> logging.Logger:
+    """
+    Get a logger instance for the given name.
+
+    Args:
+        name: Logger name, typically __name__
+
+    Returns:
+        Standard Python logger
+
+    Example:
+        logger = get_logger(__name__)
+        logger.info("Service started")
+        logger.debug("Processing email", extra={"email_id": "123"})
+    """
+    return logging.getLogger(name)

spot_sdk/ollama.py

@@ -0,0 +1,58 @@
+"""Protocol for Ollama-compatible LLM clients and shared response model."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+from pydantic import BaseModel, Field
+
+
+@runtime_checkable
+class OllamaClientProtocol(Protocol):
+    """Protocol defining the interface for Ollama-compatible clients.
+
+    Both real and mock Ollama clients should implement this protocol.
+    The protocol captures the minimal shared surface: generating a text
+    response from a prompt and (optionally) lifecycle management.
+
+    Existing client classes already satisfy this protocol structurally --
+    no code changes are needed in analyzer-llm or analyzer-fake-llm.
+    """
+
+    async def generate_response(self, prompt: str, model: str) -> str:
+        """Generate a text response from the LLM.
+
+        Args:
+            prompt: The prompt to send to the LLM.
+            model: Model name to use for generation.
+
+        Returns:
+            Generated text response (typically JSON for analysis prompts).
+        """
+        ...
+
+    async def check_connection(self) -> bool:
+        """Check if the Ollama service is reachable.
+
+        Returns:
+            True if the service is reachable, False otherwise.
+        """
+        ...
+
+    async def close(self) -> None:
+        """Release any resources held by the client."""
+        ...
+
+
+class LLMResponse(BaseModel):
+    """Schema for validating LLM JSON responses.
+
+    Shared between analyzer-llm and analyzer-fake-llm so the response
+    format is defined in a single place.
+    """
+
+    is_phishing: bool = False
+    confidence: float = Field(default=0.0, ge=0.0, le=1.0)
+    threat_level: str = "safe"
+    explanation: str = "Analysis completed"
+    indicators: list[dict[str, Any]] = Field(default_factory=list)

spot_sdk/orchestrator.py

@@ -0,0 +1,70 @@
+"""Orchestrator result models for SPOT platform."""
+
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from .analyzer import ThreatLevel
+from .results import AnalysisIndicator
+
+
+class AnalyzerResult(BaseModel):
+    """Result from a single analyzer execution."""
+
+    analyzer_id: str = Field(..., description="Analyzer ID")
+    analyzer_type: str = Field(..., description="Type of analyzer")
+    execution_time_ms: int = Field(..., description="Execution time in milliseconds")
+    status: str = Field(..., description="Execution status (success, failed, timeout)")
+    is_phishing: bool = Field(..., description="Whether phishing was detected")
+    confidence: float = Field(..., description="Confidence score (0.0 to 1.0)")
+    threat_level: ThreatLevel = Field(
+        default=ThreatLevel.SAFE, description="Threat level from analyzer"
+    )
+    indicators: list[AnalysisIndicator] = Field(
+        default_factory=list, description="Detected indicators"
+    )
+    analyzer_details: dict[str, Any] = Field(
+        default_factory=dict, description="Additional analyzer-specific details"
+    )
+
+
+class WorkflowStageResult(BaseModel):
+    """Result from a single workflow stage execution."""
+
+    stage_name: str = Field(..., description="Stage name")
+    stage_type: str = Field(..., description="Stage type")
+    execution_time_ms: int = Field(
+        ..., description="Stage execution time in milliseconds"
+    )
+    analyzer_results: list[AnalyzerResult] = Field(
+        default_factory=list, description="Results from all analyzers in this stage"
+    )
+    weighted_confidence: float = Field(..., description="Aggregated confidence score")
+
+
+class OrchestrationResult(BaseModel):
+    """Final orchestration result combining all analyzer outputs."""
+
+    id: str = Field(..., description="Result ID")
+    email_id: str = Field(..., description="Email ID")
+    workflow_id: str = Field(..., description="Workflow ID used")
+    workflow_version: int = Field(..., description="Workflow version")
+    is_phishing: bool = Field(..., description="Final phishing determination")
+    threat_level: ThreatLevel = Field(..., description="Overall threat level")
+    confidence: float = Field(..., description="Final confidence score (0.0 to 1.0)")
+    total_execution_time_ms: int = Field(
+        ..., description="Total execution time in milliseconds"
+    )
+    analyzed_at: datetime = Field(..., description="Analysis timestamp")
+    summary: str = Field(..., description="Human-readable summary")
+    recommended_action: str = Field(..., description="Recommended action")
+    reasoning: dict[str, Any] = Field(
+        default_factory=dict, description="Reasoning behind the decision"
+    )
+    config_snapshot: dict[str, Any] = Field(
+        default_factory=dict, description="Snapshot of configuration used"
+    )
+    indicators: list[AnalysisIndicator] = Field(
+        default_factory=list, description="All detected indicators"
+    )

spot_sdk/plugin.py

@@ -0,0 +1,30 @@
+"""Plugin vocabulary shared across the platform.
+
+A "plugin" is any pluggable component registered with a SPOT platform
+through the plugin catalog, installer, and sources. Plugins come in two
+kinds today:
+
+- ``analyzer``: runs ``POST /internal/analyze`` and produces an
+  ``AnalysisResult`` (a phishing verdict contributing to aggregation).
+- ``context_provider``: runs ``POST /internal/enrich`` and produces an
+  ``EnrichmentResult`` whose data populates ``analysis_context`` for
+  downstream analyzers.
+
+The ``PluginKind`` value is surfaced by the platform catalog (derived
+from OCI image labels) so the installer can route a plugin to the
+correct section of ``spot.yaml``.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class PluginKind(str, Enum):
+    """Discriminator for pluggable SPOT components."""
+
+    ANALYZER = "analyzer"
+    CONTEXT_PROVIDER = "context_provider"
+
+    def __str__(self) -> str:  # pragma: no cover - trivial
+        return self.value