spamkit-api 0.1.0__py3-none-any.whl

spamkit/client.py

@@ -0,0 +1,174 @@
+"""Python client for the Spamkit v1 API. Needs only the base install (httpx + pydantic)."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from types import TracebackType
+from typing import Any
+
+import httpx
+
+from spamkit.schemas import InfoResponse, ReadyResponse, ScoreResult
+
+RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504})
+
+
+class SpamkitError(Exception):
+    """An error response from the service. ``code`` is one of the stable codes in ``ErrorCode``."""
+
+    def __init__(self, code: str, message: str, status: int | None = None, request_id: str | None = None) -> None:
+        super().__init__(f"{code}: {message}")
+        self.code = code
+        self.message = message
+        self.status = status
+        self.request_id = request_id
+
+
+class SpamkitTransportError(SpamkitError):
+    """The request never produced an HTTP response (connection refused, reset, DNS failure)."""
+
+    def __init__(self, message: str) -> None:
+        super().__init__("transport_error", message)
+
+
+class SpamkitTimeoutError(SpamkitTransportError):
+    """The request timed out before a response arrived."""
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.code = "timeout"
+
+
+@dataclass(frozen=True)
+class ItemError:
+    code: str
+    message: str
+
+
+@dataclass(frozen=True)
+class BatchOutcome:
+    """One entry per submitted item, in input order. Exactly one of ``result`` or ``error`` is set."""
+
+    id: str | None
+    result: ScoreResult | None = None
+    error: ItemError | None = None
+
+    @property
+    def ok(self) -> bool:
+        return self.result is not None
+
+
+def _api_error(response: httpx.Response) -> SpamkitError:
+    # Never include the response body in the message: it may echo submitted text.
+    try:
+        body = response.json()["error"]
+        return SpamkitError(
+            code=str(body["code"]),
+            message=str(body["message"]),
+            status=response.status_code,
+            request_id=body.get("request_id"),
+        )
+    except (ValueError, KeyError, TypeError):
+        return SpamkitError(
+            code="unknown_error",
+            message=f"Unexpected HTTP {response.status_code} response.",
+            status=response.status_code,
+        )
+
+
+class SpamkitClient:
+    """Synchronous client. Retries are opt-in; scoring is read-only, so a retry cannot duplicate side effects."""
+
+    def __init__(
+        self,
+        base_url: str,
+        *,
+        timeout: float = 5.0,
+        api_token: str | None = None,
+        max_retries: int = 0,
+        retry_backoff: float = 0.25,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        if max_retries < 0:
+            raise ValueError("max_retries must be zero or greater")
+        headers = {"Authorization": f"Bearer {api_token}"} if api_token else {}
+        self._http = httpx.Client(
+            base_url=base_url.rstrip("/"),
+            timeout=timeout,
+            headers=headers,
+            transport=transport,
+        )
+        self._max_retries = max_retries
+        self._retry_backoff = retry_backoff
+
+    def __enter__(self) -> SpamkitClient:  # noqa: PYI034 - returns the instance for context managers
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        self._http.close()
+
+    def score(self, text: str, *, id: str | None = None, context: str | None = None) -> ScoreResult:
+        payload = _drop_none({"text": text, "id": id, "context": context})
+        response = self._request("POST", "/v1/score", payload)
+        return ScoreResult.model_validate(response.json())
+
+    def score_batch(self, items: Sequence[Mapping[str, Any]]) -> list[BatchOutcome]:
+        """Each item is a mapping with ``text`` and optionally ``id`` and ``context``."""
+        response = self._request("POST", "/v1/score/batch", {"items": [dict(item) for item in items]})
+        outcomes: list[BatchOutcome] = []
+        for entry in response.json()["results"]:
+            if entry.get("ok"):
+                outcomes.append(BatchOutcome(id=entry["result"].get("id"), result=ScoreResult.model_validate(entry["result"])))
+            else:
+                error = entry["error"]
+                outcomes.append(
+                    BatchOutcome(id=entry.get("id"), error=ItemError(code=error["code"], message=error["message"]))
+                )
+        return outcomes
+
+    def health(self) -> bool:
+        return self._request("GET", "/healthz").is_success
+
+    def ready(self) -> ReadyResponse:
+        return ReadyResponse.model_validate(self._request("GET", "/readyz").json())
+
+    def info(self) -> InfoResponse:
+        return InfoResponse.model_validate(self._request("GET", "/v1/info").json())
+
+    def _request(self, method: str, path: str, json_body: Any = None) -> httpx.Response:
+        attempts = self._max_retries + 1
+        for attempt in range(attempts):
+            error: SpamkitError
+            try:
+                response = self._http.request(method, path, json=json_body)
+            except httpx.TimeoutException as exc:
+                error = SpamkitTimeoutError(str(type(exc).__name__))
+            except httpx.TransportError as exc:
+                error = SpamkitTransportError(str(type(exc).__name__))
+            else:
+                if response.is_success:
+                    return response
+                error = _api_error(response)
+                if response.status_code not in RETRYABLE_STATUS:
+                    raise error
+            if attempt == attempts - 1:
+                raise error
+            time.sleep(self._retry_delay(attempt))
+        raise AssertionError("unreachable")  # the loop always returns or raises
+
+    def _retry_delay(self, attempt: int) -> float:
+        return self._retry_backoff * (2**attempt)
+
+
+def _drop_none(values: dict[str, Any]) -> dict[str, Any]:
+    return {key: value for key, value in values.items() if value is not None}

spamkit/errors.py

@@ -0,0 +1,40 @@
+"""Stable error codes shared by the server, the Python client, and the TypeScript client."""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class ErrorCode(str, Enum):
+    INVALID_REQUEST = "invalid_request"
+    UNAUTHORIZED = "unauthorized"
+    INPUT_TOO_LARGE = "input_too_large"
+    UNSUPPORTED_INPUT = "unsupported_input"
+    RATE_LIMITED = "rate_limited"
+    MODEL_NOT_READY = "model_not_ready"
+    INTERNAL_ERROR = "internal_error"
+
+
+HTTP_STATUS: dict[ErrorCode, int] = {
+    ErrorCode.INVALID_REQUEST: 400,
+    ErrorCode.UNAUTHORIZED: 401,
+    ErrorCode.INPUT_TOO_LARGE: 413,
+    ErrorCode.UNSUPPORTED_INPUT: 422,
+    ErrorCode.RATE_LIMITED: 429,
+    ErrorCode.MODEL_NOT_READY: 503,
+    ErrorCode.INTERNAL_ERROR: 500,
+}
+
+
+class ApiError(Exception):
+    """An error with a stable code. The message must never contain submitted text."""
+
+    def __init__(self, code: ErrorCode, message: str, headers: dict[str, str] | None = None) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.headers = headers or {}
+
+    @property
+    def status(self) -> int:
+        return HTTP_STATUS[self.code]

spamkit/evaluation.py

@@ -0,0 +1,180 @@
+"""Offline evaluation on a user's own labeled file. Nothing here uploads data or retains text."""
+
+from __future__ import annotations
+
+import csv
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+DEFAULT_SWEEP = tuple(round(step / 20, 2) for step in range(1, 20))  # 0.05 .. 0.95
+
+
+class EvaluationInputError(ValueError):
+    """Raised for malformed input. Messages name the row, never its text."""
+
+
+@dataclass(frozen=True)
+class Sample:
+    row: int
+    text: str
+    label: str  # "spam" or "clean"
+    id: str | None
+
+
+@dataclass(frozen=True)
+class Confusion:
+    tp: int
+    fp: int
+    tn: int
+    fn: int
+
+
+def load_samples(path: Path) -> list[Sample]:
+    suffix = path.suffix.lower()
+    if suffix == ".csv":
+        raw_rows = _read_csv(path)
+    elif suffix == ".jsonl":
+        raw_rows = _read_jsonl(path)
+    else:
+        raise EvaluationInputError("Input must be a .csv or .jsonl file.")
+    if not raw_rows:
+        raise EvaluationInputError("The file contains no data rows.")
+    return [_to_sample(row, number) for number, row in raw_rows]
+
+
+def _read_csv(path: Path) -> list[tuple[int, dict[str, Any]]]:
+    with path.open(newline="", encoding="utf-8") as handle:
+        reader = csv.DictReader(handle)
+        fields = {name.strip() for name in (reader.fieldnames or []) if name}
+        missing = {"text", "label"} - fields
+        if missing:
+            raise EvaluationInputError(f"CSV is missing required column(s): {', '.join(sorted(missing))}.")
+        # Row numbers are physical file lines, so they match what an editor shows.
+        rows = []
+        for row in reader:
+            rows.append((reader.line_num, row))
+        return rows
+
+
+def _read_jsonl(path: Path) -> list[tuple[int, dict[str, Any]]]:
+    # Line numbers, not record counts: blank lines are skipped but still counted.
+    rows: list[tuple[int, dict[str, Any]]] = []
+    with path.open(encoding="utf-8") as handle:
+        for number, line in enumerate(handle, start=1):
+            if not line.strip():
+                continue
+            try:
+                value = json.loads(line)
+            except ValueError as exc:
+                raise EvaluationInputError(f"row {number}: line is not valid JSON.") from exc
+            if not isinstance(value, dict):
+                raise EvaluationInputError(f"row {number}: each line must be a JSON object.")
+            rows.append((number, value))
+    return rows
+
+
+def _to_sample(row: dict[str, Any], number: int) -> Sample:
+    text = row.get("text")
+    if not isinstance(text, str) or not text.strip():
+        raise EvaluationInputError(f"row {number}: text is empty or missing.")
+    label = str(row.get("label", "")).strip().lower()
+    # "ham" is accepted because most public spam datasets use it. Internally the class is "clean".
+    label = "clean" if label == "ham" else label
+    if label not in {"spam", "clean"}:
+        raise EvaluationInputError(f"row {number}: label must be 'spam' or 'clean'.")
+    raw_id = row.get("id")
+    sample_id = str(raw_id).strip() if raw_id not in (None, "") else None
+    return Sample(row=number, text=text, label=label, id=sample_id)
+
+
+def confusion_at(samples: Sequence[Sample], scores: Sequence[float], threshold: float) -> Confusion:
+    """Predicted spam when score >= threshold. Review is not modeled here; see the sweep for trade-offs."""
+    tp = fp = tn = fn = 0
+    for sample, score in zip(samples, scores):
+        predicted_spam = score >= threshold
+        actual_spam = sample.label == "spam"
+        if predicted_spam and actual_spam:
+            tp += 1
+        elif predicted_spam:
+            fp += 1
+        elif actual_spam:
+            fn += 1
+        else:
+            tn += 1
+    return Confusion(tp=tp, fp=fp, tn=tn, fn=fn)
+
+
+def _ratio(numerator: int, denominator: int) -> float | None:
+    return None if denominator == 0 else numerator / denominator
+
+
+def metrics_for(matrix: Confusion) -> dict[str, float | None]:
+    precision = _ratio(matrix.tp, matrix.tp + matrix.fp)
+    recall = _ratio(matrix.tp, matrix.tp + matrix.fn)
+    f1 = None
+    if precision is not None and recall is not None and precision + recall > 0:
+        f1 = 2 * precision * recall / (precision + recall)
+    return {
+        "precision": precision,
+        "recall": recall,
+        "specificity": _ratio(matrix.tn, matrix.tn + matrix.fp),
+        "false_positive_rate": _ratio(matrix.fp, matrix.fp + matrix.tn),
+        "false_negative_rate": _ratio(matrix.fn, matrix.fn + matrix.tp),
+        "f1": f1,
+    }
+
+
+def build_report(
+    samples: Sequence[Sample],
+    scores: Sequence[float],
+    *,
+    threshold: float,
+    sweep: Sequence[float] = DEFAULT_SWEEP,
+    model: str,
+    model_version: str,
+    software_version: str,
+    show_text: bool = False,
+) -> dict[str, Any]:
+    if len(samples) != len(scores):
+        raise ValueError("samples and scores must have the same length")
+
+    matrix = confusion_at(samples, scores, threshold)
+    spam_count = sum(1 for sample in samples if sample.label == "spam")
+    false_positives = []
+    false_negatives = []
+    for sample, score in zip(samples, scores):
+        predicted_spam = score >= threshold
+        if predicted_spam and sample.label == "clean":
+            false_positives.append(_error_row(sample, score, show_text))
+        elif not predicted_spam and sample.label == "spam":
+            false_negatives.append(_error_row(sample, score, show_text))
+
+    return {
+        "dataset": {
+            "count": len(samples),
+            "spam": spam_count,
+            "clean": len(samples) - spam_count,
+            "note": "Metrics describe only the supplied dataset.",
+        },
+        "model": {"id": model, "version": model_version},
+        "software_version": software_version,
+        "threshold": threshold,
+        "confusion": {"tp": matrix.tp, "fp": matrix.fp, "tn": matrix.tn, "fn": matrix.fn},
+        "metrics": metrics_for(matrix),
+        "sweep": [
+            {"threshold": point, **metrics_for(confusion_at(samples, scores, point))}
+            for point in sweep
+        ],
+        "false_positives": false_positives,
+        "false_negatives": false_negatives,
+    }
+
+
+def _error_row(sample: Sample, score: float, show_text: bool) -> dict[str, Any]:
+    entry: dict[str, Any] = {"row": sample.row, "id": sample.id, "score": score}
+    if show_text:
+        entry["text"] = sample.text
+    return entry

spamkit/observability.py

@@ -0,0 +1,86 @@
+"""Structured logs and metrics. Neither ever receives submitted text or caller IDs."""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+import time
+from collections import Counter
+
+from spamkit.verdict import Verdict
+
+logger = logging.getLogger("spamkit")
+
+LATENCY_BUCKETS_MS = (5, 10, 25, 50, 100, 250, 500, 1000, 2500)
+
+
+def log_event(event: str, **fields: object) -> None:
+    record = {"event": event, "ts": round(time.time(), 3), **fields}
+    logger.info(json.dumps(record, sort_keys=True))
+
+
+def score_band(score: float) -> str:
+    """Coarse bucket for logs. Exact scores are kept out of logs to limit fingerprinting."""
+    low = min(int(score * 10), 9)
+    return f"{low / 10:.1f}-{(low + 1) / 10:.1f}"
+
+
+class Metrics:
+    """Process-local counters and a latency histogram, rendered in Prometheus text format."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._requests: Counter[tuple[str, int]] = Counter()
+        self._verdicts: Counter[str] = Counter()
+        self._errors: Counter[str] = Counter()
+        self._latency_bucket_counts = [0] * (len(LATENCY_BUCKETS_MS) + 1)
+        self._latency_sum_ms = 0.0
+        self._latency_count = 0
+
+    def record_request(self, route: str, status: int) -> None:
+        with self._lock:
+            self._requests[(route, status)] += 1
+
+    def record_verdict(self, verdict: Verdict) -> None:
+        with self._lock:
+            self._verdicts[verdict.value] += 1
+
+    def record_error(self, code: str) -> None:
+        with self._lock:
+            self._errors[code] += 1
+
+    def record_latency(self, milliseconds: float) -> None:
+        index = next(
+            (i for i, bound in enumerate(LATENCY_BUCKETS_MS) if milliseconds <= bound),
+            len(LATENCY_BUCKETS_MS),
+        )
+        with self._lock:
+            self._latency_bucket_counts[index] += 1
+            self._latency_sum_ms += milliseconds
+            self._latency_count += 1
+
+    def render(self) -> str:
+        lines: list[str] = []
+        with self._lock:
+            lines.append("# TYPE spamkit_requests_total counter")
+            for (route, status), count in sorted(self._requests.items()):
+                lines.append(f'spamkit_requests_total{{route="{route}",status="{status}"}} {count}')
+            lines.append("# TYPE spamkit_verdicts_total counter")
+            for verdict in Verdict:
+                lines.append(
+                    f'spamkit_verdicts_total{{verdict="{verdict.value}"}} {self._verdicts[verdict.value]}'
+                )
+            lines.append("# TYPE spamkit_errors_total counter")
+            for code, count in sorted(self._errors.items()):
+                lines.append(f'spamkit_errors_total{{code="{code}"}} {count}')
+            lines.append("# TYPE spamkit_score_latency_ms histogram")
+            cumulative = 0
+            for bound, count in zip(LATENCY_BUCKETS_MS, self._latency_bucket_counts):
+                cumulative += count
+                lines.append(f'spamkit_score_latency_ms_bucket{{le="{bound}"}} {cumulative}')
+            cumulative += self._latency_bucket_counts[-1]
+            lines.append(f'spamkit_score_latency_ms_bucket{{le="+Inf"}} {cumulative}')
+            lines.append(f"spamkit_score_latency_ms_sum {self._latency_sum_ms:.3f}")
+            lines.append(f"spamkit_score_latency_ms_count {self._latency_count}")
+        return "\n".join(lines) + "\n"

spamkit/schemas.py

@@ -0,0 +1,121 @@
+"""Wire types for the v1 API, plus request validation that never echoes submitted text."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from pydantic import BaseModel
+
+from spamkit.errors import ApiError, ErrorCode
+from spamkit.verdict import Verdict
+
+API_VERSION = "v1"
+MAX_ID_CHARS = 256
+MAX_CONTEXT_CHARS = 64
+
+
+class ScoreResult(BaseModel):
+    id: str | None
+    score: float
+    verdict: Verdict
+    model: str
+    model_version: str
+    latency_ms: int
+    truncated: bool
+
+
+class ErrorBody(BaseModel):
+    code: ErrorCode
+    message: str
+    request_id: str
+
+
+class ErrorResponse(BaseModel):
+    error: ErrorBody
+
+
+class BatchItemOk(BaseModel):
+    ok: Literal[True] = True
+    result: ScoreResult
+
+
+class BatchItemError(BaseModel):
+    ok: Literal[False] = False
+    id: str | None
+    error: ErrorBody
+
+
+class BatchResponse(BaseModel):
+    results: list[BatchItemOk | BatchItemError]
+
+
+class HealthResponse(BaseModel):
+    status: Literal["ok"]
+
+
+class ReadyResponse(BaseModel):
+    status: Literal["ready"]
+    model: str
+    model_version: str
+
+
+class InfoResponse(BaseModel):
+    api_version: str
+    model: str
+    model_version: str | None
+    max_tokens: int
+    max_text_bytes: int
+    max_batch_items: int
+    thresholds: dict[str, float]
+    threshold_config_version: str
+    service_version: str
+
+
+@dataclass(frozen=True)
+class ScoreInput:
+    text: str
+    id: str | None
+    context: str | None
+
+
+def require_utf8(value: str, field: str) -> str:
+    """JSON allows lone surrogates such as \\ud800. They cannot be encoded as UTF-8, so reject them."""
+    try:
+        value.encode("utf-8")
+    except UnicodeEncodeError as exc:
+        raise ApiError(ErrorCode.UNSUPPORTED_INPUT, f"{field} must be valid UTF-8 without lone surrogates.") from exc
+    return value
+
+
+def _optional_short_string(value: Any, field: str, limit: int) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        raise ApiError(ErrorCode.INVALID_REQUEST, f"{field} must be a string when provided.")
+    if len(value) > limit:
+        raise ApiError(ErrorCode.INVALID_REQUEST, f"{field} must be at most {limit} characters.")
+    return require_utf8(value, field)
+
+
+def parse_score_input(payload: Any, max_text_bytes: int) -> ScoreInput:
+    if not isinstance(payload, dict):
+        raise ApiError(ErrorCode.INVALID_REQUEST, "Request body must be a JSON object.")
+
+    text = payload.get("text")
+    if not isinstance(text, str) or not text.strip():
+        raise ApiError(ErrorCode.INVALID_REQUEST, "text is required and must be a non-empty string.")
+    require_utf8(text, "text")
+    if len(text.encode("utf-8")) > max_text_bytes:
+        raise ApiError(
+            ErrorCode.INPUT_TOO_LARGE,
+            f"text must be at most {max_text_bytes} bytes of UTF-8.",
+        )
+    if "\x00" in text:
+        raise ApiError(ErrorCode.UNSUPPORTED_INPUT, "text must not contain NUL characters.")
+
+    return ScoreInput(
+        text=text,
+        id=_optional_short_string(payload.get("id"), "id", MAX_ID_CHARS),
+        context=_optional_short_string(payload.get("context"), "context", MAX_CONTEXT_CHARS),
+    )

spamkit/settings.py

@@ -0,0 +1,67 @@
+"""Runtime configuration, read from ``SPAMKIT_*`` environment variables."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from spamkit.verdict import Thresholds
+
+DEFAULT_MODEL_DIR = Path("models/spamton")
+
+
+def _int(env: Mapping[str, str], name: str, default: int) -> int:
+    raw = env.get(name)
+    if raw is None or raw == "":
+        return default
+    try:
+        value = int(raw)
+    except ValueError as exc:
+        raise ValueError(f"{name} must be an integer") from exc
+    if value <= 0:
+        raise ValueError(f"{name} must be greater than zero")
+    return value
+
+
+def _float(env: Mapping[str, str], name: str, default: float) -> float:
+    raw = env.get(name)
+    if raw is None or raw == "":
+        return default
+    try:
+        return float(raw)
+    except ValueError as exc:
+        raise ValueError(f"{name} must be a number") from exc
+
+
+@dataclass(frozen=True)
+class Settings:
+    model_dir: Path = DEFAULT_MODEL_DIR
+    host: str = "127.0.0.1"
+    port: int = 8080
+    thresholds: Thresholds = field(default_factory=Thresholds)
+    max_text_bytes: int = 16_384
+    max_body_bytes: int = 1_048_576
+    max_batch_items: int = 100
+    max_inflight: int = 32
+    api_token: str | None = None
+
+    @classmethod
+    def from_env(cls, env: Mapping[str, str] | None = None) -> Settings:
+        env = os.environ if env is None else env
+        token = env.get("SPAMKIT_API_TOKEN") or None
+        return cls(
+            model_dir=Path(env.get("SPAMKIT_MODEL_DIR") or DEFAULT_MODEL_DIR),
+            host=env.get("SPAMKIT_HOST") or "127.0.0.1",
+            port=_int(env, "SPAMKIT_PORT", 8080),
+            thresholds=Thresholds(
+                clean=_float(env, "SPAMKIT_CLEAN_THRESHOLD", 0.0),
+                spam=_float(env, "SPAMKIT_SPAM_THRESHOLD", 1.0),
+            ),
+            max_text_bytes=_int(env, "SPAMKIT_MAX_TEXT_BYTES", 16_384),
+            max_body_bytes=_int(env, "SPAMKIT_MAX_BODY_BYTES", 1_048_576),
+            max_batch_items=_int(env, "SPAMKIT_MAX_BATCH_ITEMS", 100),
+            max_inflight=_int(env, "SPAMKIT_MAX_INFLIGHT", 32),
+            api_token=token,
+        )