spamkit-api 0.1.0__py3-none-any.whl

spamkit/__init__.py

@@ -0,0 +1,27 @@
+"""Spamkit: spam scoring for user-submitted text.
+
+The server (``spamkit.app``) needs the ``server`` extra. The client (``spamkit.client``)
+and the threshold and error types need only the base install.
+"""
+
+from spamkit.client import (
+    SpamkitClient,
+    SpamkitError,
+    SpamkitTimeoutError,
+    SpamkitTransportError,
+)
+from spamkit.errors import ErrorCode
+from spamkit.verdict import Thresholds, Verdict
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ErrorCode",
+    "SpamkitClient",
+    "SpamkitError",
+    "SpamkitTimeoutError",
+    "SpamkitTransportError",
+    "Thresholds",
+    "Verdict",
+    "__version__",
+]

spamkit/app.py

@@ -0,0 +1,314 @@
+"""FastAPI application for the v1 scoring API."""
+
+from __future__ import annotations
+
+import hmac
+import json
+import re
+import threading
+import time
+from collections.abc import AsyncIterator, Sequence
+from contextlib import asynccontextmanager
+from typing import Any
+from uuid import uuid4
+
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse, PlainTextResponse
+from starlette.concurrency import run_in_threadpool
+
+from spamkit import __version__
+from spamkit.classifier import MODEL_ID, ScoredText, Scorer, SpamtonClassifier
+from spamkit.errors import ApiError, ErrorCode
+from spamkit.observability import Metrics, log_event, logger, score_band
+from spamkit.schemas import (
+    API_VERSION,
+    BatchItemError,
+    BatchItemOk,
+    BatchResponse,
+    ErrorBody,
+    HealthResponse,
+    InfoResponse,
+    ReadyResponse,
+    ScoreInput,
+    ScoreResult,
+    parse_score_input,
+)
+from spamkit.settings import Settings
+
+_REQUEST_ID = re.compile(r"[A-Za-z0-9._-]{1,128}")
+
+
+class _ServiceState:
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+        self.scorer: Scorer | None = None
+        self.load_error: str | None = None
+        self.metrics = Metrics()
+        self._inflight = 0
+        self._gate = threading.Lock()
+
+    def try_acquire(self) -> bool:
+        """Admission control: refuse work when saturated instead of queueing without bound."""
+        with self._gate:
+            if self._inflight >= self.settings.max_inflight:
+                return False
+            self._inflight += 1
+            return True
+
+    def release(self) -> None:
+        with self._gate:
+            self._inflight -= 1
+
+
+def _state(request: Request) -> _ServiceState:
+    return request.app.state.spamkit  # type: ignore[no-any-return]
+
+
+def _require_auth(request: Request, settings: Settings) -> None:
+    if settings.api_token is None:
+        return
+    supplied = request.headers.get("authorization", "").encode()
+    expected = f"Bearer {settings.api_token}".encode()
+    if not hmac.compare_digest(supplied, expected):
+        raise ApiError(
+            ErrorCode.UNAUTHORIZED,
+            "A valid bearer token is required.",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+
+async def _read_json(request: Request, limit: int) -> Any:
+    """Read the body with a running byte count, so an oversized upload is refused as soon as it crosses the limit."""
+    declared = request.headers.get("content-length")
+    if declared is not None and declared.isdigit() and int(declared) > limit:
+        raise ApiError(ErrorCode.INPUT_TOO_LARGE, f"Request body must be at most {limit} bytes.")
+    chunks: list[bytes] = []
+    total = 0
+    async for chunk in request.stream():
+        total += len(chunk)
+        if total > limit:
+            raise ApiError(ErrorCode.INPUT_TOO_LARGE, f"Request body must be at most {limit} bytes.")
+        chunks.append(chunk)
+    try:
+        return json.loads(b"".join(chunks))
+    except RecursionError as exc:
+        # Very deep nesting exhausts the parser's stack. That is a bad request, not a server fault.
+        raise ApiError(ErrorCode.INVALID_REQUEST, "Request body is nested too deeply.") from exc
+    except ValueError as exc:
+        raise ApiError(ErrorCode.INVALID_REQUEST, "Request body must be valid JSON.") from exc
+
+
+async def _score_texts(state: _ServiceState, texts: Sequence[str]) -> tuple[Scorer, list[ScoredText], float]:
+    """Score ``texts`` and return the scorer that produced them, so callers never re-read shared state."""
+    scorer = state.scorer
+    if scorer is None:
+        raise ApiError(ErrorCode.MODEL_NOT_READY, "The model is not loaded. Check the startup logs.")
+    if not state.try_acquire():
+        raise ApiError(
+            ErrorCode.RATE_LIMITED,
+            "The service is at capacity. Retry shortly.",
+            headers={"Retry-After": "1"},
+        )
+    started = time.perf_counter()
+    try:
+        scored = await run_in_threadpool(scorer.score_many, list(texts))
+    finally:
+        state.release()
+    return scorer, scored, (time.perf_counter() - started) * 1000
+
+
+def _build_result(
+    state: _ServiceState,
+    scorer: Scorer,
+    item: ScoreInput,
+    scored: ScoredText,
+    latency_ms: float,
+    request_id: str,
+) -> ScoreResult:
+    verdict = state.settings.thresholds.verdict_for(scored.score)
+    state.metrics.record_verdict(verdict)
+    log_event(
+        "score",
+        request_id=request_id,
+        verdict=verdict.value,
+        score_band=score_band(scored.score),
+        latency_ms=round(latency_ms, 1),
+        model_version=scorer.model_version,
+        context=item.context,
+        truncated=scored.truncated,
+    )
+    return ScoreResult(
+        id=item.id,
+        score=scored.score,
+        verdict=verdict,
+        model=scorer.model_id,
+        model_version=scorer.model_version,
+        latency_ms=round(latency_ms),
+        truncated=scored.truncated,
+    )
+
+
+def _error_response(request: Request, code: ErrorCode, message: str, status: int, headers: dict[str, str] | None = None) -> JSONResponse:
+    request_id = getattr(request.state, "request_id", None) or uuid4().hex
+    state = _state(request)
+    state.metrics.record_error(code.value)
+    log_event("error", request_id=request_id, code=code.value, status=status)
+    body = {"error": ErrorBody(code=code, message=message, request_id=request_id).model_dump(mode="json")}
+    return JSONResponse(status_code=status, content=body, headers=headers)
+
+
+def create_app(settings: Settings | None = None, scorer: Scorer | None = None) -> FastAPI:
+    """Build the app. Pass ``scorer`` to skip loading the model from ``settings.model_dir``."""
+    settings = settings or Settings.from_env()
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+        state: _ServiceState = app.state.spamkit
+        if state.scorer is None and scorer is None:
+            try:
+                state.scorer = await run_in_threadpool(SpamtonClassifier, settings.model_dir)
+                log_event("model_loaded", model=state.scorer.model_id, model_version=state.scorer.model_version)
+            except Exception as exc:  # noqa: BLE001 - any load failure must leave the service not-ready, not crash it
+                state.load_error = f"{type(exc).__name__}: {exc}"
+                logger.error(
+                    json.dumps(
+                        {
+                            "event": "model_load_failed",
+                            "model_dir": str(settings.model_dir),
+                            "reason": state.load_error,
+                        },
+                        sort_keys=True,
+                    )
+                )
+        yield
+
+    app = FastAPI(
+        title="Spamkit",
+        version=__version__,
+        summary="Spam scoring for user-submitted text.",
+        lifespan=lifespan,
+    )
+    app.state.spamkit = _ServiceState(settings)
+    app.state.spamkit.scorer = scorer
+
+    @app.middleware("http")
+    async def request_context(request: Request, call_next: Any) -> Any:
+        incoming = request.headers.get("x-request-id")
+        request_id = incoming if incoming and _REQUEST_ID.fullmatch(incoming) else uuid4().hex
+        request.state.request_id = request_id
+        started = time.perf_counter()
+        response = await call_next(request)
+        response.headers["X-Request-Id"] = request_id
+        route = getattr(request.scope.get("route"), "path", "unmatched")
+        _state(request).metrics.record_request(route, response.status_code)
+        log_event(
+            "request",
+            request_id=request_id,
+            method=request.method,
+            route=route,
+            status=response.status_code,
+            latency_ms=round((time.perf_counter() - started) * 1000, 1),
+        )
+        return response
+
+    @app.exception_handler(ApiError)
+    async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse:
+        return _error_response(request, exc.code, exc.message, exc.status, exc.headers)
+
+    @app.exception_handler(Exception)
+    async def handle_unexpected(request: Request, exc: Exception) -> JSONResponse:
+        # Log only the exception type: its message could contain request content.
+        log_event("unhandled_exception", exception=type(exc).__name__)
+        return _error_response(request, ErrorCode.INTERNAL_ERROR, "An internal error occurred.", 500)
+
+    @app.get("/healthz", response_model=HealthResponse)
+    async def healthz() -> HealthResponse:
+        return HealthResponse(status="ok")
+
+    @app.get("/readyz", response_model=ReadyResponse, responses={503: {"model": ErrorBody}})
+    async def readyz(request: Request) -> Any:
+        state = _state(request)
+        if state.scorer is None:
+            raise ApiError(ErrorCode.MODEL_NOT_READY, "The model is not loaded. Check the startup logs.")
+        return ReadyResponse(status="ready", model=state.scorer.model_id, model_version=state.scorer.model_version)
+
+    @app.get("/v1/info", response_model=InfoResponse)
+    async def info(request: Request) -> InfoResponse:
+        _require_auth(request, settings)
+        state = _state(request)
+        thresholds = settings.thresholds
+        return InfoResponse(
+            api_version=API_VERSION,
+            model=MODEL_ID,
+            model_version=state.scorer.model_version if state.scorer else None,
+            max_tokens=state.scorer.max_tokens if state.scorer else 512,
+            max_text_bytes=settings.max_text_bytes,
+            max_batch_items=settings.max_batch_items,
+            thresholds={"clean": thresholds.clean, "spam": thresholds.spam},
+            threshold_config_version=thresholds.version,
+            service_version=__version__,
+        )
+
+    @app.post("/v1/score", response_model=ScoreResult)
+    async def score(request: Request) -> ScoreResult:
+        _require_auth(request, settings)
+        state = _state(request)
+        item = parse_score_input(await _read_json(request, settings.max_body_bytes), settings.max_text_bytes)
+        scorer, scored, latency_ms = await _score_texts(state, [item.text])
+        return _build_result(state, scorer, item, scored[0], latency_ms, request.state.request_id)
+
+    @app.post("/v1/score/batch", response_model=BatchResponse)
+    async def score_batch(request: Request) -> BatchResponse:
+        _require_auth(request, settings)
+        state = _state(request)
+        payload = await _read_json(request, settings.max_body_bytes)
+        items = payload.get("items") if isinstance(payload, dict) else None
+        if not isinstance(items, list) or not items:
+            raise ApiError(ErrorCode.INVALID_REQUEST, "Body must be a JSON object with a non-empty items array.")
+        if len(items) > settings.max_batch_items:
+            raise ApiError(ErrorCode.INPUT_TOO_LARGE, f"items may contain at most {settings.max_batch_items} entries.")
+
+        # Validate each item separately so one bad record does not discard the rest.
+        parsed: list[ScoreInput | ApiError] = []
+        for raw in items:
+            try:
+                parsed.append(parse_score_input(raw, settings.max_text_bytes))
+            except ApiError as exc:
+                parsed.append(exc)
+
+        valid = [(position, entry) for position, entry in enumerate(parsed) if isinstance(entry, ScoreInput)]
+        scored_by_position: dict[int, ScoredText] = {}
+        scorer: Scorer | None = None
+        latency_ms = 0.0
+        if valid:
+            scorer, scored, latency_ms = await _score_texts(state, [entry.text for _, entry in valid])
+            scored_by_position = {position: result for (position, _), result in zip(valid, scored)}
+
+        results: list[BatchItemOk | BatchItemError] = []
+        for position, entry in enumerate(parsed):
+            if isinstance(entry, ApiError):
+                results.append(
+                    BatchItemError(
+                        id=_caller_id(items[position]),
+                        error=ErrorBody(code=entry.code, message=entry.message, request_id=request.state.request_id),
+                    )
+                )
+            elif scorer is not None:
+                result = _build_result(
+                    state, scorer, entry, scored_by_position[position], latency_ms, request.state.request_id
+                )
+                results.append(BatchItemOk(result=result))
+        return BatchResponse(results=results)
+
+    @app.get("/metrics", response_class=PlainTextResponse)
+    async def metrics(request: Request) -> str:
+        _require_auth(request, settings)
+        return _state(request).metrics.render()
+
+    return app
+
+
+def _caller_id(raw: Any) -> str | None:
+    if isinstance(raw, dict) and isinstance(raw.get("id"), str) and len(raw["id"]) <= 256:
+        return raw["id"]
+    return None

spamkit/classifier.py

@@ -0,0 +1,122 @@
+"""Model runtime. The API depends only on the ``Scorer`` protocol, so the classifier can be swapped."""
+
+from __future__ import annotations
+
+import json
+import threading
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol
+
+MODEL_ID = "spamton"
+MODEL_SOURCE = "tiagozip/spamton"
+# Pinned revision of tiagozip/spamton (AGPL-3.0). Changing it is a deliberate, reviewed act.
+MODEL_REVISION = "7aa94d049d24bbc302c5a69e63cafe6c44d49bc1"
+MANIFEST_NAME = "spamkit-model.json"
+
+# Preprocessing mirrors the model author's reference script: clip characters first, then
+# truncate to 512 tokens. Index 1 of the softmax is the spam class.
+MAX_CHARS = 6000
+MAX_TOKENS = 512
+INFERENCE_CHUNK = 16
+
+
+class ModelLoadError(RuntimeError):
+    """Raised when the model directory is missing, incomplete, or does not match the expected model."""
+
+
+@dataclass(frozen=True)
+class ScoredText:
+    score: float
+    truncated: bool
+
+
+class Scorer(Protocol):
+    model_id: str
+    model_version: str
+    max_tokens: int
+
+    def score_many(self, texts: Sequence[str]) -> list[ScoredText]: ...
+
+
+def read_manifest(model_dir: Path) -> dict[str, str]:
+    manifest_path = model_dir / MANIFEST_NAME
+    if not manifest_path.is_file():
+        raise ModelLoadError(
+            f"No {MANIFEST_NAME} in {model_dir}. Run `spamkit fetch-model --dest {model_dir}` first."
+        )
+    try:
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        raise ModelLoadError(f"Could not read {manifest_path}: {exc}") from exc
+    if manifest.get("model") != MODEL_ID or not manifest.get("revision"):
+        raise ModelLoadError(f"{manifest_path} does not describe the {MODEL_ID} model.")
+    return manifest
+
+
+class SpamtonClassifier:
+    """Runs spamton on CPU from a local directory. Never downloads anything at runtime."""
+
+    model_id = MODEL_ID
+    max_tokens = MAX_TOKENS
+
+    def __init__(self, model_dir: Path) -> None:
+        manifest = read_manifest(model_dir)
+        self.model_version: str = manifest["revision"]
+
+        import torch
+        from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+        self._torch = torch
+        self._tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True)
+        self._model = AutoModelForSequenceClassification.from_pretrained(
+            model_dir, local_files_only=True
+        ).eval()
+        # One forward pass at a time: CPU inference does not gain from concurrent calls.
+        self._lock = threading.Lock()
+        # The first forward pass pays one-time allocation cost; do it before readiness flips.
+        self.score_many(["warm-up"])
+
+    def score_many(self, texts: Sequence[str]) -> list[ScoredText]:
+        results: list[ScoredText] = []
+        for start in range(0, len(texts), INFERENCE_CHUNK):
+            results.extend(self._score_chunk(texts[start : start + INFERENCE_CHUNK]))
+        return results
+
+    def _score_chunk(self, texts: Sequence[str]) -> list[ScoredText]:
+        clipped = [text[:MAX_CHARS] for text in texts]
+        # The Rust tokenizer is not safe to call from several threads at once ("Already borrowed"),
+        # so tokenization and the forward pass both run under the same lock.
+        with self._lock:
+            truncated = [len(self._tokenizer(text)["input_ids"]) > MAX_TOKENS for text in clipped]
+            encoded = self._tokenizer(
+                clipped,
+                truncation=True,
+                max_length=MAX_TOKENS,
+                padding=True,
+                return_tensors="pt",
+            )
+            with self._torch.inference_mode():
+                spam_probabilities = (
+                    self._model(**encoded).logits.float().softmax(-1)[:, 1].tolist()
+                )
+        return [
+            ScoredText(score=float(score), truncated=was_truncated)
+            for score, was_truncated in zip(spam_probabilities, truncated)
+        ]
+
+
+def fetch_model(dest: Path, revision: str = MODEL_REVISION) -> None:
+    """Download spamton at a pinned revision and write the manifest the server requires."""
+    from huggingface_hub import snapshot_download
+
+    dest.mkdir(parents=True, exist_ok=True)
+    snapshot_download(repo_id=MODEL_SOURCE, revision=revision, local_dir=dest)
+    manifest = {
+        "model": MODEL_ID,
+        "source": MODEL_SOURCE,
+        "revision": revision,
+        "license": "AGPL-3.0 (see LICENSE and NOTICE in this directory)",
+    }
+    (dest / MANIFEST_NAME).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")

spamkit/cli.py

@@ -0,0 +1,199 @@
+"""Command-line entry point: ``spamkit serve | fetch-model | evaluate``."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from dataclasses import replace
+from pathlib import Path
+from typing import Any
+
+from spamkit import __version__
+from spamkit.evaluation import (
+    DEFAULT_SWEEP,
+    EvaluationInputError,
+    build_report,
+    load_samples,
+)
+from spamkit.settings import Settings
+
+EXIT_USAGE = 2
+EXIT_FAILURE = 1
+SERVER_EXTRA_HINT = "This command needs the model runtime. Install with: pip install 'spamkit[server]'"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return args.handler(args)
+    except ModuleNotFoundError as exc:
+        print(f"{SERVER_EXTRA_HINT} (missing module: {exc.name})", file=sys.stderr)
+        return EXIT_FAILURE
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="spamkit", description="Spam scoring for user-submitted text.")
+    parser.add_argument("--version", action="version", version=f"spamkit {__version__}")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    serve = commands.add_parser("serve", help="Run the scoring API.")
+    serve.add_argument("--host", help="Bind address (default: SPAMKIT_HOST or 127.0.0.1).")
+    serve.add_argument("--port", type=int, help="Port (default: SPAMKIT_PORT or 8080).")
+    serve.add_argument("--model-dir", type=Path, help="Model directory (default: SPAMKIT_MODEL_DIR).")
+    serve.set_defaults(handler=_serve)
+
+    fetch = commands.add_parser("fetch-model", help="Download spamton at the pinned revision.")
+    fetch.add_argument("--dest", type=Path, required=True, help="Directory to write the model into.")
+    fetch.add_argument("--revision", help="Override the pinned revision (not recommended).")
+    fetch.set_defaults(handler=_fetch_model)
+
+    evaluate = commands.add_parser("evaluate", help="Score a local labeled file and report metrics.")
+    evaluate.add_argument("path", type=Path, help="CSV or JSONL file with text and label columns. Row numbers in output are file line numbers.")
+    evaluate.add_argument("--threshold", type=float, default=0.5, help="Predict spam when score >= threshold.")
+    evaluate.add_argument(
+        "--sweep",
+        type=float,
+        nargs="+",
+        default=list(DEFAULT_SWEEP),
+        help="Thresholds for the trade-off table (default: 0.05 to 0.95).",
+    )
+    evaluate.add_argument("--model-dir", type=Path, help="Model directory (default: SPAMKIT_MODEL_DIR).")
+    evaluate.add_argument("--output", type=Path, help="Write the full report as JSON to this path.")
+    evaluate.add_argument(
+        "--show-text",
+        action="store_true",
+        help="Print the text of false positives and negatives. Off by default.",
+    )
+    evaluate.set_defaults(handler=_evaluate)
+    return parser
+
+
+def _serve(args: argparse.Namespace) -> int:
+    import uvicorn
+
+    from spamkit.app import create_app
+
+    settings = Settings.from_env()
+    overrides: dict[str, Any] = {}
+    if args.host:
+        overrides["host"] = args.host
+    if args.port:
+        overrides["port"] = args.port
+    if args.model_dir:
+        overrides["model_dir"] = args.model_dir
+    settings = replace(settings, **overrides)
+
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    app = create_app(settings)
+    # One worker only: each worker would load its own copy of the model.
+    uvicorn.run(
+        app,
+        host=settings.host,
+        port=settings.port,
+        workers=1,
+        log_level="warning",
+        access_log=False,
+        timeout_graceful_shutdown=30,
+    )
+    return 0
+
+
+def _fetch_model(args: argparse.Namespace) -> int:
+    from spamkit.classifier import MODEL_REVISION, fetch_model
+
+    revision = args.revision or MODEL_REVISION
+    print(f"Fetching spamton @ {revision} into {args.dest} ...")
+    fetch_model(args.dest, revision=revision)
+    print(f"Done. Manifest written to {args.dest / 'spamkit-model.json'}")
+    return 0
+
+
+def _evaluate(args: argparse.Namespace) -> int:
+    from spamkit.classifier import SpamtonClassifier
+
+    if not 0.0 <= args.threshold <= 1.0:
+        print("--threshold must be between 0 and 1.", file=sys.stderr)
+        return EXIT_USAGE
+    try:
+        samples = load_samples(args.path)
+    except EvaluationInputError as exc:
+        print(f"Input error: {exc}", file=sys.stderr)
+        return EXIT_USAGE
+
+    settings = Settings.from_env()
+    model_dir = args.model_dir or settings.model_dir
+    try:
+        scorer = SpamtonClassifier(model_dir)
+    except Exception as exc:  # noqa: BLE001 - any load failure becomes a clear message and exit code
+        print(f"Could not load the model from {model_dir}: {exc}", file=sys.stderr)
+        return EXIT_FAILURE
+
+    scored = scorer.score_many([sample.text for sample in samples])
+    scores = [item.score for item in scored]
+    report = build_report(
+        samples,
+        scores,
+        threshold=args.threshold,
+        sweep=sorted(set(args.sweep)),
+        model=scorer.model_id,
+        model_version=scorer.model_version,
+        software_version=__version__,
+        show_text=args.show_text,
+    )
+
+    _print_report(report, show_text=args.show_text)
+    if args.output:
+        args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+        print(f"\nFull report written to {args.output}")
+    return 0
+
+
+def _fmt(value: float | None) -> str:
+    return "n/a" if value is None else f"{value:.3f}"
+
+
+def _print_report(report: dict[str, Any], *, show_text: bool) -> None:
+    dataset = report["dataset"]
+    matrix = report["confusion"]
+    metrics = report["metrics"]
+    print("Spamkit evaluation")
+    print(f"Dataset: {dataset['count']} rows (spam {dataset['spam']}, clean {dataset['clean']})")
+    print(f"Model: {report['model']['id']} @ {report['model']['version']}   software {report['software_version']}")
+    print(f"Threshold: {report['threshold']:.2f} (predict spam when score >= threshold)")
+    print()
+    print(f"Confusion matrix   TP {matrix['tp']}   FP {matrix['fp']}   TN {matrix['tn']}   FN {matrix['fn']}")
+    print(
+        f"Precision {_fmt(metrics['precision'])}   Recall {_fmt(metrics['recall'])}   "
+        f"Specificity {_fmt(metrics['specificity'])}   F1 {_fmt(metrics['f1'])}"
+    )
+    print(
+        f"False-positive rate {_fmt(metrics['false_positive_rate'])}   "
+        f"False-negative rate {_fmt(metrics['false_negative_rate'])}"
+    )
+    print()
+    print(f"{'threshold':>9}  {'precision':>9}  {'recall':>6}  {'FPR':>6}  {'FNR':>6}  {'F1':>6}")
+    for point in report["sweep"]:
+        print(
+            f"{point['threshold']:>9.2f}  {_fmt(point['precision']):>9}  {_fmt(point['recall']):>6}  "
+            f"{_fmt(point['false_positive_rate']):>6}  {_fmt(point['false_negative_rate']):>6}  {_fmt(point['f1']):>6}"
+        )
+    print()
+    _print_errors("False positives (clean flagged as spam)", report["false_positives"], show_text)
+    _print_errors("False negatives (spam not flagged)", report["false_negatives"], show_text)
+    print(dataset["note"])
+
+
+def _print_errors(title: str, rows: list[dict[str, Any]], show_text: bool) -> None:
+    print(f"{title}: {len(rows)}")
+    for row in rows:
+        label = f"row {row['row']}" + (f" (id {row['id']})" if row["id"] else "")
+        print(f"  {label}  score {row['score']:.3f}")
+        if show_text:
+            print(f"    text: {row['text']!r}")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())