spamkit-api 0.1.0__py3-none-any.whl

spamkit/verdict.py

@@ -0,0 +1,46 @@
+"""Threshold policy: maps a raw spam score to clean, review, or spam."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from enum import Enum
+
+
+class Verdict(str, Enum):
+    CLEAN = "clean"
+    REVIEW = "review"
+    SPAM = "spam"
+
+
+@dataclass(frozen=True)
+class Thresholds:
+    """Scores below ``clean`` are clean, scores above ``spam`` are spam, and everything between is review.
+
+    The defaults route every submission to review. A deployment has to choose thresholds
+    deliberately, based on its own evaluation results, before any automatic action is safe.
+    """
+
+    clean: float = 0.0
+    spam: float = 1.0
+
+    def __post_init__(self) -> None:
+        for name, value in (("clean", self.clean), ("spam", self.spam)):
+            if not 0.0 <= value <= 1.0:
+                raise ValueError(f"{name} threshold must be between 0 and 1")
+        if not self.clean < self.spam:
+            raise ValueError("clean threshold must be lower than spam threshold")
+
+    def verdict_for(self, score: float) -> Verdict:
+        if score < self.clean:
+            return Verdict.CLEAN
+        if score > self.spam:
+            return Verdict.SPAM
+        return Verdict.REVIEW
+
+    @property
+    def version(self) -> str:
+        """Short, deterministic fingerprint of the threshold values. Changes whenever they change."""
+        canonical = json.dumps({"clean": self.clean, "spam": self.spam}, sort_keys=True)
+        return hashlib.sha256(canonical.encode()).hexdigest()[:12]

spamkit_api-0.1.0.dist-info/METADATA

@@ -0,0 +1,245 @@
+Metadata-Version: 2.5
+Name: spamkit-api
+Version: 0.1.0
+Summary: Self-hostable spam-scoring service and client for user-submitted text.
+Project-URL: Homepage, https://spamkit.pages.dev
+License-Expression: AGPL-3.0-only
+License-File: LICENSE
+Keywords: classifier,moderation,self-hosted,spam
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.6
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Provides-Extra: server
+Requires-Dist: fastapi>=0.110; extra == 'server'
+Requires-Dist: huggingface-hub>=0.22; extra == 'server'
+Requires-Dist: torch>=2.2; extra == 'server'
+Requires-Dist: transformers<5,>=4.48; extra == 'server'
+Requires-Dist: uvicorn>=0.29; extra == 'server'
+Description-Content-Type: text/markdown
+
+# Spamkit
+
+Spam scoring for user-submitted text, designed to run in your own environment.
+
+Spamkit is a small HTTP service around the [spamton](https://huggingface.co/tiagozip/spamton) text classifier, plus Python and TypeScript clients and an offline evaluation CLI. It returns a score and a three-way verdict (`clean`, `review`, `spam`). Your application decides what to do with each verdict. Spamkit never deletes, blocks, or edits content.
+
+> **Status: v0.1 prototype.** The product hypothesis is in [PRD.md](PRD.md). Model quality has **not** been validated on a labeled dataset for any target segment, so no accuracy or false-positive claim is made here. Use `spamkit evaluate` on your own labeled examples before you trust any threshold.
+
+## Quick start
+
+### Docker (recommended)
+
+```bash
+docker compose up --build
+curl -s http://127.0.0.1:8080/readyz
+curl -s -X POST http://127.0.0.1:8080/v1/score \
+  -H 'content-type: application/json' \
+  -d '{"text":"I can improve your Shopify SEO. Reply for a free audit.","id":"submission_123"}'
+```
+
+The image pins the model revision and fetches the model at build time, so the running container makes no network calls. The Dockerfile and compose file have not been built in this repository's CI yet. Verify with `docker compose build` before you rely on them.
+
+### Local (Python 3.10+)
+
+```bash
+python -m venv .venv && . .venv/bin/activate
+pip install --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple torch
+pip install "spamkit-api[server]"  # or: pip install -e ".[server]" from a checkout
+spamkit fetch-model --dest models/spamton     # downloads spamton at the pinned revision
+spamkit serve                                  # 127.0.0.1:8080
+```
+
+## API
+
+Base path `/v1`. All request and response bodies are JSON.
+
+### `POST /v1/score`
+
+```json
+{ "text": "I can improve your Shopify SEO. Reply for a free audit.", "id": "submission_123", "context": "store_contact_form" }
+```
+
+```json
+{
+  "id": "submission_123",
+  "score": 0.5499,
+  "verdict": "review",
+  "model": "spamton",
+  "model_version": "7aa94d049d24bbc302c5a69e63cafe6c44d49bc1",
+  "latency_ms": 18,
+  "truncated": false
+}
+```
+
+- `text` is required, must be non-empty UTF-8, and is limited to `SPAMKIT_MAX_TEXT_BYTES` (default 16,384 bytes).
+- `id` is opaque and echoed back. It is never logged or used as a metric label.
+- `context` is a short label (max 64 characters). It is logged and counted, and it does not condition the classifier.
+- `score` is the raw spam-class output in [0, 1]. **It is not a calibrated probability.** Calibration has not been measured.
+- `truncated` is `true` when the text exceeded the model's 512-token window. The model then scores only the leading part. The service first clips to 6,000 characters, matching the model author's reference preprocessing.
+
+### `POST /v1/score/batch`
+
+```json
+{ "items": [{ "text": "...", "id": "a" }, { "text": "", "id": "b" }] }
+```
+
+Returns `results` in the same order as `items`. Each entry is `{"ok": true, "result": {...}}` or `{"ok": false, "id": ..., "error": {...}}`, so one bad item does not discard the others. Up to `SPAMKIT_MAX_BATCH_ITEMS` items (default 100).
+
+### Verdicts
+
+| Score | Verdict |
+|---|---|
+| `score < clean` | `clean` |
+| `score > spam` | `spam` |
+| otherwise | `review` |
+
+Defaults are `clean=0.0` and `spam=1.0`, so every submission is `review` until you choose thresholds from your own evaluation. Thresholds are returned by `GET /v1/info` with a `threshold_config_version` fingerprint.
+
+### Errors
+
+Every error uses one envelope, and the message never includes submitted text:
+
+```json
+{ "error": { "code": "input_too_large", "message": "text must be at most 16384 bytes of UTF-8.", "request_id": "..." } }
+```
+
+| HTTP | `code` | Meaning |
+|---|---|---|
+| 400 | `invalid_request` | Malformed JSON, missing or empty `text`, wrong field types |
+| 401 | `unauthorized` | Bearer token required and missing or wrong (only when `SPAMKIT_API_TOKEN` is set) |
+| 413 | `input_too_large` | Body, text, or batch item count over the limit |
+| 422 | `unsupported_input` | Text contains NUL characters |
+| 429 | `rate_limited` | Too many requests in flight (`SPAMKIT_MAX_INFLIGHT`). Retry after `Retry-After`. |
+| 503 | `model_not_ready` | Model failed to load or is still loading |
+| 500 | `internal_error` | Unexpected failure. Details are logged by type only. |
+
+### Operations
+
+| Endpoint | Purpose |
+|---|---|
+| `GET /healthz` | Liveness. Always 200 while the process is up. |
+| `GET /readyz` | 200 only when the model is loaded. Returns the model version. |
+| `GET /v1/info` | Model ID, version, token limit, thresholds, and size limits. No secrets. |
+| `GET /metrics` | Prometheus text: request counts, verdict counts, error codes, latency histogram. |
+
+## Evaluate on your own data
+
+Put your labeled examples in a CSV or JSONL file with a `text` column and a `label` column (`spam` or `clean`). An optional `id` column is echoed in the output.
+
+```bash
+spamkit evaluate ./samples.csv --threshold 0.82 --output report.json
+```
+
+The command runs the model locally. It does not upload the file. By default it prints row numbers only (file line numbers, so they match your editor). Pass `--show-text` to print the text of false positives and false negatives. The output includes:
+
+- dataset count and label balance,
+- confusion matrix, precision, recall, specificity, FPR, FNR, and F1,
+- a threshold sweep from 0.05 to 0.95 (or `--sweep ...`),
+- the model version and software version,
+- the note "Metrics describe only the supplied dataset."
+
+Undefined ratios (for example precision when nothing was predicted as spam) print as `n/a`.
+
+## SDKs
+
+Both SDKs use the same field names as the REST API, return the same verdicts, and map errors to the same codes. Both are tested against [`contract/fixtures.json`](contract/fixtures.json).
+
+**Python** (`pip install -e .` from this repo until the package is published; the base install has no torch):
+
+```python
+from spamkit import SpamkitClient, SpamkitError
+
+client = SpamkitClient("http://127.0.0.1:8080", timeout=2.0, max_retries=1)
+try:
+    result = client.score("Win a free cruise!!!", id="submission_123")
+    print(result.verdict, result.score)
+except SpamkitError as error:
+    print(error.code, error.status, error.request_id)
+```
+
+**TypeScript** (`sdk/typescript`, Node 18.17+):
+
+```ts
+import { SpamkitClient, SpamkitError } from "spamkit";
+
+const client = new SpamkitClient({ baseUrl: "http://127.0.0.1:8080", timeoutMs: 2000, maxRetries: 1 });
+try {
+  const result = await client.score({ text: "Win a free cruise!!!", id: "submission_123" });
+  console.log(result.verdict, result.score);
+} catch (error) {
+  if (error instanceof SpamkitError) console.error(error.code, error.status, error.requestId);
+}
+```
+
+Retries are off by default. When you turn them on, they cover transport errors, timeouts, 429, and 5xx. Scoring is read-only, so a retry cannot duplicate side effects. Samples for a Node server and a FastAPI app are in [`examples/`](examples/).
+
+## Configuration
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `SPAMKIT_MODEL_DIR` | `models/spamton` | Directory from `spamkit fetch-model` |
+| `SPAMKIT_HOST` | `127.0.0.1` | Bind address. The container sets `0.0.0.0`. |
+| `SPAMKIT_PORT` | `8080` | Listen port |
+| `SPAMKIT_CLEAN_THRESHOLD` | `0.0` | Scores below this are `clean` |
+| `SPAMKIT_SPAM_THRESHOLD` | `1.0` | Scores above this are `spam` |
+| `SPAMKIT_MAX_TEXT_BYTES` | `16384` | Per-text byte limit |
+| `SPAMKIT_MAX_BODY_BYTES` | `1048576` | Request body limit |
+| `SPAMKIT_MAX_BATCH_ITEMS` | `100` | Batch item limit |
+| `SPAMKIT_MAX_INFLIGHT` | `32` | Requests in flight (queued or running) before 429 |
+| `SPAMKIT_API_TOKEN` | unset | When set, `/v1/*` and `/metrics` require `Authorization: Bearer <token>` |
+
+Run one worker per process. Each worker loads its own copy of the model. `spamkit serve` already runs one.
+
+## Privacy and data handling
+
+This describes what the code does today.
+
+- Submitted text is held in process memory for the duration of scoring. It is not written to disk, a database, or a cache.
+- Logs are JSON lines with request ID, route, status, latency, verdict, score band (for example `0.1-0.2`), model version, error code, and `context`. They never include `text` or `id`. The test suite checks this.
+- Metrics contain only fixed labels: route, status, verdict, and error code.
+- The `evaluate` command reads your file locally and writes only aggregate metrics and the error-row list. Text is printed only with `--show-text`.
+- There is no telemetry and no outbound network call at runtime.
+- Local deployment reduces data exposure. It does not make a deployment GDPR-compliant or "zero data processing". Your own operations still matter, including TLS, access control, and log retention.
+
+## Known gaps
+
+- **No measured accuracy.** No labeled evaluation set has been run for any segment. PRD section 11 lists the release gates. Until they pass, do not present the service as production-ready.
+- **Score is uncalibrated.** Do not treat it as a probability.
+- **English only.** The model's training data is English.
+- **Text only.** No image, link, sender, or account signals.
+- **Rate limiting is only admission control.** `SPAMKIT_MAX_INFLIGHT` returns 429 when saturated. There is no per-client limit. Apply one at the reverse proxy.
+- **Body size is enforced while streaming**, but there is no per-client limit. Put a proxy-level limit in front of the service for untrusted traffic.
+- **No debug text logging, no auth beyond one shared bearer token, no multi-tenant support.**
+- **Performance was measured only on Apple Silicon, not on the x86 target in the PRD.** On an M-series Mac with the pinned model, warm batch-size-1 latency was p50 9 ms and p95 24 ms over 30 runs. Startup includes one warm-up pass. Memory and concurrency have not been benchmarked on target hardware.
+- **Batch latency** reported per item is the latency of the whole batch scoring call.
+
+## Model and license
+
+- Model: `tiagozip/spamton` at revision `7aa94d049d24bbc302c5a69e63cafe6c44d49bc1`, a 32M-parameter ModernBERT classifier fine-tuned from `jhu-clsp/ettin-encoder-32m`.
+- License: **AGPL-3.0** for the model. The `LICENSE` and `NOTICE` files are kept in the model directory and in the image. See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md).
+- PRD section 14 flags that AGPL obligations for hosted or redistributed use have not been reviewed by counsel. Resolve that before any commercial release.
+- The license for this repository's own code has not been chosen.
+
+## Development
+
+```bash
+# Python: unit tests run without the model; model tests run when models/spamton exists.
+pip install -e ".[server,dev]" && pytest
+
+# TypeScript SDK
+cd sdk/typescript && npm ci && npm run typecheck && npm test
+```
+
+Layout:
+
+```
+src/spamkit/        server (app, classifier, settings), evaluation, client, CLI
+sdk/typescript/     spamkit
+contract/           shared request/response fixtures used by both SDK suites
+examples/           Node and FastAPI integration samples
+tests/              pytest suite
+Dockerfile          single-node image with the model pinned at build time
+```

spamkit_api-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+spamkit/__init__.py,sha256=Bh6bJeaC_7OUTzqjsJIbi5I3ofgab-OQqrA-Lem9_zk,616
+spamkit/app.py,sha256=DFtt75YAf6RMyxiBV3epo56BoTPzoru4OxLtB8kNrv8,12822
+spamkit/classifier.py,sha256=T8Lftx1hbiwbjmYyEdTwlYo4PID5gH8QSFk8o-O272w,4638
+spamkit/cli.py,sha256=RpH69OUjHOYhu3gixvbZk8kpGK24MGOXNFacR4NOLIA,7580
+spamkit/client.py,sha256=Z-Vk1quiC6jWpESOv0yPwYs1Bll1UX3BKFQypNJgSHo,6162
+spamkit/errors.py,sha256=eHa_0z4MeAw2cjykKbxDm1qObpkiA0MhwhhR_WOv1TI,1168
+spamkit/evaluation.py,sha256=90Lk91k28YHJCeSvvGJeN4Gu8zhCl1hyLniTYssIAKw,6473
+spamkit/observability.py,sha256=ERTVd7OdsO4WAjwAfnT8O4BqyA5529TIybTt6Iz3ido,3432
+spamkit/schemas.py,sha256=d9AR4VrJmGvR6zy5sHR0e3vzIGqI6yoHaNjVlEDk-Nk,3205
+spamkit/settings.py,sha256=rxa2rp_EGcJo54x7R73EJGn3c2AcieJH1Ig4xs1Dlkc,2220
+spamkit/verdict.py,sha256=dDq8g-Fmd36ttgSBMDTG3mauVxN61QD6QbaAXx4Lido,1530
+spamkit_api-0.1.0.dist-info/METADATA,sha256=3V2_r-JO_9dIuYhZv01JNMb9I4BIrv2S_DxioC-4FMY,12006
+spamkit_api-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+spamkit_api-0.1.0.dist-info/entry_points.txt,sha256=iroCCVAdDu_JlOJJmml_xcgePgr3_ITEPhAb-Dlo9Ys,45
+spamkit_api-0.1.0.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+spamkit_api-0.1.0.dist-info/RECORD,,

spamkit_api-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

spamkit_api-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+spamkit = spamkit.cli:main