freesolo-flash-dev 0.2.25__py3-none-any.whl

flash/server/auth.py

@@ -0,0 +1,263 @@
+"""Bearer auth for the managed control plane.
+
+User authentication is freesolo API keys only — there is no native key system. A bearer
+token equal to the operator's shared ``FREESOLO_INTERNAL_KEY`` resolves to the service
+identity; any other token is verified against the freesolo backend and (on success)
+resolved to a per-token user identity. A failed/unreachable verify returns False (the key
+is treated as unverified), so a backend outage never admits an unverified key.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+import urllib.error
+import urllib.request
+from typing import Any
+
+from . import db
+
+# Operators set this to the shared freesolo internal key; a bearer token equal to it
+# authenticates as the service identity (see db.ensure_internal_key).
+INTERNAL_KEY_ENV = "FREESOLO_INTERNAL_KEY"
+
+# Freesolo USER-key acceptance: a user who `flash login`s with a freesolo API key sends it as
+# the bearer to this control plane. Any non-internal token is verified against the freesolo
+# backend and (on success) resolved to a per-token identity.
+FREESOLO_BASE_URL_ENV = "FREESOLO_BASE_URL"
+DEFAULT_FREESOLO_BASE_URL = "https://api.freesolo.co"
+_VERIFY_TIMEOUT_S = 5.0
+_VERIFY_CACHE_TTL_S = 300.0  # short TTL so it isn't a backend round-trip per request
+# Negative verdicts get a much SHORTER TTL than positives. The freesolo verify endpoint
+# returns 401 not only for a genuinely-bad key but also when the backend converts an
+# auth-LOOKUP infra exception (authenticate_api_key failure) into a 401 — a transient outage.
+# Caching such a negative for the full 300s would lock out an otherwise-valid key for 5
+# minutes after the backend recovers. A short negative TTL keeps persistent bad tokens
+# rate-limited (~30s, so they don't hammer the backend) while letting a transient 401 clear
+# quickly. Positives keep the long TTL.
+_VERIFY_CACHE_NEG_TTL_S = 30.0
+# Upper bound on a bearer token we'll cache/verify. Real freesolo API keys are short; an
+# arbitrarily long bearer is rejected up front so it can't bloat _verify_cache (keyed by the
+# raw token) or produce an oversized outbound Authorization header.
+_MAX_TOKEN_LEN = 256
+
+# In-process verify cache: token -> (verified_bool, expires_at). Caches positives AND
+# negatives so a burst of requests for the same token hits the backend at most once per TTL.
+# Bounded: pruned of expired entries on every write and capped at _VERIFY_CACHE_MAX so a
+# stream of unique bearer tokens can't grow it without bound (each token is a distinct key).
+_verify_cache: dict[str, tuple[bool, float]] = {}
+_identity_cache: dict[str, tuple[dict[str, Any], float]] = {}
+_verify_cache_lock = threading.Lock()
+_VERIFY_CACHE_MAX = 1024
+
+
+def _prune_verify_cache_locked(now: float) -> None:
+    """Drop expired entries, then cap the cache size (oldest-expiry first).
+
+    Caller must hold ``_verify_cache_lock``. Keeps the cache from growing unbounded as
+    many distinct bearer tokens are verified over time.
+    """
+    for tok in [t for t, (_v, exp) in _verify_cache.items() if exp <= now]:
+        del _verify_cache[tok]
+        _identity_cache.pop(tok, None)
+    for tok in [t for t, (_v, exp) in _identity_cache.items() if exp <= now]:
+        del _identity_cache[tok]
+    if len(_verify_cache) >= _VERIFY_CACHE_MAX:
+        # Still over the cap after dropping expired entries: evict the soonest-to-expire
+        # (oldest) entries until we're back under the cap.
+        for tok, _exp in sorted(_verify_cache.items(), key=lambda kv: kv[1][1])[
+            : len(_verify_cache) - _VERIFY_CACHE_MAX + 1
+        ]:
+            del _verify_cache[tok]
+            _identity_cache.pop(tok, None)
+
+
+def _freesolo_key_prefix(token: str) -> str:
+    """Non-secret preview of a Freesolo API key, matching the dashboard-style public prefix."""
+    parts = token.split("_", 2)
+    if len(parts) >= 2 and parts[0] == "fslo" and parts[1]:
+        return f"fslo_{parts[1]}"
+    return f"fslo_{db.hash_key(token)[:12]}"
+
+
+def _external_key_prefix(token: str, identity: dict[str, Any]) -> str:
+    prefix = _str_field(identity.get("key_prefix"))
+    if prefix and prefix.startswith("fslo_"):
+        return prefix
+    if not identity and token.startswith("fslo-user-"):
+        return "freesolo"
+    return _freesolo_key_prefix(token)
+
+
+def _str_field(value: Any) -> str | None:
+    if isinstance(value, str) and value.strip():
+        return value.strip()
+    return None
+
+
+def _identity_from_verify_body(raw: bytes) -> dict[str, Any]:
+    """Extract optional identity fields from the freesolo verify response.
+
+    The backend historically returned only ``{"ok": true}``. This parser is deliberately
+    tolerant so Flash surfaces real fields when the backend includes them.
+    """
+    if not raw:
+        return {}
+    try:
+        body = json.loads(raw)
+    except (TypeError, ValueError):
+        return {}
+    if not isinstance(body, dict):
+        return {}
+
+    user = body.get("user") if isinstance(body.get("user"), dict) else {}
+    org = body.get("org") if isinstance(body.get("org"), dict) else {}
+    api_key = body.get("api_key") if isinstance(body.get("api_key"), dict) else {}
+
+    fields = {
+        "email": _str_field(body.get("email")) or _str_field(user.get("email")),
+        "user_id": (
+            _str_field(body.get("user_id"))
+            or _str_field(body.get("created_by"))
+            or _str_field(user.get("id"))
+        ),
+        "org_id": _str_field(body.get("org_id")) or _str_field(org.get("id")),
+        "api_key_id": (
+            _str_field(body.get("api_key_id"))
+            or _str_field(body.get("key_id"))
+            or _str_field(api_key.get("id"))
+        ),
+        "key_prefix": _str_field(body.get("key_prefix")) or _str_field(api_key.get("key_prefix")),
+        "training_agent_job_id": _str_field(body.get("training_agent_job_id")),
+        "project_id": _str_field(body.get("project_id")),
+    }
+    return {k: v for k, v in fields.items() if v}
+
+
+def _response_body(resp: Any) -> bytes:
+    read = getattr(resp, "read", None)
+    if not callable(read):
+        return b""
+    data = read()
+    if isinstance(data, bytes):
+        return data
+    if isinstance(data, str):
+        return data.encode()
+    return b""
+
+
+def _cached_identity(token: str) -> dict[str, Any]:
+    now = time.time()
+    with _verify_cache_lock:
+        cached = _identity_cache.get(token)
+        if cached is not None and cached[1] > now:
+            return dict(cached[0])
+    return {}
+
+
+def _external_row(row: dict, token: str, identity: dict[str, Any]) -> dict:
+    out = dict(row)
+    out["auth_kind"] = "freesolo_api_key"
+    out["key_prefix"] = _external_key_prefix(token, identity)
+    if identity.get("email"):
+        out["email"] = identity["email"]
+    for key in ("user_id", "org_id", "api_key_id", "training_agent_job_id", "project_id"):
+        if identity.get(key):
+            out[key] = identity[key]
+    return out
+
+
+def _identity_email(identity: dict[str, Any]) -> str:
+    email = str(identity.get("email") or "").strip()
+    return email if "@" in email else ""
+
+
+def freesolo_base_url() -> str:
+    """The freesolo backend base URL (``FREESOLO_BASE_URL`` env, else the default), trailing
+    slash trimmed. Shared by auth verify and the billing client."""
+    return (os.environ.get(FREESOLO_BASE_URL_ENV) or DEFAULT_FREESOLO_BASE_URL).rstrip("/")
+
+
+def _freesolo_verify(token: str) -> bool:
+    """Verify a token against the freesolo backend (cached, short TTL, network errors = False).
+
+    Never raises — a swallowed network/HTTP error is treated as "not authenticated" (returns
+    False), never a 500."""
+    # Reject obviously-invalid oversized tokens before they touch the cache or the network.
+    if not token or len(token) > _MAX_TOKEN_LEN:
+        return False
+    now = time.time()
+    with _verify_cache_lock:
+        cached = _verify_cache.get(token)
+        if cached is not None and cached[1] > now:
+            return cached[0]
+    url = f"{freesolo_base_url()}/api/auth/verify"
+    req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
+    identity: dict[str, Any] = {}
+    try:
+        with urllib.request.urlopen(req, timeout=_VERIFY_TIMEOUT_S) as resp:
+            verified = resp.status == 200
+            if verified:
+                identity = _identity_from_verify_body(_response_body(resp))
+    except urllib.error.HTTPError as exc:
+        # Only a DEFINITIVE rejection (4xx other than 429) is a verdict worth caching as a bad
+        # key. A 5xx or 429 is a transient backend hiccup — treat it like a network error
+        # (return False WITHOUT caching) so a valid key isn't locked out for the whole TTL
+        # while the backend is briefly unhealthy.
+        if exc.code >= 500 or exc.code == 429:
+            return False
+        verified = False
+    except (urllib.error.URLError, OSError, ValueError):
+        # A TRANSIENT network/connection error is NOT a verdict: don't cache it, so a valid
+        # key isn't locked out for the whole TTL after the backend recovers.
+        return False
+    with _verify_cache_lock:
+        # Prune expired entries and cap the size before inserting so unbounded distinct
+        # tokens can't grow the cache.
+        _prune_verify_cache_locked(now)
+        # Pick the TTL by verdict: positives last the full TTL; a negative (which may be a
+        # transient backend 401 rather than a real rejection) expires quickly so a valid key
+        # isn't locked out for 5 minutes after the backend recovers.
+        ttl = _VERIFY_CACHE_TTL_S if verified else _VERIFY_CACHE_NEG_TTL_S
+        _verify_cache[token] = (verified, now + ttl)
+        if verified:
+            _identity_cache[token] = (identity, now + ttl)
+        else:
+            _identity_cache.pop(token, None)
+    return verified
+
+
+def authenticate(authorization: str | None) -> dict | None:
+    """Resolve an ``Authorization: Bearer ...`` header to a key row.
+
+    Freesolo keys are the only user auth. When the operator has configured
+    ``FREESOLO_INTERNAL_KEY``, that shared internal key resolves to a single service
+    identity. Any other token is verified against the freesolo backend and (on success)
+    resolved to a per-token user identity so a user who ``flash login``s with their freesolo
+    key can drive the control plane. A token that can't be verified (bad key, or the backend
+    is unreachable) is treated as unverified -> authenticate returns None."""
+    if not authorization or not authorization.startswith("Bearer "):
+        return None
+    token = authorization.removeprefix("Bearer ").strip()
+    internal = os.environ.get(INTERNAL_KEY_ENV)
+    if internal and token == internal:
+        row = db.lookup_key(token) or db.ensure_internal_key(token)
+        out = dict(row)
+        out["auth_kind"] = "internal"
+        return out
+    # Any non-internal token is a freesolo USER key: verify it against the freesolo backend.
+    if _freesolo_verify(token):
+        # A verified freesolo key gets its own per-token run-ownership identity.
+        identity = _cached_identity(token)
+        email = _identity_email(identity)
+        if not email:
+            return None
+        row = db.lookup_key(token) or db.ensure_external_key(
+            token,
+            key_prefix=_external_key_prefix(token, identity),
+            email=email,
+        )
+        return _external_row(row, token, identity) if row is not None else None
+    return None

flash/server/billing.py

@@ -0,0 +1,124 @@
+"""Charge completed Flash training runs to the freesolo backend.
+
+The control plane records the submitting org id when a run is accepted, then POSTs the final
+``RunStatus.cost_usd`` after the run reaches ``done``. The call is authenticated with the
+operator internal key, so Flash never persists a user's freesolo API key while training runs.
+Tests stub the network boundary directly."""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from decimal import ROUND_HALF_UP, Decimal
+
+from .auth import freesolo_base_url
+
+# A charge does more than a verify; allow a bit more than auth's 5s but stay bounded.
+_CHARGE_TIMEOUT_S = 10.0
+_COMPLETION_CHARGE_PATH = "/api/billing/training-usage/internal"
+
+
+class BillingError(Exception):
+    """A run charge that didn't succeed. ``status_code`` (402 insufficient balance, 503 backend
+    unreachable) is surfaced to the client with ``detail``."""
+
+    def __init__(self, status_code: int, detail: str) -> None:
+        super().__init__(detail)
+        self.status_code = status_code
+        self.detail = detail
+
+
+def _cents(usd: float) -> int:
+    """Whole cents for a USD amount, round-HALF-UP (not Python's banker's rounding, which would
+    undercharge a half-cent tie), never negative."""
+    cents = Decimal(str(usd)).scaleb(2).quantize(Decimal("1"), rounding=ROUND_HALF_UP)
+    return max(0, int(cents))
+
+
+def _http_reason(exc: urllib.error.HTTPError) -> str:
+    """The backend's status reason phrase (``.reason``/``.msg``), else the bare code."""
+    reason = getattr(exc, "reason", None) or getattr(exc, "msg", None)
+    return str(reason).strip() if reason else str(exc.code)
+
+
+def _http_error_detail(exc: urllib.error.HTTPError) -> str:
+    """Clean message from the backend's ``{"detail": {"error", "code"}}`` JSON error body, else
+    the status reason (never a bare code)."""
+
+    def _fallback() -> str:
+        return f"billing failed ({exc.code} {_http_reason(exc)})"
+
+    try:
+        body = json.loads(exc.read() or b"{}")
+    except (ValueError, OSError):
+        return _fallback()
+    detail = body.get("detail") if isinstance(body, dict) else None
+    if isinstance(detail, dict):
+        return str(detail.get("error") or detail.get("code") or _fallback())
+    if isinstance(detail, str) and detail:
+        return detail
+    return _fallback()
+
+
+def _post_billing(*, token: str, path: str, body: dict) -> dict:
+    """POST a JSON body to the backend billing ``path`` and return the parsed response.
+
+    Raises ``BillingError`` (the route's status + a clean detail) on a non-2xx, and ``503``
+    when the service is unreachable -- the same translation the charge and its reversal share.
+    """
+    url = f"{freesolo_base_url()}{path}"
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(body).encode("utf-8"),
+        method="POST",
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+        },
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=_CHARGE_TIMEOUT_S) as resp:
+            raw = resp.read()
+    except urllib.error.HTTPError as exc:
+        raise BillingError(exc.code, _http_error_detail(exc)) from exc
+    except (urllib.error.URLError, OSError) as exc:
+        raise BillingError(503, f"billing service unavailable: {exc}") from exc
+    try:
+        return json.loads(raw or b"{}")
+    except ValueError as exc:
+        # The backend responded but the body isn't JSON -- a bad gateway, not an outage.
+        raise BillingError(502, f"billing service returned an invalid response: {exc}") from exc
+
+
+def charge_completed_run(*, internal_key: str, status) -> dict:
+    """Charge one completed external run using its persisted non-secret billing context.
+
+    The backend route is idempotent by ``runId``. Raises ``BillingError`` on a non-2xx or
+    unreachable backend; callers should record that billing failed without changing the run's
+    terminal training state.
+    """
+    context = status.billing_context if isinstance(status.billing_context, dict) else {}
+    org_id = str(context.get("org_id") or "").strip()
+    if not org_id:
+        raise BillingError(400, "missing billing org id for completed training run")
+    spec = status.spec or {}
+    remote = status.remote or {}
+    gpu = remote.get("allocated_gpu") or (spec.get("gpu") or {}).get("type")
+    provider = remote.get("provider")
+    total_usd = float(status.cost_usd or 0.0)
+    body = {
+        "orgId": org_id,
+        "runId": status.run_id,
+        "costCents": _cents(total_usd),
+        "gpu": gpu,
+        "provider": provider,
+        "method": spec.get("algorithm"),
+        "model": spec.get("model"),
+        "estimate": {
+            "totalUsd": total_usd,
+            "costBasis": "final",
+            "costSource": "run_status.cost_usd",
+        },
+    }
+    return _post_billing(token=internal_key, path=_COMPLETION_CHARGE_PATH, body=body)

flash/server/checkpoints.py

@@ -0,0 +1,110 @@
+"""Mirror a run's deployable RL checkpoints to the freesolo backend.
+
+The worker streams each step's LoRA adapter to the run's HF repo; HF is the source of truth
+for what's deployable. This module persists that list to the backend's ``run_checkpoints``
+store so the dashboard/SDK can enumerate a run's checkpoints without crawling HF, and so a
+cancelled run's checkpoints survive in one queryable place.
+
+Like ``flash.server.billing``, the POST is authenticated with the operator INTERNAL key (the
+control plane never persists a user's freesolo key) and carries the org id from the run's
+non-secret billing context. Unlike billing, checkpoint persistence is STRICTLY best-effort:
+a failure here must never disturb a run or a deploy, so the public entry point swallows
+everything."""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+
+from flash.runner.checkpoints import list_checkpoints
+from flash.spec import JobSpec
+
+from .auth import INTERNAL_KEY_ENV, freesolo_base_url
+
+_TIMEOUT_S = 10.0
+_RECORD_PATH = "/api/runs/internal/checkpoints"
+
+
+def _post_checkpoints(*, token: str, body: dict) -> dict:
+    """POST the checkpoint batch to the backend; raise on any non-2xx/unreachable.
+
+    Callers in this module always wrap this in a best-effort guard — the raise exists so the
+    one network boundary is easy for tests to stub/assert."""
+    url = f"{freesolo_base_url()}{_RECORD_PATH}"
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(body).encode("utf-8"),
+        method="POST",
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+        },
+    )
+    with urllib.request.urlopen(req, timeout=_TIMEOUT_S) as resp:
+        raw = resp.read()
+    try:
+        return json.loads(raw or b"{}")
+    except ValueError:
+        return {}
+
+
+def register_run_checkpoints(*, internal_key: str, status, checkpoints: list[dict]) -> dict:
+    """Upsert ``checkpoints`` for one run into the backend store (idempotent by run_id+step).
+
+    Pulls the org id from the run's persisted billing context (same source as billing). Raises
+    ``ValueError`` when there's nothing to record or no org id; raises ``urllib`` errors through
+    on a backend failure — ``register_checkpoints_best_effort`` is the guarded wrapper most
+    callers use."""
+    if not checkpoints:
+        raise ValueError("no checkpoints to record")
+    context = status.billing_context if isinstance(status.billing_context, dict) else {}
+    org_id = str(context.get("org_id") or "").strip()
+    if not org_id:
+        raise ValueError("missing org id for run checkpoints")
+    spec = status.spec or {}
+    first = checkpoints[0]
+    body = {
+        "orgId": org_id,
+        "runId": status.run_id,
+        "baseModel": spec.get("model"),
+        "repoId": first.get("repo_id"),
+        "repoType": first.get("repo_type", "dataset"),
+        "checkpoints": [
+            {"step": c["step"], "subfolder": c["subfolder"]} for c in checkpoints
+        ],
+    }
+    return _post_checkpoints(token=internal_key, body=body)
+
+
+def register_checkpoints_best_effort(status, *, log=None) -> int:
+    """List ``status``'s deployable checkpoints from HF and mirror them to the backend.
+
+    Returns the number of checkpoints submitted (0 if none, or if persistence was skipped /
+    failed). Never raises: the HF copy remains the source of truth, so a persistence miss only
+    costs the convenience of a DB-backed listing — not correctness."""
+
+    def _log(msg: str) -> None:
+        print(msg, file=log, flush=True) if log is not None else print(msg)
+
+    internal_key = os.environ.get(INTERNAL_KEY_ENV, "").strip()
+    if not internal_key:
+        return 0  # local/dev control plane: HF still has the checkpoints
+    try:
+        spec = JobSpec.from_dict(status.spec)
+    except Exception as exc:
+        _log(f"[ckpt] register skipped ({status.run_id}): bad spec: {exc}")
+        return 0
+    checkpoints = list_checkpoints(spec)
+    if not checkpoints:
+        return 0
+    try:
+        register_run_checkpoints(
+            internal_key=internal_key, status=status, checkpoints=checkpoints
+        )
+    except (ValueError, urllib.error.URLError, urllib.error.HTTPError, OSError) as exc:
+        _log(f"[ckpt] backend register warn ({status.run_id}): {exc}")
+        return 0
+    _log(f"[ckpt] registered {len(checkpoints)} checkpoint(s) for {status.run_id}")
+    return len(checkpoints)

flash/server/db.py

@@ -0,0 +1,160 @@
+"""SQLite store for the managed control plane: API keys + run ownership.
+
+Run *state* stays in the runner's JSON files (``runner.RUNS_DIR``) — the
+battle-tested submit/attach/cancel paths all read those. This database is only the
+key registry and the run -> key ownership index that makes the server multi-tenant.
+
+Connections are opened per operation (cheap for SQLite, avoids cross-thread state;
+the runner runs jobs in daemon threads inside the same process).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import sqlite3
+import time
+from pathlib import Path
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS api_keys (
+  id           INTEGER PRIMARY KEY AUTOINCREMENT,
+  key_hash     TEXT NOT NULL UNIQUE,
+  key_prefix   TEXT NOT NULL,
+  email        TEXT,
+  created_at   REAL NOT NULL,
+  last_used_at REAL,
+  disabled     INTEGER NOT NULL DEFAULT 0
+);
+CREATE TABLE IF NOT EXISTS runs (
+  run_id     TEXT PRIMARY KEY,
+  key_id     INTEGER NOT NULL REFERENCES api_keys(id),
+  kind       TEXT NOT NULL DEFAULT 'train',
+  created_at REAL NOT NULL
+);
+CREATE INDEX IF NOT EXISTS runs_key_idx ON runs(key_id);
+"""
+
+
+# Fixed location for the keys/run-ownership SQLite DB (not operator-configurable). Tests
+# point it elsewhere with monkeypatch.setattr(db, "DB_PATH", tmp).
+DB_PATH = str(Path.home() / ".flash" / "server.db")
+
+
+def db_path() -> str:
+    return DB_PATH
+
+
+def _connect() -> sqlite3.Connection:
+    path = db_path()
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(path, timeout=30.0)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA foreign_keys=ON")
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.executescript(_SCHEMA)
+    return conn
+
+
+def hash_key(api_key: str) -> str:
+    # API keys are 192-bit random tokens (secrets.token_hex(24)), not passwords:
+    # brute-forcing the keyspace is infeasible, so an unsalted fast hash is the
+    # standard at-rest form and keeps O(1) lookup by hash. (CodeQL's
+    # password-hashing rule does not apply to high-entropy machine tokens.)
+    return hashlib.sha256(api_key.encode()).hexdigest()
+
+
+def ensure_internal_key(api_key: str) -> dict:
+    """Provision a row for the shared freesolo internal/service key (idempotent).
+
+    The freesolo platform/SDK authenticate to the control plane with the same
+    ``FREESOLO_INTERNAL_KEY`` they already hold. Backing it with a real row
+    (inserted once, by hash) means run ownership and
+    the runs.key_id foreign key work exactly as for a normal key — all
+    internal-key runs share this single service identity (no per-user isolation;
+    the platform scopes users upstream)."""
+    now = time.time()
+    internal_email = "internal@freesolo.co"
+    with _connect() as conn:
+        conn.execute(
+            "INSERT OR IGNORE INTO api_keys (key_hash, key_prefix, email, created_at) "
+            "VALUES (?, ?, ?, ?)",
+            (hash_key(api_key), "internal", internal_email, now),
+        )
+        conn.execute(
+            "UPDATE api_keys SET email = ? WHERE key_hash = ? AND key_prefix = ?",
+            (internal_email, hash_key(api_key), "internal"),
+        )
+    row = lookup_key(api_key)
+    if row is None:  # pragma: no cover - the row was just inserted
+        raise RuntimeError("failed to provision the internal service key")
+    return row
+
+
+def ensure_external_key(
+    api_key: str, *, key_prefix: str | None = None, email: str | None = None
+) -> dict | None:
+    """Provision a per-token row for a verified external (freesolo USER) key (idempotent).
+
+    Unlike :func:`ensure_internal_key` (one shared service identity), this keys a distinct
+    row by the presented token's hash, so each freesolo user key gets its OWN run-ownership
+    identity (the runs.key_id foreign key then scopes runs per user). The full token is never
+    stored — only its sha256.
+
+    Returns ``None`` (not a row) when the token's row already exists but is DISABLED:
+    ``INSERT OR IGNORE`` won't revive it and ``lookup_key`` filters disabled rows, so a
+    revoked key is rejected (401) by the caller instead of surfacing as a 500."""
+    now = time.time()
+    with _connect() as conn:
+        conn.execute(
+            "INSERT OR IGNORE INTO api_keys (key_hash, key_prefix, email, created_at) "
+            "VALUES (?, ?, ?, ?)",
+            (hash_key(api_key), key_prefix or "freesolo", email, now),
+        )
+    return lookup_key(api_key)
+
+
+def lookup_key(api_key: str) -> dict | None:
+    """Resolve a presented key to its row (and touch last_used_at); None if unknown/disabled."""
+    with _connect() as conn:
+        row = conn.execute(
+            "SELECT * FROM api_keys WHERE key_hash = ? AND disabled = 0",
+            (hash_key(api_key),),
+        ).fetchone()
+        if row is None:
+            return None
+        conn.execute("UPDATE api_keys SET last_used_at = ? WHERE id = ?", (time.time(), row["id"]))
+        return dict(row)
+
+
+def record_run(run_id: str, key_id: int) -> None:
+    with _connect() as conn:
+        conn.execute(
+            "INSERT INTO runs (run_id, key_id, kind, created_at) VALUES (?, ?, ?, ?)",
+            (run_id, key_id, "train", time.time()),
+        )
+
+
+def delete_run(run_id: str) -> None:
+    with _connect() as conn:
+        conn.execute("DELETE FROM runs WHERE run_id = ?", (run_id,))
+
+
+def run_owner(run_id: str) -> int | None:
+    with _connect() as conn:
+        row = conn.execute("SELECT key_id FROM runs WHERE run_id = ?", (run_id,)).fetchone()
+        return row["key_id"] if row else None
+
+
+def runs_for_key(key_id: int) -> list[dict]:
+    with _connect() as conn:
+        rows = conn.execute(
+            "SELECT run_id, kind, created_at FROM runs WHERE key_id = ? ORDER BY created_at",
+            (key_id,),
+        ).fetchall()
+        return [dict(r) for r in rows]
+
+
+def all_runs() -> list[dict]:
+    with _connect() as conn:
+        rows = conn.execute("SELECT run_id, key_id, kind, created_at FROM runs").fetchall()
+        return [dict(r) for r in rows]