swarph-cli 0.37.0__tar.gz → 0.38.0__tar.gz

{swarph_cli-0.37.0/src/swarph_cli.egg-info → swarph_cli-0.38.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: swarph-cli
-Version: 0.37.0
+Version: 0.38.0
 Summary: The `swarph` binary — a multi-LLM CLI and mesh-gateway client + bundled server (`swarph gateway`): one-shot prompts across providers, interactive `chat`, multi-provider `spawn` (claude/codex/antigravity via a ProviderMembrane with subprocess billing-scrub), `mesh` send/inbox/register, `brain-ask` over the swarph-brain (gbrain) semantic memory, git-backed `assisted_memory`, session `import`, and a stranded-session `watchdog`.
 Author: Pierre Samson, Claude Opus
 License: MIT
@@ -31,6 +31,8 @@ Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Provides-Extra: mcp
 Requires-Dist: mcp>=1.0; extra == "mcp"
+Provides-Extra: bench
+Requires-Dist: google-genai>=0.3; extra == "bench"
 Provides-Extra: gateway
 Requires-Dist: fastapi>=0.115; extra == "gateway"
 Requires-Dist: uvicorn[standard]>=0.32; extra == "gateway"

{swarph_cli-0.37.0 → swarph_cli-0.38.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "swarph-cli"
-version = "0.37.0"
+version = "0.38.0"
 description = "The `swarph` binary — a multi-LLM CLI and mesh-gateway client + bundled server (`swarph gateway`): one-shot prompts across providers, interactive `chat`, multi-provider `spawn` (claude/codex/antigravity via a ProviderMembrane with subprocess billing-scrub), `mesh` send/inbox/register, `brain-ask` over the swarph-brain (gbrain) semantic memory, git-backed `assisted_memory`, session `import`, and a stranded-session `watchdog`."
 readme = "README.md"
 license = { text = "MIT" }
@@ -60,6 +60,12 @@ dev = ["pytest>=7.0"]
 # so the core one-shot/mesh paths don't pull it in; `swarph mcp-server`
 # prints a `pip install swarph-cli[mcp]` hint when it's absent.
 mcp = ["mcp>=1.0"]
+# `swarph bench run` (metered backend) calls the google-genai Developer API
+# directly. The dependency is an extra so `bench`'s pack loader/validate/
+# distance-engine paths (and every other swarph-cli verb) stay usable without
+# it; MeteredGeminiBackend prints a `pip install swarph-cli[bench]` hint via
+# its ImportError message when it's absent.
+bench = ["google-genai>=0.3"]
 # `swarph gateway serve` runs the bundled mesh-gateway HTTP server. The
 # FastAPI/uvicorn stack is an extra so the core client paths stay
 # dependency-light; the verb prints a `pip install swarph-cli[gateway]`

{swarph_cli-0.37.0 → swarph_cli-0.38.0}/src/swarph_cli/__init__.py

@@ -16,6 +16,6 @@ The architecture splits CLI from substrate so:
 
 from __future__ import annotations
 
-__version__ = "0.37.0"
+__version__ = "0.38.0"
 
 __all__ = ["__version__"]

swarph_cli-0.38.0/src/swarph_cli/bench/__init__.py

@@ -0,0 +1,22 @@
+"""``swarph bench`` — deterministic LLM benchmark-pack runner (board card #101).
+
+A *pack* is a self-contained, subject-agnostic unit of {skill, tests, expected
+results} — one JSON file with three required parts: ``system`` (skill/context),
+``tasks[].prompt`` (tests), ``tasks[].expected`` (ground truth). Every score is a
+DISTANCE in [0,1] (0 = perfect); the engine is fixed and domain-agnostic, packs
+ship data only, never code.
+
+Ported from the reference implementation (``hedge-fund-mcp/scripts/{model_showdown,
+bench_quality,bench_news98,build_judgment_pack}.py``) per
+``docs/2026-07-24-swarph-bench-pack-spec.md``. See submodules:
+
+- :mod:`swarph_cli.bench.quality` — the distance engine (numeric/categorical/
+  ranking/text) + answer parsing. Ported verbatim from ``bench_quality.py``.
+- :mod:`swarph_cli.bench.pack` — pack loading + normative JSON-Schema validation.
+- :mod:`swarph_cli.bench.prices` — a small model-id -> ($/1M in, $/1M out) table.
+- :mod:`swarph_cli.bench.backends` — the backend abstraction (metered google-genai
+  Developer API + a subscription stub).
+- :mod:`swarph_cli.bench.runner` — the N-way showdown loop -> confusion view.
+- :mod:`swarph_cli.bench.validate` — the four ``validate`` disciplines.
+"""
+from __future__ import annotations

swarph_cli-0.38.0/src/swarph_cli/bench/backends.py

@@ -0,0 +1,191 @@
+"""The backend abstraction (spec §4). ``bench`` OWNS a thin, provider-agnostic
+backend interface — it does NOT reuse the LLM services layer's :878x/:879x
+lane map (decision #3: bench owns its own).
+
+A backend maps ``(model_id, prompt, system) -> BackendResult`` (tokens in /
+thought / out, latency, text, whether tokens are ESTIMATED). ``system``
+threads to every backend (metered: ``system_instruction``; a CLI-shelling
+backend would prepend it — see :class:`SubscriptionBackend`).
+
+Ships two backends:
+
+- :class:`MeteredGeminiBackend` — google-genai Developer API, DEFAULT for v1
+  (real ``usage_metadata``: prompt/candidates/thoughts tokens). CRITICAL:
+  passes ``vertexai=False`` EXPLICITLY — a ``GOOGLE_GENAI_USE_VERTEXAI=true``
+  env 401s API-key auth (this exact bug bit the reference lab; see
+  ``model_showdown.py::run_metered``).
+- :class:`SubscriptionBackend` — a STUB/interface only. Tokens are always
+  ESTIMATED (``~len(text)/4``, flagged via ``estimated=True``) and it does
+  NOT couple to any specific subscription lane map; a caller wires an actual
+  $0 CLI/OIDC path in by passing ``call_fn``. Not exercised against a live
+  provider anywhere in this package.
+"""
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass
+from typing import Callable, Optional, Protocol
+
+
+@dataclass
+class BackendResult:
+    text: str
+    tokens_in: int
+    tokens_thought: int
+    tokens_out: int
+    latency_s: float
+    estimated: bool
+    error: Optional[str] = None
+
+    @property
+    def total_tokens(self) -> int:
+        return self.tokens_in + (self.tokens_thought or 0) + self.tokens_out
+
+
+class Backend(Protocol):
+    def generate(self, model_id: str, prompt: str, system: str = "") -> BackendResult:
+        ...
+
+    def missing_creds(self) -> list[str]:
+        """-> human-readable names of missing REQUIRED credentials, e.g.
+        ``["GEMINI_API_KEY (or GOOGLE_API_KEY)"]``. Empty = ready to dispatch.
+        Checked by :func:`swarph_cli.bench.runner.preflight` BEFORE any
+        network call, so a missing key surfaces as one clear warning instead
+        of a mid-run 401 traceback."""
+        ...
+
+
+def estimate_tokens(text: Optional[str]) -> int:
+    """Rough ~4-chars/token estimate for backends that report no usage
+    metadata (ported from the reference ``_est_tokens``)."""
+    return max(1, round(len(text or "") / 4))
+
+
+class MeteredGeminiBackend:
+    """google-genai Developer API — the v1 DEFAULT metered backend. Real
+    ``usage_metadata`` (prompt/candidates/thoughts token counts), pennies per
+    call, no subscription-quota throttling.
+
+    ``google-genai`` is imported LAZILY (inside :meth:`generate`, not at
+    module import time) so importing :mod:`swarph_cli.bench.backends` never
+    requires the dependency unless the metered backend is actually used —
+    consistent with swarph-cli's dependency-light-core-paths convention
+    (``[mcp]``/``[gateway]``/``[service]`` extras).
+    """
+
+    #: env vars checked (in order) when no api_key is passed to __init__.
+    ENV_VARS = ("GEMINI_API_KEY", "GOOGLE_API_KEY")
+
+    def __init__(self, api_key: Optional[str] = None):
+        self._api_key = api_key
+
+    def missing_creds(self) -> list[str]:
+        if self._api_key:
+            return []
+        if any(os.environ.get(v) for v in self.ENV_VARS):
+            return []
+        return [f"{self.ENV_VARS[0]} (or {self.ENV_VARS[1]})"]
+
+    def credentials_ok(self) -> bool:
+        return not self.missing_creds()
+
+    def generate(self, model_id: str, prompt: str, system: str = "") -> BackendResult:
+        try:
+            from google import genai
+            from google.genai import types
+        except ImportError as exc:
+            return BackendResult(
+                text="", tokens_in=0, tokens_thought=0, tokens_out=0,
+                latency_s=0.0, estimated=False,
+                error=f"google-genai not installed: {exc} "
+                      f"(pip install swarph-cli[bench] or `pip install google-genai`)",
+            )
+        api_key = self._api_key or os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
+        if not api_key:
+            return BackendResult(
+                text="", tokens_in=0, tokens_thought=0, tokens_out=0,
+                latency_s=0.0, estimated=False,
+                error="no API key: set GEMINI_API_KEY (or GOOGLE_API_KEY)",
+            )
+        t0 = time.time()
+        try:
+            # vertexai=False EXPLICITLY: if GOOGLE_GENAI_USE_VERTEXAI=true is
+            # set in the environment, Vertex rejects API-key auth with a 401.
+            # This is not optional — it's the exact bug that bit swarph-news.
+            client = genai.Client(api_key=api_key, vertexai=False)
+            cfg = types.GenerateContentConfig(system_instruction=system) if system else None
+            resp = client.models.generate_content(model=model_id, contents=prompt, config=cfg)
+        except Exception as exc:  # provider/network errors surface, not swallow silently
+            return BackendResult(
+                text="", tokens_in=0, tokens_thought=0, tokens_out=0,
+                latency_s=round(time.time() - t0, 2), estimated=False, error=str(exc),
+            )
+        latency_s = round(time.time() - t0, 2)
+        usage = getattr(resp, "usage_metadata", None)
+        return BackendResult(
+            text=getattr(resp, "text", "") or "",
+            tokens_in=(usage.prompt_token_count or 0) if usage else 0,
+            tokens_thought=(getattr(usage, "thoughts_token_count", 0) or 0) if usage else 0,
+            tokens_out=(usage.candidates_token_count or 0) if usage else 0,
+            latency_s=latency_s,
+            estimated=False,
+        )
+
+
+class SubscriptionBackend:
+    """STUB/interface-only backend for a $0 subscription path (e.g. a node's
+    own CLI/OIDC lane). Deliberately NOT coupled to the reference stack's
+    :878x/:879x lane map (decision #3) — a caller supplies ``call_fn`` to
+    wire an actual subscription transport; without one, every call fails
+    clearly rather than silently degrading.
+
+    Token counts are ALWAYS estimated (``estimated=True``) — subscription
+    CLIs commonly emit no usage metadata at all. This is UNVERIFIED against
+    any live subscription provider; it exists so ``bench`` has a second
+    backend shape to validate against without hardcoding a real one.
+    """
+
+    def __init__(self, call_fn: Optional[Callable[[str, str, str], str]] = None):
+        # call_fn(model_id, prompt, system) -> raw response text
+        self._call_fn = call_fn
+
+    def missing_creds(self) -> list[str]:
+        """Not env-checkable (the credential is a node's own CLI/OIDC
+        session, which varies per deployment) — per decision, this FLAGS
+        rather than hard-checks: no ``call_fn`` wired means there is no
+        transport at all, which the preflight treats the same as a missing
+        credential so it's skipped with a clear reason instead of erroring
+        mid-run."""
+        if self._call_fn is None:
+            return ["subscription CLI/OIDC auth (not env-checkable; no call_fn wired — stub backend)"]
+        return []
+
+    def credentials_ok(self) -> bool:
+        return not self.missing_creds()
+
+    def generate(self, model_id: str, prompt: str, system: str = "") -> BackendResult:
+        if self._call_fn is None:
+            return BackendResult(
+                text="", tokens_in=0, tokens_thought=0, tokens_out=0,
+                latency_s=0.0, estimated=True,
+                error="SubscriptionBackend is a stub — no call_fn wired for a real "
+                      "$0 subscription transport",
+            )
+        t0 = time.time()
+        try:
+            text = self._call_fn(model_id, prompt, system) or ""
+        except Exception as exc:
+            return BackendResult(
+                text="", tokens_in=0, tokens_thought=0, tokens_out=0,
+                latency_s=round(time.time() - t0, 2), estimated=True, error=str(exc),
+            )
+        latency_s = round(time.time() - t0, 2)
+        return BackendResult(
+            text=text,
+            tokens_in=estimate_tokens(prompt),
+            tokens_thought=0,
+            tokens_out=estimate_tokens(text),
+            latency_s=latency_s,
+            estimated=True,
+        )

swarph_cli-0.38.0/src/swarph_cli/bench/pack.py

@@ -0,0 +1,145 @@
+"""Pack loader + the normative JSON-Schema validation (spec §1).
+
+A pack is one JSON file, three required parts: ``system`` (skill/context),
+``tasks[].prompt`` (tests), ``tasks[].expected`` (ground truth). Hand-rolled
+schema check (no ``jsonschema`` dependency — swarph-cli keeps its core paths
+dependency-light, per the existing ``codegraph``/``compress`` pattern) that
+enforces exactly the normative schema from the spec plus the constraints
+listed beneath it:
+
+- ``id`` unique within a pack
+- ``expected`` shape matches ``type`` (numeric->number, categorical->string,
+  ranking->array, text->string)
+- ``system`` SHOULD be present (warning, not an error, if absent)
+"""
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+TASK_TYPES = ("numeric", "categorical", "ranking", "text")
+
+
+class PackError(Exception):
+    """Raised by :func:`load_pack` when the JSON itself can't be read/parsed."""
+
+
+def load_pack(path: str | Path) -> dict:
+    """Read + JSON-parse a pack file. Raises :class:`PackError` on I/O or
+    JSON-decode failure (schema validity is a separate step — see
+    :func:`validate_schema` — so callers can report *why* a pack is invalid
+    rather than crash)."""
+    p = Path(path)
+    try:
+        raw = p.read_text()
+    except OSError as exc:
+        raise PackError(f"cannot read pack {p}: {exc}") from exc
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise PackError(f"pack {p} is not valid JSON: {exc}") from exc
+    if not isinstance(data, dict):
+        raise PackError(f"pack {p} must be a JSON object, got {type(data).__name__}")
+    return data
+
+
+def _expected_matches_type(expected: Any, ttype: str) -> bool:
+    if ttype == "numeric":
+        return isinstance(expected, (int, float)) and not isinstance(expected, bool)
+    if ttype == "categorical":
+        return isinstance(expected, str)
+    if ttype == "ranking":
+        return isinstance(expected, list) and all(isinstance(x, (str, int, float)) for x in expected)
+    if ttype == "text":
+        return isinstance(expected, str)
+    return False
+
+
+def validate_schema(pack: dict) -> tuple[list[str], list[str]]:
+    """Validate ``pack`` against the normative JSON-Schema (spec §1) plus the
+    constraints beneath it. -> ``(errors, warnings)``. Non-empty ``errors``
+    means the pack is INVALID (schema violation, dup id, type/expected
+    mismatch); ``warnings`` are advisory (e.g. missing ``system``) and don't
+    fail the pack on their own."""
+    errors: list[str] = []
+    warnings: list[str] = []
+
+    if not isinstance(pack, dict):
+        return ([f"pack must be a JSON object, got {type(pack).__name__}"], warnings)
+
+    theme = pack.get("theme")
+    if not isinstance(theme, str) or not theme.strip():
+        errors.append("'theme' is required and must be a non-empty string")
+
+    if "version" in pack and not isinstance(pack["version"], str):
+        errors.append("'version' must be a string when present")
+
+    if "system" not in pack or not str(pack.get("system") or "").strip():
+        warnings.append(
+            "pack has no 'system' — a pack without a system tests raw model priors, "
+            "not applying a real operating context (spec §5d: system is the DOMINANT "
+            "variable and must mirror real deployment)"
+        )
+    elif not isinstance(pack.get("system"), str):
+        errors.append("'system' must be a string when present")
+
+    if "description" in pack and not isinstance(pack["description"], str):
+        errors.append("'description' must be a string when present")
+
+    tasks = pack.get("tasks")
+    if not isinstance(tasks, list) or len(tasks) < 1:
+        errors.append("'tasks' is required and must be a non-empty array")
+        return (errors, warnings)
+
+    seen_ids: set[str] = set()
+    for i, task in enumerate(tasks):
+        where = f"tasks[{i}]"
+        if not isinstance(task, dict):
+            errors.append(f"{where} must be an object")
+            continue
+
+        tid = task.get("id")
+        if not isinstance(tid, str) or not tid.strip():
+            errors.append(f"{where}.id is required and must be a non-empty string")
+        elif tid in seen_ids:
+            errors.append(f"duplicate task id {tid!r}")
+        else:
+            seen_ids.add(tid)
+
+        prompt = task.get("prompt")
+        if not isinstance(prompt, str) or not prompt.strip():
+            errors.append(f"{where}.prompt is required and must be a non-empty string")
+
+        ttype = task.get("type")
+        if ttype not in TASK_TYPES:
+            errors.append(f"{where}.type must be one of {TASK_TYPES}, got {ttype!r}")
+            continue  # can't check expected-shape against an unknown type
+
+        if "expected" not in task:
+            errors.append(f"{where}.expected is required")
+        elif not _expected_matches_type(task["expected"], ttype):
+            errors.append(
+                f"{where}.expected does not match type={ttype!r} "
+                f"(got {type(task['expected']).__name__})"
+            )
+
+        if "weight" in task:
+            w = task["weight"]
+            if not isinstance(w, (int, float)) or isinstance(w, bool) or w <= 0:
+                errors.append(f"{where}.weight must be a number > 0, got {w!r}")
+
+        if "meta" in task and not isinstance(task["meta"], dict):
+            errors.append(f"{where}.meta must be an object when present")
+
+    return (errors, warnings)
+
+
+def slugify_theme(theme: str) -> str:
+    """``theme`` -> a filesystem-safe pack slug: lowercase, non-alnum runs ->
+    ``_``, trimmed. Drives the ``packs/<theme>.json`` naming convention
+    (spec §6 / ``bench add``) — the pack's OWN header names the file, no
+    separate name is ever asked for on the command line."""
+    s = re.sub(r"[^a-z0-9]+", "_", theme.strip().lower()).strip("_")
+    return s or "pack"

swarph_cli-0.38.0/src/swarph_cli/bench/prices.py

@@ -0,0 +1,103 @@
+"""The auto-refreshed LLM LIST-price table (spec §4) — LIFTED from the
+reference lab's already-tested ``workers/llm_prices.py`` (adapted imports/
+cache path only; lookup semantics, fallback table, and substring-order bug
+guard are unchanged). Do NOT reinvent this — see
+``docs/2026-07-24-model-showdown-findings.md`` shipped-alongside note: it
+already caught a real ~5x under-count on gemini-2.5-flash-lite.
+
+The LIST-price BASE is auto-refreshed from the community-maintained LiteLLM
+repo (:mod:`swarph_cli.bench.refresh_prices` -> a local JSON cache; TODO wire
+a weekly cron the way the reference lab does), so new models are never
+silently mis-priced by a stale hardcoded bucket.
+
+Public API:
+    lookup(model) -> (input_$/Mtok, output_$/Mtok)   # never raises
+    cost_usd(model, tokens_in, tokens_thought, tokens_out) -> $
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+
+log = logging.getLogger("swarph.bench.prices")
+
+_CACHE_PATH = os.path.join(os.path.dirname(__file__), "data", "llm_prices.json")
+_MAP: dict | None = None
+
+# Reconciled fallbacks ($/Mtok input, output) — used ONLY when a model isn't in
+# the refreshed cache. The cache is the real source; these are the safety net.
+_FALLBACK = {
+    "pro": (1.25, 10.0),
+    "lite": (0.019, 0.075),
+    "flash": (0.30, 2.50),
+}
+
+
+def load(force: bool = False) -> dict:
+    """The cached ``{normalized_model: {"in": $/Mtok, "out": $/Mtok}}`` map.
+    Cached in-process; ``force=True`` re-reads the file. Never raises."""
+    global _MAP
+    if _MAP is not None and not force:
+        return _MAP
+    try:
+        with open(_CACHE_PATH) as f:
+            _MAP = (json.load(f) or {}).get("prices", {}) or {}
+    except Exception as e:  # missing/corrupt cache -> fall back to built-ins
+        log.debug("llm price cache unavailable (%s) — using fallbacks", e)
+        _MAP = {}
+    return _MAP
+
+
+def _norm(model: str) -> str:
+    """Strip provider prefix (gemini/, vertex_ai/, ...) + lowercase."""
+    return (model or "").split("/")[-1].strip().lower()
+
+
+def lookup(model: str) -> tuple[float, float]:
+    """(input_$/Mtok, output_$/Mtok) for ``model``. Exact cache hit ->
+    substring fallback (lite before flash so 'flash-lite' resolves as lite —
+    order-sensitive, don't reorder). Never raises."""
+    m = _norm(model)
+    prices = load()
+    p = prices.get(m)
+    if p and "in" in p and "out" in p:
+        return (float(p["in"]), float(p["out"]))
+    # substring fallback — ORDER MATTERS: lite is a substring of flash-lite
+    # and must win over flash.
+    if "lite" in m:
+        return _FALLBACK["lite"]
+    if "pro" in m:
+        return _FALLBACK["pro"]
+    return _FALLBACK["flash"]  # flash + safest default
+
+
+def is_known(model_id: str) -> bool:
+    """True if ``model_id`` has an exact hit in the refreshed cache (as
+    opposed to resolving via the substring fallback)."""
+    p = load().get(_norm(model_id))
+    return bool(p and "in" in p and "out" in p)
+
+
+def cost_usd(model_id: str, tokens_in: int, tokens_thought: int, tokens_out: int) -> float:
+    """Metered $ for one call: in-tokens at the input price, (thought + out)
+    tokens at the output price (mirrors the reference ``metered_usd()``)."""
+    price_in, price_out = lookup(model_id)
+    thought = tokens_thought or 0
+    return (tokens_in * price_in + (thought + tokens_out) * price_out) / 1_000_000
+
+
+def all_cached(force: bool = False) -> dict[str, tuple[float, float]]:
+    """Every ``{model_id: (in, out)}`` currently in the refreshed cache — the
+    full LiteLLM-sourced price list, not just the models ``bench`` happens to
+    price. Exposed so the cache is a reusable mesh asset (e.g. a
+    cost-governor/ledger can read the same cache via ``swarph bench prices``)
+    rather than plumbing private to the runner."""
+    return {m: (float(v["in"]), float(v["out"])) for m, v in load(force=force).items()
+            if isinstance(v, dict) and "in" in v and "out" in v}
+
+
+def fallback_buckets() -> dict[str, tuple[float, float]]:
+    """The 3 built-in substring-fallback buckets (pro/lite/flash), for
+    display when the cache is empty/stale."""
+    return dict(_FALLBACK)

swarph_cli-0.38.0/src/swarph_cli/bench/quality.py

@@ -0,0 +1,122 @@
+"""The FIXED distance engine — ported VERBATIM (math unchanged) from the
+reference ``bench_quality.py`` (``d_numeric/d_categorical/d_ranking/d_text``,
+``parse_answer``, ``score``). See spec §2.
+
+Every score is a DISTANCE in [0, 1] where 0 = perfect. The engine is fixed and
+domain-agnostic — a pack never ships code, only data (``system`` + tasks +
+expected answers); this module is the ONLY place distance math lives.
+
+**Parse-fail -> distance = 1.0** is a load-bearing rule, not an edge case: a
+model that can't follow the structured-output instruction gets the worst
+possible score, because that IS a real quality signal (findings.md).
+"""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Callable, Optional
+
+# ---- distance functions (0 = exact, 1 = worst) -----------------------------
+
+
+def d_numeric(provided: Any, expected: Any) -> float:
+    """Capped relative error. ``e == 0`` is a special case: 0.0 if ``p == 0``
+    else 1.0 (relative error is undefined at zero)."""
+    try:
+        p, e = float(provided), float(expected)
+    except (TypeError, ValueError):
+        return 1.0
+    if e == 0:
+        return 0.0 if p == 0 else 1.0
+    return min(abs(p - e) / abs(e), 1.0)
+
+
+def d_categorical(provided: Any, expected: Any) -> float:
+    """0 if ``upper(strip(provided)) == upper(strip(expected))`` else 1."""
+    return 0.0 if str(provided).strip().upper() == str(expected).strip().upper() else 1.0
+
+
+def d_ranking(provided: Any, expected: Any) -> float:
+    """Normalized Kendall-tau distance on the ranked list (0 = same order,
+    1 = fully reversed). Wrong item SET (not a list, or items don't match the
+    expected set) -> 1.0 (worst), per spec §2."""
+    if not isinstance(provided, list) or sorted(map(str, provided)) != sorted(map(str, expected)):
+        return 1.0
+    pos = {str(x): i for i, x in enumerate(provided)}
+    exp = [str(x) for x in expected]
+    n = len(exp)
+    if n < 2:
+        return 0.0
+    discord = sum(1 for i in range(n) for j in range(i + 1, n) if pos[exp[i]] > pos[exp[j]])
+    return discord / (n * (n - 1) / 2)
+
+
+def d_text(provided: Any, expected: Any, embedder: Optional[Callable[[str], Any]] = None) -> float:
+    """Free-text distance: ``1 - Jaccard(tokens(p), tokens(e))`` — a cheap, $0,
+    deterministic proxy with no external dependency (v1 default, spec §2 /
+    decision #2).
+
+    ``embedder`` is a clearly-marked SEAM for an optional embedding-cosine
+    distance (e.g. gbrain's $0 embeddings) — NOT wired in v1. When provided it
+    must be a ``str -> vector`` callable; cosine distance replaces Jaccard.
+    Left unimplemented deliberately: gbrain is not coupled here yet.
+    """
+    if embedder is not None:
+        raise NotImplementedError(
+            "d_text(embedder=...) is a seam for a future gbrain-cosine embedder; "
+            "not wired in v1 — call d_text(provided, expected) without embedder."
+        )
+    pa = set(str(provided).lower().split())
+    ea = set(str(expected).lower().split())
+    if not ea:
+        return 0.0 if not pa else 1.0
+    union = len(pa | ea)
+    return 1.0 - (len(pa & ea) / union if union else 0.0)
+
+
+DISTANCE: dict[str, Callable[[Any, Any], float]] = {
+    "numeric": d_numeric,
+    "categorical": d_categorical,
+    "ranking": d_ranking,
+    "text": d_text,
+}
+
+
+# ---- answer extraction ------------------------------------------------------
+
+
+def parse_answer(text: Optional[str]) -> Any:
+    """Pull the LAST ``{"answer": ...}`` JSON object from a model reply
+    (tolerant of code fences / surrounding prose). Returns ``None`` on parse
+    failure — the caller (``score``) turns that into distance=1.0."""
+    if not text:
+        return None
+    last = None
+    found = False
+    for m in re.finditer(r'\{[^{}]*"answer"[^{}]*\}', text, re.DOTALL):
+        try:
+            last = json.loads(m.group(0)).get("answer")
+            found = True
+        except Exception:
+            continue
+    return last if found else None
+
+
+def score(task: dict, text: str) -> dict:
+    """-> ``{distance, parsed, parse_ok}``.
+
+    ``numeric``/``categorical``/``ranking`` are parsed from the structured
+    ``{"answer": ...}`` JSON; ``text`` scores the raw response verbatim.
+    Unparseable structured answers are the worst score (1.0) — a real quality
+    signal, not a bug (spec §2).
+    """
+    ttype = task["type"]
+    if ttype == "text":
+        ans = text or ""
+        dist = d_text(ans, task["expected"])
+        return {"distance": round(dist, 4), "parsed": ans, "parse_ok": True}
+    ans = parse_answer(text)
+    if ans is None:
+        return {"distance": 1.0, "parsed": None, "parse_ok": False}
+    dist = DISTANCE[ttype](ans, task["expected"])
+    return {"distance": round(dist, 4), "parsed": ans, "parse_ok": True}