pytest-jev 0.1.0__py3-none-any.whl

pytest_jev/__init__.py

@@ -0,0 +1,8 @@
+"""Semantic assertions for pytest, judged by TypeSafe's Jev."""
+
+from pytest_jev.judge import Jev
+from pytest_jev.verdicts import ChoiceVerdict, Claim, Claims, ScoreVerdict
+
+__version__ = "0.1.0"
+
+__all__ = ["ChoiceVerdict", "Claim", "Claims", "Jev", "ScoreVerdict", "__version__"]

pytest_jev/judge.py

@@ -0,0 +1,343 @@
+"""The `jev` fixture's object: builds Jev questions, caches answers and returns verdicts."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+import pytest
+from typesafe_sdk import RetryPolicy, TypeSafeAuthenticationError, TypeSafeClient, TypeSafeError
+
+from pytest_jev.provider import ConfigError, resolve_provider
+from pytest_jev.verdicts import ChoiceVerdict, Claim, Claims, ScoreVerdict
+
+# Jev bills input tokens only: $0.042 per million (docs.typesafe.ai/models, checked 2026-09-21).
+JEV_USD_PER_MTOK = 0.042
+
+# Bump when the questions pytest-jev sends change meaning, so old cached answers are not reused.
+CACHE_PREFIX = "jev/v1/"
+
+MAX_CHOICE_OPTIONS = 255  # Jev's limit for one Choice question
+
+# Exponential backoff on timeouts, rate limits and server errors, handled inside the SDK.
+RETRY_POLICY = RetryPolicy(
+    max_retries=4,
+    backoff_initial=0.5,
+    backoff_max=8.0,
+    http_statuses={408, 429, *range(500, 600)},
+    timeout=60.0,
+)
+REQUEST_TIMEOUT = 30.0
+
+
+class Client(Protocol):
+    """Anything with the TypeSafe SDK's `system_one` signature, e.g. system-one-adapter's client."""
+
+    def system_one(self, *, state: Any, questions: Any, model: str) -> Any: ...
+
+
+class DefaultClient:
+    """A TypeSafe client created on the first question, so a run without a key only skips the
+    tests that actually need Jev."""
+
+    def __init__(self, provider: str = "auto") -> None:
+        self.provider = provider
+        self.name: str | None = None  # the provider that answered, once known
+        self._client: TypeSafeClient | None = None
+
+    def system_one(self, *, state: Any, questions: Any, model: str) -> Any:
+        if self._client is None:
+            provider = resolve_provider(self.provider)
+            self._client = TypeSafeClient(
+                api_key=provider.api_key,
+                base_url=provider.base_url,
+                retry=RETRY_POLICY,
+                timeout=REQUEST_TIMEOUT,
+            )
+            self.name = provider.name
+        return self._client.system_one(state=state, questions=questions, model=model)
+
+    def close(self) -> None:
+        if self._client is not None:
+            self._client.close()
+            self._client = None
+
+
+@dataclass
+class Stats:
+    """Totals for the run, printed in the terminal summary."""
+
+    questions: int = 0
+    requests: int = 0  # sent to Jev
+    cached: int = 0  # answered from .pytest_cache instead
+    input_tokens: int = 0
+    seconds: float = 0.0
+    models: set[str] = field(default_factory=set)  # versioned model IDs that answered
+    providers: set[str] = field(default_factory=set)
+
+    @property
+    def cost_usd(self) -> float:
+        return self.input_tokens * JEV_USD_PER_MTOK / 1_000_000
+
+
+class JevSession:
+    """State shared by every `jev` fixture in a run: the answer cache and the usage totals."""
+
+    def __init__(self, cache: Any, *, read_cache: bool = True, require_key: bool = False) -> None:
+        self.cache = cache  # pytest's config.cache, or None with -p no:cacheprovider
+        self.read_cache = read_cache
+        self.require_key = require_key
+        self.stats = Stats()
+
+    def ask(
+        self, client: Client, model: str, state: dict[str, Any], questions: dict[str, Any]
+    ) -> dict[str, dict[str, Any]]:
+        """Answer every question about `state` in one request, or from the cache."""
+        __tracebackhide__ = True
+        key = cache_key(model, state, questions)
+        self.stats.questions += len(questions)
+        if self.read_cache and self.cache is not None:
+            hit = self.cache.get(key, None)
+            if isinstance(hit, dict) and set(questions) <= set(hit.get("answers", {})):
+                self.stats.cached += 1
+                self.stats.models.add(hit.get("model", model))
+                return hit["answers"]
+
+        started = time.perf_counter()
+        try:
+            response = client.system_one(state=state, questions=questions, model=model)
+        except ConfigError as exc:
+            if self.require_key:
+                pytest.fail(f"pytest-jev: {exc}", pytrace=False)
+            pytest.skip(f"pytest-jev: {exc}")
+        except TypeSafeAuthenticationError as exc:
+            pytest.fail(
+                f"pytest-jev: authentication failed, check your API key ({exc})", pytrace=False
+            )
+        except TypeSafeError as exc:
+            pytest.fail(f"pytest-jev: the Jev request failed: {exc}", pytrace=False)
+        elapsed = time.perf_counter() - started
+
+        missing = [name for name in questions if name not in response.answers]
+        if missing:
+            pytest.fail(f"pytest-jev: Jev did not answer {', '.join(missing)}", pytrace=False)
+        answers = {name: plain_answer(response.answers[name]) for name in questions}
+
+        stats = self.stats
+        stats.requests += 1
+        stats.seconds += elapsed
+        stats.input_tokens += getattr(response.usage, "input_tokens", None) or 0
+        stats.models.add(response.model)
+        if getattr(client, "name", None):
+            stats.providers.add(client.name)
+        if self.cache is not None:
+            self.cache.set(key, {"model": response.model, "answers": answers})
+        return answers
+
+
+def cache_key(model: str, state: dict[str, Any], questions: dict[str, Any]) -> str:
+    payload = {"model": model, "state": state, "questions": questions}
+    try:
+        encoded = json.dumps(payload, sort_keys=True, ensure_ascii=False)
+    except (TypeError, ValueError) as exc:
+        raise TypeError(f"jev: the text and context must be JSON-serializable ({exc})") from None
+    return CACHE_PREFIX + hashlib.sha256(encoded.encode()).hexdigest()
+
+
+def plain_answer(answer: Any) -> dict[str, Any]:
+    """An SDK answer as a JSON-ready dict, the form the cache stores."""
+    if answer.type == "noul":
+        return {"type": "noul", "noul": float(answer.noul)}
+    probabilities = {str(key): float(p) for key, p in answer.probabilities.items()}
+    if answer.type == "choice":
+        return {
+            "type": "choice",
+            "choice": answer.choice,
+            "confidence": float(answer.confidence),
+            "probabilities": probabilities,
+        }
+    return {
+        "type": "score",
+        "score": float(answer.score),
+        "confidence": float(answer.confidence),
+        "probabilities": probabilities,
+    }
+
+
+def check_threshold(threshold: float) -> float:
+    # Below 0.5, a claim could pass both `holds` and `lacks` at once.
+    if not 0.5 <= threshold <= 1.0:
+        raise ValueError(f"jev threshold must be between 0.5 and 1, got {threshold}")
+    return float(threshold)
+
+
+def claim_question(claim: str) -> str:
+    if not isinstance(claim, str) or not claim.strip():
+        raise ValueError("a jev claim must be a non-empty string")
+    return f"Does `text` satisfy: {claim.strip().rstrip('?.! ')}?"
+
+
+def build_state(text: Any, context: Mapping[str, Any] | None) -> dict[str, Any]:
+    state = {"text": text}
+    if context:
+        if "text" in context:
+            raise ValueError("jev context can't use the key 'text', it holds the text under test")
+        state.update(context)
+    return state
+
+
+def claim_list(claims: str | Iterable[str]) -> list[str]:
+    return [claims] if isinstance(claims, str) else list(claims)
+
+
+def check_question(question: str) -> str:
+    if not isinstance(question, str) or not question.strip():
+        raise ValueError("a jev question must be a non-empty string")
+    return question.strip()
+
+
+def option_criteria(options: Mapping[str, str | None] | Sequence[str]) -> dict[str, str | None]:
+    if isinstance(options, str):
+        raise ValueError("jev.choice takes a list of options or a dict of label -> description")
+    criteria = dict(options) if isinstance(options, Mapping) else dict.fromkeys(options)
+    if len(criteria) != len(options) or not 2 <= len(criteria) <= MAX_CHOICE_OPTIONS:
+        raise ValueError(f"jev.choice needs 2 to {MAX_CHOICE_OPTIONS} distinct options")
+    return criteria
+
+
+def score_levels(levels: Mapping[str, str] | Sequence[str]) -> tuple[tuple[str, ...], list[str]]:
+    """Level labels for comparisons, and the descriptions Jev reads, lowest level first."""
+    if isinstance(levels, str):
+        raise ValueError("jev.score takes a list of levels or a dict of label -> description")
+    labels = tuple(levels)
+    descriptions = list(levels.values()) if isinstance(levels, Mapping) else list(levels)
+    if len(labels) < 2 or len(set(labels)) != len(labels):
+        raise ValueError("jev.score needs at least 2 distinct levels, lowest first")
+    return labels, descriptions
+
+
+class Jev:
+    """Semantic assertions about text, answered by TypeSafe's Jev. This is the `jev` fixture."""
+
+    def __init__(self, session: JevSession, client: Client, *, model: str, threshold: float):
+        self._session = session
+        self._client = client
+        self.model = model
+        self.threshold = check_threshold(threshold)
+
+    def __repr__(self) -> str:
+        return f"<jev model={self.model} threshold={self.threshold}>"
+
+    def holds(
+        self,
+        text: Any,
+        claim: str,
+        *,
+        context: Mapping[str, Any] | None = None,
+        threshold: float | None = None,
+    ) -> Claim:
+        """Is `claim` true of `text`? Truthy when Jev's probability is at least the threshold."""
+        __tracebackhide__ = True
+        return self._judge(text, [(claim, "holds")], context, threshold)[0]
+
+    def lacks(
+        self,
+        text: Any,
+        claim: str,
+        *,
+        context: Mapping[str, Any] | None = None,
+        threshold: float | None = None,
+    ) -> Claim:
+        """Is `claim` false of `text`? Truthy when Jev's probability is at most 1 - threshold."""
+        __tracebackhide__ = True
+        return self._judge(text, [(claim, "lacks")], context, threshold)[0]
+
+    def expect(
+        self,
+        text: Any,
+        *,
+        holds: str | Iterable[str] = (),
+        lacks: str | Iterable[str] = (),
+        context: Mapping[str, Any] | None = None,
+        threshold: float | None = None,
+    ) -> Claims:
+        """Check several claims in one request; fail with a report of every claim if any fails."""
+        __tracebackhide__ = True
+        pairs = [(claim, "holds") for claim in claim_list(holds)]
+        pairs += [(claim, "lacks") for claim in claim_list(lacks)]
+        if not pairs:
+            raise ValueError("jev.expect needs at least one claim in holds= or lacks=")
+        claims = self._judge(text, pairs, context, threshold)
+        if not claims.passed:
+            raise AssertionError(claims.report())
+        return claims
+
+    def choice(
+        self,
+        text: Any,
+        question: str,
+        options: Mapping[str, str | None] | Sequence[str],
+        *,
+        context: Mapping[str, Any] | None = None,
+    ) -> ChoiceVerdict:
+        """Which option fits `text`? Map labels to descriptions, or pass bare labels."""
+        __tracebackhide__ = True
+        question = check_question(question)
+        criteria = option_criteria(options)
+        asked = {"type": "choice", "instructions": question, "criteria": criteria}
+        answer = self._ask(text, context, {"choice": asked})["choice"]
+        probabilities = {label: answer["probabilities"].get(label, 0.0) for label in criteria}
+        return ChoiceVerdict(question, answer["choice"], probabilities, answer["confidence"])
+
+    def score(
+        self,
+        text: Any,
+        question: str,
+        levels: Mapping[str, str] | Sequence[str],
+        *,
+        context: Mapping[str, Any] | None = None,
+        threshold: float | None = None,
+    ) -> ScoreVerdict:
+        """Rate `text` on ordered levels, lowest first. Map short labels to descriptions, or pass
+        descriptions alone. Compare the result with a label: `assert tone >= "neutral"`."""
+        __tracebackhide__ = True
+        question = check_question(question)
+        threshold = self.threshold if threshold is None else check_threshold(threshold)
+        labels, descriptions = score_levels(levels)
+        asked = {"type": "score", "instructions": question, "criteria": descriptions}
+        answer = self._ask(text, context, {"score": asked})["score"]
+        spread = tuple(answer["probabilities"].get(str(i), 0.0) for i in range(len(labels)))
+        return ScoreVerdict(
+            question, labels, spread, answer["score"], answer["confidence"], threshold
+        )
+
+    def _judge(
+        self,
+        text: Any,
+        pairs: list[tuple[str, str]],
+        context: Mapping[str, Any] | None,
+        threshold: float | None,
+    ) -> Claims:
+        __tracebackhide__ = True
+        threshold = self.threshold if threshold is None else check_threshold(threshold)
+        questions = {
+            f"claim_{i}": {"type": "noul", "instructions": claim_question(claim)}
+            for i, (claim, _) in enumerate(pairs)
+        }
+        answers = self._ask(text, context, questions)
+        claims = [
+            Claim(claim.strip(), expect, answers[f"claim_{i}"]["noul"], threshold)
+            for i, (claim, expect) in enumerate(pairs)
+        ]
+        return Claims(claims, text)
+
+    def _ask(
+        self, text: Any, context: Mapping[str, Any] | None, questions: dict[str, Any]
+    ) -> dict[str, dict[str, Any]]:
+        __tracebackhide__ = True
+        state = build_state(text, context)
+        return self._session.ask(self._client, self.model, state, questions)

pytest_jev/plugin.py

@@ -0,0 +1,155 @@
+"""pytest hooks: options, the `jev` fixture, the automatic `jev` marker and the run summary."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterator
+from typing import Any
+
+import pytest
+
+from pytest_jev.judge import Client, DefaultClient, Jev, JevSession, check_threshold
+from pytest_jev.provider import PROVIDERS
+from pytest_jev.verdicts import FLIPPED, ChoiceVerdict, ScoreVerdict
+
+DEFAULT_MODEL = "jev-latest"
+DEFAULT_THRESHOLD = 0.8
+MARKER_OPTIONS = {"model", "threshold"}
+
+SESSION = pytest.StashKey[JevSession]()
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("jev", "semantic assertions judged by TypeSafe's Jev")
+    group.addoption(
+        "--jev-model",
+        help=f"Jev model (default {DEFAULT_MODEL}). Pin a version such as jev-1.13 "
+        "for reproducible runs.",
+    )
+    group.addoption(
+        "--jev-threshold",
+        type=float,
+        help=f"Probability a claim needs for jev.holds (default {DEFAULT_THRESHOLD}); "
+        "jev.lacks needs at most 1 minus this.",
+    )
+    group.addoption(
+        "--jev-provider",
+        choices=PROVIDERS,
+        help="Where Jev runs (default auto: OpenRouter if OPENROUTER_API_KEY is set, "
+        "else TypeSafe).",
+    )
+    group.addoption(
+        "--jev-no-cache",
+        action="store_true",
+        help="Ask Jev again instead of reusing answers cached in .pytest_cache.",
+    )
+    group.addoption(
+        "--jev-require",
+        action="store_true",
+        help="Fail tests that need Jev when no API key is set, instead of skipping them.",
+    )
+    parser.addini("jev_model", f"Jev model (default {DEFAULT_MODEL}).", default=DEFAULT_MODEL)
+    parser.addini("jev_threshold", "Default claim threshold (0.8).", default=str(DEFAULT_THRESHOLD))
+    parser.addini("jev_provider", "auto, openrouter or typesafe.", default="auto")
+    parser.addini("jev_require", "Fail instead of skip without an API key.", type="bool")
+
+
+def _setting(config: pytest.Config, name: str) -> Any:
+    value = config.getoption(f"--jev-{name}")
+    return config.getini(f"jev_{name}") if value is None else value
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line(
+        "markers",
+        "jev(threshold=None, model=None): a test that asks TypeSafe's Jev; "
+        "the keyword arguments override the run's defaults for that test.",
+    )
+    try:
+        check_threshold(float(_setting(config, "threshold")))
+    except ValueError as exc:
+        raise pytest.UsageError(str(exc)) from None
+    if _setting(config, "provider") not in PROVIDERS:
+        raise pytest.UsageError(f"jev_provider must be one of: {', '.join(PROVIDERS)}")
+    config.stash[SESSION] = JevSession(
+        getattr(config, "cache", None),
+        read_cache=not config.getoption("--jev-no-cache"),
+        require_key=config.getoption("--jev-require") or config.getini("jev_require"),
+    )
+
+
+def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
+    # Mark every test that uses Jev, so `-m "not jev"` runs the rest offline.
+    for item in items:
+        if "jev" in getattr(item, "fixturenames", ()) and item.get_closest_marker("jev") is None:
+            item.add_marker("jev")
+
+
+@pytest.fixture(scope="session")
+def jev_client(pytestconfig: pytest.Config) -> Iterator[Client]:
+    """Sends questions to Jev. Override it to use a fake or another backend."""
+    client = DefaultClient(_setting(pytestconfig, "provider"))
+    yield client
+    client.close()
+
+
+@pytest.fixture
+def jev(request: pytest.FixtureRequest, jev_client: Client) -> Jev:
+    """Semantic assertions judged by TypeSafe's Jev: holds, lacks, expect, choice and score."""
+    config = request.config
+    marker = request.node.get_closest_marker("jev")
+    overrides = dict(marker.kwargs) if marker else {}
+    unknown = set(overrides) - MARKER_OPTIONS
+    if unknown:
+        raise TypeError(f"@pytest.mark.jev got unknown options: {', '.join(sorted(unknown))}")
+    return Jev(
+        config.stash[SESSION],
+        jev_client,
+        model=overrides.get("model") or _setting(config, "model"),
+        threshold=float(overrides.get("threshold") or _setting(config, "threshold")),
+    )
+
+
+def pytest_assertrepr_compare(op: str, left: object, right: object) -> list[str] | None:
+    if isinstance(left, (ChoiceVerdict, ScoreVerdict)):
+        return left.explain(op, right)
+    if isinstance(right, (ChoiceVerdict, ScoreVerdict)) and op in FLIPPED:
+        return right.explain(FLIPPED[op], left)
+    return None
+
+
+def pytest_terminal_summary(terminalreporter: Any, config: pytest.Config) -> None:
+    session = config.stash.get(SESSION, None)
+    if session is None or not session.stats.questions:
+        return
+    terminalreporter.write_sep("-", "jev")
+    terminalreporter.write_line(format_summary(session))
+
+
+def format_summary(session: JevSession) -> str:
+    stats = session.stats
+    requests = f"{stats.requests} request{'' if stats.requests == 1 else 's'}"
+    if stats.cached:
+        requests += f" + {stats.cached} cached"
+    parts = [
+        f"{stats.questions} question{'' if stats.questions == 1 else 's'}",
+        requests,
+        f"{stats.input_tokens:,} input tokens",
+        format_usd(stats.cost_usd),
+        f"{stats.seconds:.2f} s in Jev",
+    ]
+    if stats.models:
+        answered = ", ".join(sorted(stats.models))
+        if stats.providers:
+            answered += f" via {', '.join(sorted(stats.providers))}"
+        parts.append(answered)
+    return "jev: " + " · ".join(parts)
+
+
+def format_usd(cost: float) -> str:
+    """Two significant digits: a run of tests often costs a few millionths of a dollar."""
+    if cost <= 0:
+        return "$0"
+    if cost >= 0.01:
+        return f"${cost:.2f}"
+    return f"${cost:.{1 - math.floor(math.log10(cost))}f}"

pytest_jev/provider.py

@@ -0,0 +1,56 @@
+"""Choose where Jev requests go (OpenRouter or TypeSafe) and which key they use."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+# The TypeSafe SDK appends /v1/systemone to this base URL.
+OPENROUTER_BASE_URL = "https://openrouter.ai/api"
+
+OPENROUTER_KEY_ENV = "OPENROUTER_API_KEY"
+TYPESAFE_KEY_ENV = "TYPESAFE_API_KEY"
+
+PROVIDERS = ("auto", "openrouter", "typesafe")
+
+NO_KEY_MESSAGE = f"no API key found, set {TYPESAFE_KEY_ENV} or {OPENROUTER_KEY_ENV}"
+
+
+class ConfigError(Exception):
+    """A setup problem the user can fix, such as a missing key."""
+
+
+@dataclass(frozen=True)
+class Provider:
+    name: str  # "openrouter" or "typesafe"
+    api_key: str
+    base_url: str | None  # None lets the SDK use TYPESAFE_BASE_URL or its default
+
+
+def env_key(name: str, env: Mapping[str, str] | None = None) -> str | None:
+    value = (os.environ if env is None else env).get(name, "").strip()
+    return value or None
+
+
+def resolve_provider(requested: str = "auto", env: Mapping[str, str] | None = None) -> Provider:
+    """Pick the provider; `auto` prefers OpenRouter when both keys are set."""
+    openrouter_key = env_key(OPENROUTER_KEY_ENV, env)
+    typesafe_key = env_key(TYPESAFE_KEY_ENV, env)
+    if requested == "auto":
+        if openrouter_key:
+            requested = "openrouter"
+        elif typesafe_key:
+            requested = "typesafe"
+        else:
+            raise ConfigError(NO_KEY_MESSAGE)
+
+    if requested == "openrouter":
+        if not openrouter_key:
+            raise ConfigError(f"--jev-provider openrouter needs {OPENROUTER_KEY_ENV} to be set")
+        return Provider("openrouter", openrouter_key, OPENROUTER_BASE_URL)
+    if requested == "typesafe":
+        if not typesafe_key:
+            raise ConfigError(f"--jev-provider typesafe needs {TYPESAFE_KEY_ENV} to be set")
+        return Provider("typesafe", typesafe_key, None)
+    raise ConfigError(f"unknown provider {requested!r}; choose from {', '.join(PROVIDERS)}")

pytest_jev/verdicts.py

@@ -0,0 +1,259 @@
+"""What the `jev` fixture returns, and the text pytest shows when an assertion on it fails."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, overload
+
+# Probabilities are floats: 1 - 0.8 is 0.19999999999999996, and level probabilities sum to 0.9999.
+EPSILON = 1e-9
+BAR_WIDTH = 20
+PREVIEW_CHARS = 120
+
+OPERATORS = (">=", ">", "<=", "<", "==", "!=")
+# The same comparison with the operands swapped: `"neutral" <= tone` means `tone >= "neutral"`.
+FLIPPED = {">=": "<=", ">": "<", "<=": ">=", "<": ">", "==": "==", "!=": "!="}
+
+
+def bar(p: float) -> str:
+    filled = round(max(0.0, min(1.0, p)) * BAR_WIDTH)
+    return "█" * filled + "░" * (BAR_WIDTH - filled)
+
+
+def preview(text: Any) -> str:
+    shown = text if isinstance(text, str) else repr(text)
+    shown = " ".join(shown.split())
+    if len(shown) > PREVIEW_CHARS:
+        shown = shown[: PREVIEW_CHARS - 3] + "..."
+    return repr(shown) if isinstance(text, str) else shown
+
+
+@dataclass(frozen=True)
+class Claim:
+    """A yes/no claim about the text and Jev's probability that it is true. Truthy if it passed.
+
+    `holds` needs p >= threshold; `lacks` needs p <= 1 - threshold. A probability in between
+    fails both: Jev is unsure, and a test should not pass on a coin flip.
+    """
+
+    claim: str
+    expect: str  # "holds" or "lacks"
+    p: float  # probability that the claim is true
+    threshold: float
+
+    @property
+    def passed(self) -> bool:
+        if self.expect == "holds":
+            return self.p >= self.threshold - EPSILON
+        return self.p <= 1 - self.threshold + EPSILON
+
+    @property
+    def needs(self) -> str:
+        if self.expect == "holds":
+            return f">= {self.threshold:.2f}"
+        return f"<= {1 - self.threshold:.2f}"
+
+    def __bool__(self) -> bool:
+        return self.passed
+
+    def __repr__(self) -> str:
+        return f"<jev {self.expect} {self.claim!r}: p={self.p:.2f}, needs {self.needs}>"
+
+    def report_line(self) -> str:
+        mark = "✓" if self.passed else "✗"
+        needs = "" if self.passed else f"  (needs {self.needs})"
+        return f"  {mark} {self.expect}  p={self.p:.2f}  {self.claim}{needs}"
+
+
+class Claims(Sequence[Claim]):
+    """The claims from one `jev.expect` call, all answered in one request."""
+
+    def __init__(self, claims: Sequence[Claim], text: Any) -> None:
+        self._claims = tuple(claims)
+        self.text = text
+
+    @overload
+    def __getitem__(self, index: int) -> Claim: ...
+    @overload
+    def __getitem__(self, index: slice) -> Sequence[Claim]: ...
+    def __getitem__(self, index: int | slice) -> Claim | Sequence[Claim]:
+        return self._claims[index]
+
+    def __len__(self) -> int:
+        return len(self._claims)
+
+    def __iter__(self) -> Iterator[Claim]:
+        return iter(self._claims)
+
+    @property
+    def failed(self) -> list[Claim]:
+        return [claim for claim in self._claims if not claim.passed]
+
+    @property
+    def passed(self) -> bool:
+        return not self.failed
+
+    def __bool__(self) -> bool:
+        return self.passed
+
+    def __repr__(self) -> str:
+        return f"<jev {len(self._claims) - len(self.failed)} of {len(self._claims)} claims passed>"
+
+    def report(self) -> str:
+        failed = len(self.failed)
+        noun = "claim" if len(self._claims) == 1 else "claims"
+        head = (
+            f"jev: {failed} of {len(self._claims)} {noun} failed"
+            if failed
+            else f"jev: all {len(self._claims)} {noun} passed"
+        )
+        lines = [head, f"text: {preview(self.text)}"]
+        lines += [claim.report_line() for claim in self._claims]
+        return "\n".join(lines)
+
+
+@dataclass(frozen=True, eq=False)
+class ChoiceVerdict:
+    """Jev's pick from a set of options. Compare it with an option label: `verdict == "billing"`."""
+
+    question: str
+    choice: str
+    probabilities: Mapping[str, float]  # every option, in the order they were given
+    confidence: float
+
+    def label(self, label: object) -> str:
+        if not isinstance(label, str) or label not in self.probabilities:
+            options = ", ".join(map(repr, self.probabilities))
+            raise ValueError(f"{label!r} is not one of the options: {options}")
+        return label
+
+    def p(self, label: str) -> float:
+        """The probability Jev gave `label`."""
+        return self.probabilities[self.label(label)]
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, str):
+            return self.choice == self.label(other)
+        return NotImplemented
+
+    __hash__ = None  # type: ignore[assignment]
+
+    def ranked(self) -> list[tuple[str, float]]:
+        return sorted(self.probabilities.items(), key=lambda item: -item[1])
+
+    def __repr__(self) -> str:
+        ranked = ", ".join(f"{label} {p:.2f}" for label, p in self.ranked())
+        return f"<jev choice {self.choice!r} ({ranked})>"
+
+    def explain(self, op: str, other: object) -> list[str] | None:
+        if op not in ("==", "!=") or not isinstance(other, str):
+            return None
+        if op == "==":
+            head = f"jev chose {self.choice!r}, not {other!r}"
+        else:
+            head = f"jev chose {self.choice!r}"
+        width = max(len(label) for label in self.probabilities)
+        lines = [head, f"question: {self.question}"]
+        lines += [f"  {label:<{width}}  {p:.2f}  {bar(p)}" for label, p in self.ranked()]
+        lines.append(f"confidence {self.confidence:.2f}")
+        return lines
+
+
+@dataclass(frozen=True, eq=False)
+class ScoreVerdict:
+    """Jev's rating on an ordered rubric. Compare it with a level label or index.
+
+    Comparisons are probabilistic: `tone >= "neutral"` passes when Jev puts at least `threshold`
+    of its probability on "neutral" or a higher level. `expected` is Jev's raw expected level.
+    """
+
+    question: str
+    levels: tuple[str, ...]  # labels, lowest level first
+    probabilities: tuple[float, ...]  # one per level
+    expected: float
+    confidence: float
+    threshold: float
+
+    @property
+    def level(self) -> str:
+        """The most likely level."""
+        best = max(range(len(self.levels)), key=lambda i: self.probabilities[i])
+        return self.levels[best]
+
+    def index(self, level: object) -> int:
+        if isinstance(level, str):
+            if level not in self.levels:
+                levels = ", ".join(map(repr, self.levels))
+                raise ValueError(f"{level!r} is not one of the levels: {levels}")
+            return self.levels.index(level)
+        if isinstance(level, int) and not isinstance(level, bool):
+            if not 0 <= level < len(self.levels):
+                raise ValueError(f"level {level} is out of range 0..{len(self.levels) - 1}")
+            return level
+        raise TypeError(f"compare a score with a level label or index, not {type(level).__name__}")
+
+    def mass(self, op: str, level: object) -> float:
+        """The probability Jev puts on the levels that satisfy `score <op> level`."""
+        i, n = self.index(level), len(self.levels)
+        if op == "!=":
+            return 1 - self.probabilities[i]
+        first, last = {
+            ">=": (i, n),
+            ">": (i + 1, n),
+            "<=": (0, i + 1),
+            "<": (0, i),
+            "==": (i, i + 1),
+        }[op]
+        return sum(self.probabilities[first:last])
+
+    def _holds(self, op: str, level: object) -> bool:
+        return self.mass(op, level) >= self.threshold - EPSILON
+
+    def __ge__(self, level: object) -> bool:
+        return self._holds(">=", level)
+
+    def __gt__(self, level: object) -> bool:
+        return self._holds(">", level)
+
+    def __le__(self, level: object) -> bool:
+        return self._holds("<=", level)
+
+    def __lt__(self, level: object) -> bool:
+        return self._holds("<", level)
+
+    def __eq__(self, level: object) -> bool:
+        if isinstance(level, (int, str)):
+            return self._holds("==", level)
+        return NotImplemented
+
+    def __ne__(self, level: object) -> bool:
+        if isinstance(level, (int, str)):
+            return self._holds("!=", level)
+        return NotImplemented
+
+    __hash__ = None  # type: ignore[assignment]
+
+    def __repr__(self) -> str:
+        spread = ", ".join(f"{p:.2f}" for p in self.probabilities)
+        return f"<jev score {self.level!r} (expected level {self.expected:.2f}; p=[{spread}])>"
+
+    def explain(self, op: str, other: object) -> list[str] | None:
+        if op not in OPERATORS or not isinstance(other, (int, str)) or isinstance(other, bool):
+            return None
+        try:
+            mass = self.mass(op, other)
+        except (ValueError, TypeError):
+            return None
+        target = self.levels[self.index(other)]
+        width = max(len(label) for label in self.levels)
+        lines = [
+            f"jev gave P(level {op} {target!r}) = {mass:.2f}, needs >= {self.threshold:.2f}",
+            f"question: {self.question}",
+        ]
+        lines += [
+            f"  {i} {label:<{width}}  {p:.2f}  {bar(p)}"
+            for i, (label, p) in enumerate(zip(self.levels, self.probabilities, strict=True))
+        ]
+        lines.append(f"expected level {self.expected:.2f}, confidence {self.confidence:.2f}")
+        return lines

pytest_jev-0.1.0.dist-info/METADATA

@@ -0,0 +1,295 @@
+Metadata-Version: 2.5
+Name: pytest-jev
+Version: 0.1.0
+Summary: Semantic assertions for pytest: test what text means, not the exact words, judged by TypeSafe's Jev.
+Project-URL: Homepage, https://github.com/allebee/pytest-jev
+Project-URL: Issues, https://github.com/allebee/pytest-jev/issues
+Author: allebee
+License-Expression: MIT
+License-File: LICENSE
+Keywords: assertions,evals,jev,llm,pytest,semantic,testing,typesafe
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: pytest>=7.4
+Requires-Dist: typesafe-sdk<0.8,>=0.7.1
+Description-Content-Type: text/markdown
+
+# pytest-jev
+
+[![CI](https://github.com/allebee/pytest-jev/actions/workflows/ci.yml/badge.svg)](https://github.com/allebee/pytest-jev/actions/workflows/ci.yml)
+
+**Semantic assertions for pytest.** Test what your LLM app's output *means* ("apologizes",
+"offers a refund", "doesn't leak the system prompt") instead of the exact words. Each claim is
+judged by [Jev](https://docs.typesafe.ai/introduction), TypeSafe's model that returns calibrated
+probabilities instead of text.
+
+```python
+def test_refund_reply(jev):
+    reply = support_bot("I was charged twice for order #1042.")
+
+    jev.expect(
+        reply,
+        holds=["apologizes to the customer", "says the duplicate payment was refunded"],
+        lacks=["blames the customer", "asks for a password or a full card number"],
+    )
+```
+
+When a prompt change breaks the reply, the failure says which claim broke and how sure Jev was:
+
+```
+>       jev.expect(
+E       AssertionError: jev: 3 of 4 claims failed
+E       text: "Double charges happen when you click twice. You'll get store credit within 24 hours."
+E         ✗ holds  p=0.04  apologizes to the customer  (needs >= 0.80)
+E         ✗ holds  p=0.21  says the duplicate payment was refunded  (needs >= 0.80)
+E         ✗ lacks  p=0.79  blames the customer  (needs <= 0.20)
+E         ✓ lacks  p=0.01  asks for a password or a full card number
+------------------------------------- jev --------------------------------------
+jev: 4 questions · 1 request · 365 input tokens · $0.000015 · 1.40 s in Jev · typesafe/jev-1.13-20260917 via openrouter
+```
+
+Every output in this README is from a real run of [`examples/`](https://github.com/allebee/pytest-jev/tree/main/examples) against `jev-1.13`.
+
+## Why
+
+String assertions break every time the model rewords a reply. Using an LLM as the judge works,
+but it's slow, costs real money per test, and returns text you then have to parse. pytest-jev
+sends each check to Jev instead:
+
+- **One request per text.** Every claim in `jev.expect` goes into a single request, answered in
+  parallel.
+- **Typed answers.** Jev answers each claim with a probability, picks from the options you list, or
+  rates on the levels you define. There is no output to parse and nothing outside the answer space.
+- **Cheap enough for every commit.** Jev costs $0.042 per million input tokens and output is free.
+  The 7 tests in [`examples/test_support_bot.py`](https://github.com/allebee/pytest-jev/blob/main/examples/test_support_bot.py) ran in 3.9 s for
+  $0.0001. The summary line prints the tokens and cost of every run.
+- **No passing on a coin flip.** A claim `holds` at p ≥ 0.8 and `lacks` at p ≤ 0.2. When Jev is
+  unsure, both fail.
+- **Free, stable reruns.** Answers are cached in `.pytest_cache`, so rerunning unchanged tests
+  makes no requests and gives the same verdicts.
+
+## Install
+
+```bash
+pip install pytest-jev
+```
+
+It needs Python 3.10+ and pytest 7.4+.
+
+Set one API key:
+
+```bash
+export TYPESAFE_API_KEY=...     # https://console.typesafe.ai (early access)
+export OPENROUTER_API_KEY=...   # or https://openrouter.ai/settings/keys (no waitlist)
+```
+
+Without a key, tests that use `jev` are skipped (see [CI](#ci-and-running-without-a-key)).
+
+## Usage
+
+The `jev` fixture has five methods. Each sends one request and returns a result that works in a
+plain `assert`.
+
+### `holds` and `lacks`: one claim
+
+```python
+def test_reply_confirms_the_refund(jev):
+    reply = support_bot("I was charged twice for order #1042.")
+    assert jev.holds(reply, "says the duplicate payment was refunded")
+    assert jev.lacks(reply, "asks for a password or a full card number")
+```
+
+With the broken reply from above:
+
+```
+E       assert <jev holds 'says the duplicate payment was refunded': p=0.16, needs >= 0.80>
+```
+
+The result also carries the probability: `jev.holds(reply, "...").p`.
+
+### `expect`: many claims, one request
+
+```python
+jev.expect(reply, holds=["apologizes", "offers a refund"], lacks=["blames the customer"])
+```
+
+It fails with a report of every claim, as shown at the top. It returns the claims when they pass.
+
+### `context`: check the text against something else
+
+Extra state goes in `context`, such as a policy or the documents a RAG app retrieved. A claim can
+name it in backticks:
+
+```python
+def test_reply_matches_the_policy(jev):
+    reply = support_bot("I was charged twice for order #1042.")
+    assert jev.lacks(reply, "contradicts the policy in `policy`", context={"policy": REFUND_POLICY})
+```
+
+The broken reply promises store credit in 24 hours; the policy says refunds to the card in 5
+business days:
+
+```
+E       assert <jev lacks 'contradicts the policy in `policy`': p=0.95, needs <= 0.20>
+```
+
+The text under test is always `text`, so `context` can't use that key.
+
+### `choice`: which option fits
+
+```python
+TEAMS = {
+    "billing": "Payments, charges, invoices and refunds",
+    "technical": "Bugs, errors, crashes and integrations",
+    "account": "Logins, passwords and account settings",
+    "other": "Anything that fits none of the teams above",
+}
+
+
+def test_checkout_errors_go_to_billing(jev):
+    ticket = "Your checkout page throws a 500 error when I enter my card."
+    assert jev.choice(ticket, "Which team should handle this ticket?", TEAMS) == "billing"
+```
+
+```
+E       AssertionError: assert jev chose 'technical', not 'billing'
+E         question: Which team should handle this ticket?
+E           technical  0.90  ██████████████████░░
+E           billing    0.10  ██░░░░░░░░░░░░░░░░░░
+E           account    0.00  ░░░░░░░░░░░░░░░░░░░░
+E           other      0.00  ░░░░░░░░░░░░░░░░░░░░
+E         confidence 0.87
+```
+
+Sometimes the failure means the test's expectation needs another look: a 500 error at checkout is
+arguably a bug first.
+
+Options can be a dict of label to description, or a plain list of labels. Comparing with a label
+that isn't an option (`team == "biling"`) raises an error instead of quietly failing.
+
+### `score`: rate on ordered levels
+
+```python
+POLITENESS = {
+    "rude": "Rude, dismissive or blaming the customer",
+    "neutral": "Neutral and matter-of-fact, no warmth",
+    "warm": "Warm and polite, acknowledges the customer's frustration",
+}
+
+
+def test_reply_is_warm(jev):
+    tone = jev.score(reply, "How polite is this support reply?", POLITENESS)
+    assert tone >= "warm"
+```
+
+Levels go lowest first. The comparison is probabilistic: `tone >= "warm"` passes when Jev puts at
+least 80% of its probability on "warm" or higher. `>`, `<=`, `<`, `==` and `!=` work the same way,
+with labels or level indices. The broken reply passes `tone >= "neutral"` (0.83) but not this:
+
+```
+E       AssertionError: assert jev gave P(level >= 'warm') = 0.00, needs >= 0.80
+E         question: How polite is this support reply?
+E           0 rude     0.17  ███░░░░░░░░░░░░░░░░░
+E           1 neutral  0.83  █████████████████░░░
+E           2 warm     0.00  ░░░░░░░░░░░░░░░░░░░░
+E         expected level 0.84, confidence 0.75
+```
+
+## Thresholds and models
+
+The threshold defaults to 0.8: `holds` needs p ≥ 0.8, `lacks` needs p ≤ 0.2. It must be between 0.5
+and 1. Set it per call, per test, or for the whole run:
+
+```python
+assert jev.holds(reply, "offers a refund", threshold=0.9)
+
+
+@pytest.mark.jev(threshold=0.9, model="jev-1.13")
+def test_strict(jev): ...
+```
+
+```ini
+# pytest.ini (or [tool.pytest.ini_options] in pyproject.toml)
+[pytest]
+jev_model = jev-1.13
+jev_threshold = 0.85
+```
+
+`jev-latest` changes when TypeSafe ships a new version, so pin `jev-1.13` when runs must be
+reproducible.
+
+| Option | ini | Default | |
+|---|---|---|---|
+| `--jev-model` | `jev_model` | `jev-latest` | Jev model to ask |
+| `--jev-threshold` | `jev_threshold` | `0.8` | Claim threshold |
+| `--jev-provider` | `jev_provider` | `auto` | `typesafe`, `openrouter`, or `auto` (OpenRouter if its key is set) |
+| `--jev-no-cache` | | off | Ask again instead of reusing cached answers |
+| `--jev-require` | `jev_require` | off | Fail instead of skip when no key is set |
+
+## CI and running without a key
+
+- Tests that use `jev` get the `jev` marker automatically. `pytest -m "not jev"` runs everything
+  else offline.
+- Without a key, `jev` tests are **skipped** and say why. In CI, pass `--jev-require` (or set
+  `jev_require = true`) so a missing secret fails the build instead.
+- Answers are cached by model, text, context and question. Pass `--jev-no-cache` to ask again.
+
+## Use another backend
+
+Requests go through the session-scoped `jev_client` fixture. Override it in `conftest.py` with
+anything that has the TypeSafe SDK's `system_one(state=, questions=, model=)` method, such as a fake
+for offline unit tests, or [system-one-adapter](https://github.com/typesafe-ai/system-one-adapter-python)
+to run the same assertions through an LLM and compare:
+
+```python
+@pytest.fixture(scope="session")
+def jev_client():
+    return MyFakeJev()
+```
+
+## Writing claims that work
+
+Jev reads claims literally ([Jev 1.13 known limits](https://docs.typesafe.ai/model-jaggedness/jev-1.13)):
+
+- **One condition per claim.** Write "apologizes" and "offers a refund" as two claims, not one
+  joined with "and".
+- **Say exactly what you mean.** "Says the duplicate payment was refunded" works better than
+  "handles the refund correctly".
+- **Keep numbers, counts and dates in code.** `assert "5 business days" in reply` is exact; Jev is
+  not a calculator.
+- **Name the context.** "contradicts `docs`" points Jev at the right part of the state.
+
+## How it works
+
+Each call sends one request to Jev's `/v1/systemone` endpoint with `state = {"text": text,
+**context}`. Every claim becomes a Noul question, ``Does `text` satisfy: <claim>?``, which returns
+the probability it is true. `choice` sends a Choice question and `score` sends a Score question.
+The thresholds and comparisons are ordinary Python in this plugin.
+
+## Limitations
+
+- Jev can be wrong. Treat a threshold as a policy you tune on your own cases, and read the failure
+  report before trusting a pass or fail.
+- Jev's probabilities move a little between calls. In five calls while this README was written,
+  "says the duplicate payment was refunded" scored between 0.16 and 0.23 on the same reply. The unsure band between 0.2 and 0.8 absorbs
+  this, and the cache keeps reruns identical.
+- Text only: no images or audio.
+- The text and context you assert on are sent to TypeSafe or OpenRouter. Keep secrets and personal
+  data out of test fixtures.
+- Not affiliated with or endorsed by TypeSafe AI.
+
+## Development
+
+```bash
+uv sync
+uv run pytest          # offline: a fake Jev answers every question
+uv run ruff check .
+```
+
+## License
+
+MIT

pytest_jev-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+pytest_jev/__init__.py,sha256=lR5ZRWlWK9VyxUd4uvJ9T1wvyxIyIx0JMVjdt_Wf33Y,282
+pytest_jev/judge.py,sha256=mKKpSIlJoKXNc1EzPsYelJqeuvWzCmKmX3-zEaUwIoM,13581
+pytest_jev/plugin.py,sha256=FKs8r7X4H4K_p07fJlBSOEU1C0h4DTuPSZocnLRqNao,5818
+pytest_jev/provider.py,sha256=aqwR0LisIDGO4FcqN-9UwIG-C07xYFo63viVFVhj8ZI,2022
+pytest_jev/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pytest_jev/verdicts.py,sha256=dbL63I5U92pLJmtGrGeSxmW5Kb8eIgURNyOmeiqKxwA,9296
+pytest_jev-0.1.0.dist-info/METADATA,sha256=l-hiSUZf09cY7iuuuSFQd5NI-u5M_4Q_zD10s4CWtlw,11668
+pytest_jev-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+pytest_jev-0.1.0.dist-info/entry_points.txt,sha256=qepy4DZCOZny6tbd53MfEiONz6kK21t1fmXoK2Pl4Yo,35
+pytest_jev-0.1.0.dist-info/licenses/LICENSE,sha256=pd8fkxTemEajmHsyHVVp2IYDqQPmWGk-A9IqinkZETc,1064
+pytest_jev-0.1.0.dist-info/RECORD,,

pytest_jev-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

pytest_jev-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+jev = pytest_jev.plugin

pytest_jev-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 allebee
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.