jevkit-pytest 0.1.0__tar.gz

jevkit_pytest-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.coverage
+htmlcov/
+.ruff_cache/
+uv.lock

jevkit_pytest-0.1.0/PKG-INFO

@@ -0,0 +1,103 @@
+Metadata-Version: 2.5
+Name: jevkit-pytest
+Version: 0.1.0
+Summary: Record and replay TypeSafe Jev requests in pytest. Cassettes are .jevl files, so a test recording doubles as a drift golden set.
+Project-URL: Homepage, https://github.com/pjdurden/jevkit-py
+Project-URL: Issues, https://github.com/pjdurden/jevkit-py/issues
+Author: Prajjwal Chittori
+License-Expression: MIT
+Keywords: cassette,jev,pytest,system-one,testing,typesafe,vcr
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: jevkit-core>=0.2.0
+Requires-Dist: pytest>=7.0
+Description-Content-Type: text/markdown
+
+# jevkit-pytest
+
+Record and replay TypeSafe Jev requests in pytest.
+
+A model call in a test is slow, costs money, needs a key in CI, and can change
+its answer under you when the alias moves. The fix is the one VCR established for
+HTTP: record real responses once, replay them forever, re-record on purpose.
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+pip install jevkit-pytest
+```
+
+## Use
+
+```python
+def test_routes_billing_questions(jev_cassette):
+    response = jev_cassette.system_one(
+        "I was charged twice for the same order",
+        {"team": {"type": "choice", "instructions": "Which team should handle this",
+                  "criteria": {"billing": "Payment issues", "technical": "Bugs",
+                               "unknown": "None apply"}}},
+    )
+    assert response.answers["team"]["choice"] == "billing"
+```
+
+Record the first time, then never again:
+
+```bash
+pytest --jev-record       # record anything missing
+pytest                    # replay only; a miss is a failure
+pytest --jev-rerecord     # replace every recording
+```
+
+To record you need a live client. Supply one by overriding the `jev_client`
+fixture in your `conftest.py`:
+
+```python
+import pytest
+from typesafe_sdk import TypeSafeClient
+
+@pytest.fixture
+def jev_client():
+    with TypeSafeClient() as client:
+        yield client
+```
+
+Replay-only runs need no client and no API key, which is the point: CI stays
+green without a secret.
+
+## Assertions that explain themselves
+
+```python
+from jevkit_pytest import assert_answer, assert_confident
+
+assert_answer(response.answers["team"], "billing", min_probability=0.6)
+assert_confident(response.answers["urgency"], 0.8)
+```
+
+On failure these print the whole distribution, because a 0.51/0.49 split and a
+0.99/0.01 split are different bugs and `assert x == y` cannot tell them apart.
+
+`assert_confident` uses the API's confidence for Choice and Score. A Noul carries
+none, so its distance from 0.5 is used and the message says which it used.
+
+## Cassettes are golden sets
+
+A cassette is a `.jevl` file, the same format `jevkit-drift`, `jevkit-bench` and
+`jevkit-calibrate` read. So the recordings your tests already make are a golden
+set you can replay against the next model version:
+
+```bash
+jevkit-drift tests/cassettes/test_routes_billing_questions.jevl candidate.jevl
+```
+
+That is the whole reason the format was defined before any of these packages.
+
+## License
+
+MIT

jevkit_pytest-0.1.0/README.md

@@ -0,0 +1,81 @@
+# jevkit-pytest
+
+Record and replay TypeSafe Jev requests in pytest.
+
+A model call in a test is slow, costs money, needs a key in CI, and can change
+its answer under you when the alias moves. The fix is the one VCR established for
+HTTP: record real responses once, replay them forever, re-record on purpose.
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+pip install jevkit-pytest
+```
+
+## Use
+
+```python
+def test_routes_billing_questions(jev_cassette):
+    response = jev_cassette.system_one(
+        "I was charged twice for the same order",
+        {"team": {"type": "choice", "instructions": "Which team should handle this",
+                  "criteria": {"billing": "Payment issues", "technical": "Bugs",
+                               "unknown": "None apply"}}},
+    )
+    assert response.answers["team"]["choice"] == "billing"
+```
+
+Record the first time, then never again:
+
+```bash
+pytest --jev-record       # record anything missing
+pytest                    # replay only; a miss is a failure
+pytest --jev-rerecord     # replace every recording
+```
+
+To record you need a live client. Supply one by overriding the `jev_client`
+fixture in your `conftest.py`:
+
+```python
+import pytest
+from typesafe_sdk import TypeSafeClient
+
+@pytest.fixture
+def jev_client():
+    with TypeSafeClient() as client:
+        yield client
+```
+
+Replay-only runs need no client and no API key, which is the point: CI stays
+green without a secret.
+
+## Assertions that explain themselves
+
+```python
+from jevkit_pytest import assert_answer, assert_confident
+
+assert_answer(response.answers["team"], "billing", min_probability=0.6)
+assert_confident(response.answers["urgency"], 0.8)
+```
+
+On failure these print the whole distribution, because a 0.51/0.49 split and a
+0.99/0.01 split are different bugs and `assert x == y` cannot tell them apart.
+
+`assert_confident` uses the API's confidence for Choice and Score. A Noul carries
+none, so its distance from 0.5 is used and the message says which it used.
+
+## Cassettes are golden sets
+
+A cassette is a `.jevl` file, the same format `jevkit-drift`, `jevkit-bench` and
+`jevkit-calibrate` read. So the recordings your tests already make are a golden
+set you can replay against the next model version:
+
+```bash
+jevkit-drift tests/cassettes/test_routes_billing_questions.jevl candidate.jevl
+```
+
+That is the whole reason the format was defined before any of these packages.
+
+## License
+
+MIT

jevkit_pytest-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "jevkit-pytest"
+version = "0.1.0"
+description = "Record and replay TypeSafe Jev requests in pytest. Cassettes are .jevl files, so a test recording doubles as a drift golden set."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "Prajjwal Chittori" }]
+keywords = ["jev", "typesafe", "system-one", "pytest", "testing", "cassette", "vcr"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Framework :: Pytest",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = ["jevkit-core>=0.2.0", "pytest>=7.0"]
+
+[project.entry-points.pytest11]
+jevkit = "jevkit_pytest.plugin"
+
+[project.urls]
+Homepage = "https://github.com/pjdurden/jevkit-py"
+Issues = "https://github.com/pjdurden/jevkit-py/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/jevkit_pytest"]

jevkit_pytest-0.1.0/src/jevkit_pytest/__init__.py

@@ -0,0 +1,13 @@
+"""Record and replay TypeSafe Jev requests in pytest.
+
+Cassettes are `.jevl` files, the same format the drift, bench and calibrate
+packages read, so a recording made by your tests doubles as a golden set.
+"""
+
+from .assertions import assert_answer, assert_confident, describe_answer
+from .cassette import Cassette, CassetteMiss
+
+__version__ = "0.1.0"
+
+__all__ = ["Cassette", "CassetteMiss", "assert_answer", "assert_confident",
+           "describe_answer", "__version__"]

jevkit_pytest-0.1.0/src/jevkit_pytest/assertions.py

@@ -0,0 +1,83 @@
+"""Assertions over jev answers, with failure messages that say enough.
+
+A bare ``assert answer.choice == "billing"`` tells you nothing about *how*
+close the call was. These assertions print the distribution on failure, because
+a 0.51/0.49 split and a 0.99/0.01 split are different bugs.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from jevkit_core import parse_answer
+
+__all__ = ["assert_answer", "assert_confident", "describe_answer"]
+
+
+def describe_answer(qid: str, raw: Any) -> str:
+    answer = parse_answer(qid, raw)
+    probs = ", ".join(
+        f"{k}={v:.3f}" for k, v in sorted(answer.probabilities.items(), key=lambda kv: -kv[1])
+    )
+    parts = [f"predicted={answer.predicted()!r}"]
+    if answer.confidence is not None:
+        parts.append(f"confidence={answer.confidence:.3f}")
+    if answer.score is not None:
+        parts.append(f"score={answer.score:.3f}")
+    return f"{qid}: {', '.join(parts)}\n    probabilities: {probs}"
+
+
+def assert_answer(
+    raw: Any,
+    expected: Any,
+    *,
+    question_id: str = "answer",
+    min_confidence: float | None = None,
+    min_probability: float | None = None,
+) -> None:
+    """Assert an answer selected ``expected``, optionally with enough certainty."""
+    answer = parse_answer(question_id, raw)
+    predicted = answer.predicted()
+
+    if not answer.is_correct(expected):
+        raise AssertionError(
+            f"expected {expected!r} but got {predicted!r}\n  {describe_answer(question_id, raw)}"
+        )
+
+    if min_probability is not None:
+        actual = answer.probability_of(expected)
+        if actual < min_probability:
+            raise AssertionError(
+                f"{expected!r} was selected but carried only {actual:.3f} probability, "
+                f"below the required {min_probability:.3f}\n"
+                f"  {describe_answer(question_id, raw)}"
+            )
+
+    if min_confidence is not None:
+        if answer.confidence is None:
+            raise AssertionError(
+                f"min_confidence was given but a {answer.type} answer carries no "
+                f"confidence. Use min_probability instead."
+            )
+        if answer.confidence < min_confidence:
+            raise AssertionError(
+                f"{expected!r} was selected but confidence was {answer.confidence:.3f}, "
+                f"below the required {min_confidence:.3f}\n"
+                f"  {describe_answer(question_id, raw)}"
+            )
+
+
+def assert_confident(raw: Any, minimum: float, *, question_id: str = "answer") -> None:
+    """Assert an answer is decisive, without caring which way it went.
+
+    Uses the API's confidence for Choice and Score. A Noul has none, so its
+    distance from 0.5 is used and the message says so.
+    """
+    answer = parse_answer(question_id, raw)
+    value = answer.decisiveness
+    if value < minimum:
+        quantity = "decisiveness (|noul - 0.5| * 2)" if answer.type == "noul" else "confidence"
+        raise AssertionError(
+            f"{quantity} was {value:.3f}, below the required {minimum:.3f}\n"
+            f"  {describe_answer(question_id, raw)}"
+        )

jevkit_pytest-0.1.0/src/jevkit_pytest/cassette.py

@@ -0,0 +1,204 @@
+"""Record and replay jev requests so tests do not hit the API.
+
+A model call in a test is slow, costs money, needs a key in CI, and can change
+its answer under you when the alias moves. The fix is the one VCR established
+for HTTP: record real responses once, replay them forever, re-record on purpose.
+
+A cassette is a `.jevl` file, the same format `drift`, `bench` and `calibrate`
+read, so a recording made by your test suite is also a golden set you can replay
+against the next model version.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Callable
+
+from jevkit_core import Record, append_record, read_records, record_id
+
+__all__ = ["Cassette", "CassetteMiss", "Mode"]
+
+Mode = str  # "replay" | "record" | "auto" | "passthrough"
+
+_MODES = ("replay", "record", "auto", "passthrough")
+
+
+class CassetteMiss(LookupError):
+    """A request was not on the cassette and the mode forbids recording."""
+
+
+def _response_to_parts(response: Any) -> tuple[str, dict[str, Any], dict[str, Any] | None]:
+    """Pull (model, answers, usage) out of an SDK response or a plain dict."""
+    if isinstance(response, dict):
+        model, answers, usage = response.get("model", ""), response.get("answers", {}), response.get("usage")
+    else:
+        model = getattr(response, "model", "")
+        answers = getattr(response, "answers", {})
+        usage = getattr(response, "usage", None)
+
+    def plain(value: Any) -> Any:
+        if isinstance(value, dict):
+            return value
+        if hasattr(value, "model_dump"):
+            return value.model_dump()
+        if hasattr(value, "__dict__"):
+            return {k: v for k, v in vars(value).items() if not k.startswith("_")}
+        return value
+
+    return str(model), {qid: plain(a) for qid, a in (answers or {}).items()}, (
+        plain(usage) if usage is not None else None
+    )
+
+
+class ReplayedAnswer(dict):
+    """A replayed answer.
+
+    Subclasses ``dict`` so ``answer["choice"]`` works, and mirrors the keys onto
+    attributes so ``answer.choice`` works too. Test code written against either
+    the SDK objects or the raw JSON keeps working without a shim.
+    """
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            return self[name]
+        except KeyError as exc:
+            raise AttributeError(
+                f"answer has no field {name!r}; recorded fields are "
+                f"{', '.join(sorted(self)) or '(none)'}"
+            ) from exc
+
+
+class ReplayedResponse:
+    """What a cassette hands back in place of a live API response."""
+
+    def __init__(self, record: Record) -> None:
+        self._record = record
+        self.model = record.model
+        self.answers = {qid: ReplayedAnswer(a) for qid, a in record.answers.items()}
+        self.usage = record.usage
+        self.id = record.id
+
+    def __repr__(self) -> str:  # pragma: no cover - debug aid
+        return f"ReplayedResponse(model={self.model!r}, answers={list(self.answers)})"
+
+
+class Cassette:
+    """A recorded set of jev requests, replayed by request digest.
+
+    ``mode`` controls what happens on a miss:
+
+    - ``replay``      raise ``CassetteMiss``. The right default for CI.
+    - ``auto``        replay a hit, call through and append on a miss.
+    - ``record``      always call through and append, ignoring existing entries.
+    - ``passthrough`` never touch the cassette.
+    """
+
+    def __init__(
+        self,
+        path: str | os.PathLike[str],
+        system_one: Callable[..., Any] | None = None,
+        *,
+        mode: Mode = "replay",
+        model: str = "jev-latest",
+    ) -> None:
+        if mode not in _MODES:
+            raise ValueError(f"mode must be one of {_MODES}, got {mode!r}")
+        self.path = Path(path)
+        self.mode = mode
+        self.model = model
+        self._system_one = system_one
+        self._entries = self._load() if self.path.exists() and mode != "record" else {}
+        self.played: list[str] = []
+        self.recorded: list[str] = []
+
+    def _load(self) -> dict[str, Record]:
+        """Index the file by *requested*-model digest.
+
+        A record stores the model that actually answered, which is what the
+        format requires: `jev-latest` is an alias and the response says
+        `jev-1.13.0`. But a test asks for the alias, so indexing on the answering
+        model would miss every lookup. The requested model is kept in `meta` at
+        record time and used to rebuild the index here.
+
+        Later entries win, so re-recording a request appends rather than
+        requiring a rewrite.
+        """
+        entries: dict[str, Record] = {}
+        for record in read_records(self.path):
+            requested = str(record.meta.get("requested_model") or record.model)
+            entries[record_id(requested, record.state, record.questions)] = record
+        return entries
+
+    # -- introspection -----------------------------------------------------
+
+    def __len__(self) -> int:
+        return len(self._entries)
+
+    @property
+    def unplayed(self) -> list[str]:
+        """Entries on the cassette that no test asked for.
+
+        Usually means a test was deleted or a question was reworded, leaving a
+        stale recording that will quietly rot.
+        """
+        return sorted(set(self._entries) - set(self.played))
+
+    def contains(self, state: Any, questions: dict[str, Any], *, model: str | None = None) -> bool:
+        return record_id(model or self.model, state, questions) in self._entries
+
+    # -- the call ----------------------------------------------------------
+
+    def system_one(self, state: Any, questions: dict[str, Any], **kwargs: Any) -> Any:
+        """Drop-in for a client's ``system_one``.
+
+        Signature matches what both official SDKs expose, so a cassette can be
+        passed anywhere a client is expected in a test.
+        """
+        model = kwargs.pop("model", None) or self.model
+
+        if self.mode == "passthrough":
+            return self._call_through(state, questions, model=model, **kwargs)
+
+        key = record_id(model, state, questions)
+
+        if self.mode != "record":
+            hit = self._entries.get(key)
+            if hit is not None:
+                self.played.append(key)
+                return ReplayedResponse(hit)
+
+        if self.mode == "replay":
+            raise CassetteMiss(
+                f"no recording for this request on {self.path}.\n"
+                f"  digest: {key}\n"
+                f"  Re-run with mode='auto' (or --jev-record) to record it, and commit the "
+                f"updated cassette."
+            )
+
+        response = self._call_through(state, questions, model=model, **kwargs)
+        actual_model, answers, usage = _response_to_parts(response)
+        record = Record(
+            model=actual_model or model,
+            state=state,
+            questions=questions,
+            answers=answers,
+            usage=usage,
+            meta={"requested_model": model},
+        )
+        append_record(self.path, record)
+        self._entries[key] = record
+        # The live response is returned rather than the record: recording must
+        # not change what the code under test sees.
+        self.recorded.append(key)
+        return response
+
+    __call__ = system_one
+
+    def _call_through(self, state: Any, questions: dict[str, Any], **kwargs: Any) -> Any:
+        if self._system_one is None:
+            raise CassetteMiss(
+                f"cassette {self.path} is in mode {self.mode!r} and needs to call the API, "
+                f"but no client was supplied. Pass system_one= when constructing it."
+            )
+        return self._system_one(state, questions, **kwargs)

jevkit_pytest-0.1.0/src/jevkit_pytest/plugin.py

@@ -0,0 +1,90 @@
+"""The pytest plugin: fixtures and the ``--jev-record`` flag."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Iterator
+
+import pytest
+
+from .assertions import assert_answer
+from .cassette import Cassette
+
+__all__ = ["jev_cassette", "jev_cassette_dir", "jev_client"]
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("jev", "TypeSafe Jev cassettes")
+    group.addoption(
+        "--jev-record", action="store_true", default=False,
+        help="record any request missing from its cassette (mode 'auto')",
+    )
+    group.addoption(
+        "--jev-rerecord", action="store_true", default=False,
+        help="re-record every request, replacing the cassettes (mode 'record')",
+    )
+    group.addoption(
+        "--jev-cassette-dir", action="store", default=None,
+        help="where cassettes live (default: tests/cassettes next to the test file)",
+    )
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line("markers", "jev_cassette(name): use a named cassette file")
+
+
+def _mode(config: pytest.Config) -> str:
+    if config.getoption("--jev-rerecord"):
+        return "record"
+    if config.getoption("--jev-record"):
+        return "auto"
+    return "replay"
+
+
+@pytest.fixture(scope="session")
+def jev_cassette_dir(request: pytest.FixtureRequest) -> Path:
+    """Directory holding cassettes. Override in a conftest to relocate them."""
+    configured = request.config.getoption("--jev-cassette-dir")
+    if configured:
+        return Path(configured)
+    return Path(str(request.config.rootpath)) / "tests" / "cassettes"
+
+
+@pytest.fixture
+def jev_client() -> Any:
+    """The live client, used only when recording.
+
+    Override this in your conftest to return something with a ``system_one``
+    method. Left as ``None`` a replay-only run still works; a recording run
+    fails with a message saying exactly this.
+    """
+    return None
+
+
+@pytest.fixture
+def jev_cassette(
+    request: pytest.FixtureRequest,
+    jev_cassette_dir: Path,
+    jev_client: Any,
+) -> Iterator[Cassette]:
+    """A cassette scoped to the current test.
+
+    The file is named after the test by default, so one test's recordings never
+    collide with another's. Override with ``@pytest.mark.jev_cassette("name")``
+    to share one cassette across several tests.
+    """
+    marker = request.node.get_closest_marker("jev_cassette")
+    name = marker.args[0] if marker and marker.args else request.node.name
+    safe = "".join(c if c.isalnum() or c in "-_." else "_" for c in str(name))
+
+    jev_cassette_dir.mkdir(parents=True, exist_ok=True)
+    path = jev_cassette_dir / f"{safe}.jevl"
+
+    system_one = getattr(jev_client, "system_one", None) if jev_client is not None else None
+    cassette = Cassette(path, system_one, mode=_mode(request.config))
+    yield cassette
+
+
+# Re-exported so ``from jevkit_pytest import assert_answer`` works after the
+# plugin is loaded as an entry point.
+__all__ += ["assert_answer"]

jevkit_pytest-0.1.0/tests/test_assertions.py

@@ -0,0 +1,49 @@
+import pytest
+from jevkit_pytest import assert_answer, assert_confident, describe_answer
+
+CHOICE = {"type": "choice", "choice": "billing",
+          "probabilities": {"billing": 0.55, "technical": 0.45}, "confidence": 0.3}
+NOUL = {"type": "noul", "noul": 0.99}
+
+
+def test_a_matching_answer_passes():
+    assert_answer(CHOICE, "billing")
+
+
+def test_a_mismatch_reports_the_distribution():
+    with pytest.raises(AssertionError) as exc:
+        assert_answer(CHOICE, "technical")
+    assert "probabilities" in str(exc.value)
+    assert "billing=0.550" in str(exc.value)
+
+
+def test_min_probability_catches_a_narrow_win():
+    assert_answer(CHOICE, "billing", min_probability=0.5)
+    with pytest.raises(AssertionError, match="carried only"):
+        assert_answer(CHOICE, "billing", min_probability=0.8)
+
+
+def test_min_confidence_catches_a_low_confidence_win():
+    with pytest.raises(AssertionError, match="confidence was"):
+        assert_answer(CHOICE, "billing", min_confidence=0.8)
+
+
+def test_min_confidence_on_a_noul_explains_the_alternative():
+    with pytest.raises(AssertionError, match="carries no\\s+confidence"):
+        assert_answer(NOUL, True, min_confidence=0.5)
+
+
+def test_assert_confident_uses_decisiveness_for_a_noul():
+    assert_confident(NOUL, 0.9)
+    with pytest.raises(AssertionError, match="decisiveness"):
+        assert_confident({"type": "noul", "noul": 0.52}, 0.5)
+
+
+def test_assert_confident_uses_confidence_for_a_choice():
+    with pytest.raises(AssertionError, match="confidence was"):
+        assert_confident(CHOICE, 0.9)
+
+
+def test_describe_orders_probabilities_by_mass():
+    out = describe_answer("team", CHOICE)
+    assert out.index("billing=") < out.index("technical=")

jevkit_pytest-0.1.0/tests/test_cassette.py

@@ -0,0 +1,164 @@
+import json
+import pytest
+from jevkit_core import Record, read_records
+from jevkit_pytest import Cassette, CassetteMiss, assert_answer, assert_confident
+
+QUESTIONS = {"team": {"type": "choice", "instructions": "Which team handles this"}}
+STATE = "I was charged twice"
+
+LIVE = {"model": "jev-1.13.0",
+        "answers": {"team": {"type": "choice", "choice": "billing",
+                             "probabilities": {"billing": 0.9, "technical": 0.1},
+                             "confidence": 0.8}},
+        "usage": {"input_tokens": 12}}
+
+
+def client(calls):
+    def system_one(state, questions, **kw):
+        calls.append((state, questions))
+        return LIVE
+    return system_one
+
+
+def test_replay_mode_raises_on_a_miss(tmp_path):
+    cassette = Cassette(tmp_path / "c.jevl", mode="replay")
+    with pytest.raises(CassetteMiss, match="no recording"):
+        cassette.system_one(STATE, QUESTIONS)
+
+
+def test_auto_mode_records_then_replays(tmp_path):
+    path = tmp_path / "c.jevl"
+    calls = []
+
+    first = Cassette(path, client(calls), mode="auto")
+    first.system_one(STATE, QUESTIONS)
+    assert len(calls) == 1
+    assert path.exists()
+
+    second = Cassette(path, client(calls), mode="auto")
+    response = second.system_one(STATE, QUESTIONS)
+    assert len(calls) == 1, "second run must not call through"
+    assert response.answers["team"]["choice"] == "billing"
+
+
+def test_recording_returns_the_live_response_unchanged(tmp_path):
+    cassette = Cassette(tmp_path / "c.jevl", client([]), mode="auto")
+    assert cassette.system_one(STATE, QUESTIONS) is LIVE
+
+
+def test_replayed_answers_support_both_key_and_attribute_access(tmp_path):
+    path = tmp_path / "c.jevl"
+    Cassette(path, client([]), mode="auto").system_one(STATE, QUESTIONS)
+    response = Cassette(path, mode="replay").system_one(STATE, QUESTIONS)
+    assert response.answers["team"]["choice"] == "billing"
+    assert response.answers["team"].choice == "billing"
+    assert response.answers["team"].confidence == 0.8
+
+
+def test_unknown_field_gives_a_helpful_attribute_error(tmp_path):
+    path = tmp_path / "c.jevl"
+    Cassette(path, client([]), mode="auto").system_one(STATE, QUESTIONS)
+    response = Cassette(path, mode="replay").system_one(STATE, QUESTIONS)
+    with pytest.raises(AttributeError, match="recorded fields are"):
+        _ = response.answers["team"].nonsense
+
+
+def test_a_changed_question_is_a_miss(tmp_path):
+    path = tmp_path / "c.jevl"
+    Cassette(path, client([]), mode="auto").system_one(STATE, QUESTIONS)
+    reworded = {"team": {"type": "choice", "instructions": "Which department handles this"}}
+    with pytest.raises(CassetteMiss):
+        Cassette(path, mode="replay").system_one(STATE, reworded)
+
+
+def test_record_mode_always_calls_through(tmp_path):
+    path = tmp_path / "c.jevl"
+    calls = []
+    Cassette(path, client(calls), mode="auto").system_one(STATE, QUESTIONS)
+    Cassette(path, client(calls), mode="record").system_one(STATE, QUESTIONS)
+    assert len(calls) == 2
+
+
+def test_passthrough_never_writes(tmp_path):
+    path = tmp_path / "c.jevl"
+    Cassette(path, client([]), mode="passthrough").system_one(STATE, QUESTIONS)
+    assert not path.exists()
+
+
+def test_recording_without_a_client_explains_itself(tmp_path):
+    cassette = Cassette(tmp_path / "c.jevl", mode="auto")
+    with pytest.raises(CassetteMiss, match="no client was supplied"):
+        cassette.system_one(STATE, QUESTIONS)
+
+
+def test_invalid_mode_is_rejected(tmp_path):
+    with pytest.raises(ValueError, match="mode must be one of"):
+        Cassette(tmp_path / "c.jevl", mode="nonsense")
+
+
+def test_unplayed_entries_are_reported(tmp_path):
+    path = tmp_path / "c.jevl"
+    Cassette(path, client([]), mode="auto").system_one(STATE, QUESTIONS)
+    cassette = Cassette(path, mode="replay")
+    assert len(cassette.unplayed) == 1
+    cassette.system_one(STATE, QUESTIONS)
+    assert cassette.unplayed == []
+
+
+def test_a_cassette_is_a_valid_jevl_golden_set(tmp_path):
+    path = tmp_path / "c.jevl"
+    Cassette(path, client([]), mode="auto").system_one(STATE, QUESTIONS)
+    (record,) = list(read_records(path))
+    assert isinstance(record, Record)
+    assert record.model == "jev-1.13.0"
+    assert record.usage == {"input_tokens": 12}
+
+
+def test_contains_reports_membership(tmp_path):
+    path = tmp_path / "c.jevl"
+    cassette = Cassette(path, client([]), mode="auto")
+    assert not cassette.contains(STATE, QUESTIONS)
+    cassette.system_one(STATE, QUESTIONS)
+    assert Cassette(path, mode="replay").contains(STATE, QUESTIONS)
+
+
+def test_sdk_style_response_objects_are_recorded(tmp_path):
+    class Answer:
+        def __init__(self):
+            self.type, self.choice = "choice", "billing"
+            self.probabilities, self.confidence = {"billing": 1.0}, 0.9
+
+    class Response:
+        model = "jev-1.13.0"
+        answers = {"team": Answer()}
+        usage = None
+
+    path = tmp_path / "c.jevl"
+    Cassette(path, lambda s, q, **k: Response(), mode="auto").system_one(STATE, QUESTIONS)
+    (record,) = list(read_records(path))
+    assert record.answers["team"]["choice"] == "billing"
+
+
+def test_recording_under_an_alias_replays_under_the_same_alias(tmp_path):
+    """Regression: the record stores the answering model (jev-1.13.0) while the
+    test asks for the alias (jev-latest). Indexing on the answering model made
+    every replay a miss."""
+    path = tmp_path / "c.jevl"
+    calls = []
+    Cassette(path, client(calls), mode="auto", model="jev-latest").system_one(STATE, QUESTIONS)
+
+    replayed = Cassette(path, client(calls), mode="auto", model="jev-latest")
+    replayed.system_one(STATE, QUESTIONS)
+    assert len(calls) == 1, "replay must not call through"
+
+    (record,) = list(read_records(path))
+    assert record.model == "jev-1.13.0", "the record states who actually answered"
+    assert record.meta["requested_model"] == "jev-latest"
+
+
+def test_a_different_requested_model_is_a_separate_entry(tmp_path):
+    path = tmp_path / "c.jevl"
+    calls = []
+    Cassette(path, client(calls), mode="auto", model="jev-latest").system_one(STATE, QUESTIONS)
+    Cassette(path, client(calls), mode="auto", model="jev-1.14.0").system_one(STATE, QUESTIONS)
+    assert len(calls) == 2