jevkit-pytest 0.1.0__py3-none-any.whl

jevkit_pytest/__init__.py

@@ -0,0 +1,13 @@
+"""Record and replay TypeSafe Jev requests in pytest.
+
+Cassettes are `.jevl` files, the same format the drift, bench and calibrate
+packages read, so a recording made by your tests doubles as a golden set.
+"""
+
+from .assertions import assert_answer, assert_confident, describe_answer
+from .cassette import Cassette, CassetteMiss
+
+__version__ = "0.1.0"
+
+__all__ = ["Cassette", "CassetteMiss", "assert_answer", "assert_confident",
+           "describe_answer", "__version__"]

jevkit_pytest/assertions.py

@@ -0,0 +1,83 @@
+"""Assertions over jev answers, with failure messages that say enough.
+
+A bare ``assert answer.choice == "billing"`` tells you nothing about *how*
+close the call was. These assertions print the distribution on failure, because
+a 0.51/0.49 split and a 0.99/0.01 split are different bugs.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from jevkit_core import parse_answer
+
+__all__ = ["assert_answer", "assert_confident", "describe_answer"]
+
+
+def describe_answer(qid: str, raw: Any) -> str:
+    answer = parse_answer(qid, raw)
+    probs = ", ".join(
+        f"{k}={v:.3f}" for k, v in sorted(answer.probabilities.items(), key=lambda kv: -kv[1])
+    )
+    parts = [f"predicted={answer.predicted()!r}"]
+    if answer.confidence is not None:
+        parts.append(f"confidence={answer.confidence:.3f}")
+    if answer.score is not None:
+        parts.append(f"score={answer.score:.3f}")
+    return f"{qid}: {', '.join(parts)}\n    probabilities: {probs}"
+
+
+def assert_answer(
+    raw: Any,
+    expected: Any,
+    *,
+    question_id: str = "answer",
+    min_confidence: float | None = None,
+    min_probability: float | None = None,
+) -> None:
+    """Assert an answer selected ``expected``, optionally with enough certainty."""
+    answer = parse_answer(question_id, raw)
+    predicted = answer.predicted()
+
+    if not answer.is_correct(expected):
+        raise AssertionError(
+            f"expected {expected!r} but got {predicted!r}\n  {describe_answer(question_id, raw)}"
+        )
+
+    if min_probability is not None:
+        actual = answer.probability_of(expected)
+        if actual < min_probability:
+            raise AssertionError(
+                f"{expected!r} was selected but carried only {actual:.3f} probability, "
+                f"below the required {min_probability:.3f}\n"
+                f"  {describe_answer(question_id, raw)}"
+            )
+
+    if min_confidence is not None:
+        if answer.confidence is None:
+            raise AssertionError(
+                f"min_confidence was given but a {answer.type} answer carries no "
+                f"confidence. Use min_probability instead."
+            )
+        if answer.confidence < min_confidence:
+            raise AssertionError(
+                f"{expected!r} was selected but confidence was {answer.confidence:.3f}, "
+                f"below the required {min_confidence:.3f}\n"
+                f"  {describe_answer(question_id, raw)}"
+            )
+
+
+def assert_confident(raw: Any, minimum: float, *, question_id: str = "answer") -> None:
+    """Assert an answer is decisive, without caring which way it went.
+
+    Uses the API's confidence for Choice and Score. A Noul has none, so its
+    distance from 0.5 is used and the message says so.
+    """
+    answer = parse_answer(question_id, raw)
+    value = answer.decisiveness
+    if value < minimum:
+        quantity = "decisiveness (|noul - 0.5| * 2)" if answer.type == "noul" else "confidence"
+        raise AssertionError(
+            f"{quantity} was {value:.3f}, below the required {minimum:.3f}\n"
+            f"  {describe_answer(question_id, raw)}"
+        )

jevkit_pytest/cassette.py

@@ -0,0 +1,204 @@
+"""Record and replay jev requests so tests do not hit the API.
+
+A model call in a test is slow, costs money, needs a key in CI, and can change
+its answer under you when the alias moves. The fix is the one VCR established
+for HTTP: record real responses once, replay them forever, re-record on purpose.
+
+A cassette is a `.jevl` file, the same format `drift`, `bench` and `calibrate`
+read, so a recording made by your test suite is also a golden set you can replay
+against the next model version.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Callable
+
+from jevkit_core import Record, append_record, read_records, record_id
+
+__all__ = ["Cassette", "CassetteMiss", "Mode"]
+
+Mode = str  # "replay" | "record" | "auto" | "passthrough"
+
+_MODES = ("replay", "record", "auto", "passthrough")
+
+
+class CassetteMiss(LookupError):
+    """A request was not on the cassette and the mode forbids recording."""
+
+
+def _response_to_parts(response: Any) -> tuple[str, dict[str, Any], dict[str, Any] | None]:
+    """Pull (model, answers, usage) out of an SDK response or a plain dict."""
+    if isinstance(response, dict):
+        model, answers, usage = response.get("model", ""), response.get("answers", {}), response.get("usage")
+    else:
+        model = getattr(response, "model", "")
+        answers = getattr(response, "answers", {})
+        usage = getattr(response, "usage", None)
+
+    def plain(value: Any) -> Any:
+        if isinstance(value, dict):
+            return value
+        if hasattr(value, "model_dump"):
+            return value.model_dump()
+        if hasattr(value, "__dict__"):
+            return {k: v for k, v in vars(value).items() if not k.startswith("_")}
+        return value
+
+    return str(model), {qid: plain(a) for qid, a in (answers or {}).items()}, (
+        plain(usage) if usage is not None else None
+    )
+
+
+class ReplayedAnswer(dict):
+    """A replayed answer.
+
+    Subclasses ``dict`` so ``answer["choice"]`` works, and mirrors the keys onto
+    attributes so ``answer.choice`` works too. Test code written against either
+    the SDK objects or the raw JSON keeps working without a shim.
+    """
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            return self[name]
+        except KeyError as exc:
+            raise AttributeError(
+                f"answer has no field {name!r}; recorded fields are "
+                f"{', '.join(sorted(self)) or '(none)'}"
+            ) from exc
+
+
+class ReplayedResponse:
+    """What a cassette hands back in place of a live API response."""
+
+    def __init__(self, record: Record) -> None:
+        self._record = record
+        self.model = record.model
+        self.answers = {qid: ReplayedAnswer(a) for qid, a in record.answers.items()}
+        self.usage = record.usage
+        self.id = record.id
+
+    def __repr__(self) -> str:  # pragma: no cover - debug aid
+        return f"ReplayedResponse(model={self.model!r}, answers={list(self.answers)})"
+
+
+class Cassette:
+    """A recorded set of jev requests, replayed by request digest.
+
+    ``mode`` controls what happens on a miss:
+
+    - ``replay``      raise ``CassetteMiss``. The right default for CI.
+    - ``auto``        replay a hit, call through and append on a miss.
+    - ``record``      always call through and append, ignoring existing entries.
+    - ``passthrough`` never touch the cassette.
+    """
+
+    def __init__(
+        self,
+        path: str | os.PathLike[str],
+        system_one: Callable[..., Any] | None = None,
+        *,
+        mode: Mode = "replay",
+        model: str = "jev-latest",
+    ) -> None:
+        if mode not in _MODES:
+            raise ValueError(f"mode must be one of {_MODES}, got {mode!r}")
+        self.path = Path(path)
+        self.mode = mode
+        self.model = model
+        self._system_one = system_one
+        self._entries = self._load() if self.path.exists() and mode != "record" else {}
+        self.played: list[str] = []
+        self.recorded: list[str] = []
+
+    def _load(self) -> dict[str, Record]:
+        """Index the file by *requested*-model digest.
+
+        A record stores the model that actually answered, which is what the
+        format requires: `jev-latest` is an alias and the response says
+        `jev-1.13.0`. But a test asks for the alias, so indexing on the answering
+        model would miss every lookup. The requested model is kept in `meta` at
+        record time and used to rebuild the index here.
+
+        Later entries win, so re-recording a request appends rather than
+        requiring a rewrite.
+        """
+        entries: dict[str, Record] = {}
+        for record in read_records(self.path):
+            requested = str(record.meta.get("requested_model") or record.model)
+            entries[record_id(requested, record.state, record.questions)] = record
+        return entries
+
+    # -- introspection -----------------------------------------------------
+
+    def __len__(self) -> int:
+        return len(self._entries)
+
+    @property
+    def unplayed(self) -> list[str]:
+        """Entries on the cassette that no test asked for.
+
+        Usually means a test was deleted or a question was reworded, leaving a
+        stale recording that will quietly rot.
+        """
+        return sorted(set(self._entries) - set(self.played))
+
+    def contains(self, state: Any, questions: dict[str, Any], *, model: str | None = None) -> bool:
+        return record_id(model or self.model, state, questions) in self._entries
+
+    # -- the call ----------------------------------------------------------
+
+    def system_one(self, state: Any, questions: dict[str, Any], **kwargs: Any) -> Any:
+        """Drop-in for a client's ``system_one``.
+
+        Signature matches what both official SDKs expose, so a cassette can be
+        passed anywhere a client is expected in a test.
+        """
+        model = kwargs.pop("model", None) or self.model
+
+        if self.mode == "passthrough":
+            return self._call_through(state, questions, model=model, **kwargs)
+
+        key = record_id(model, state, questions)
+
+        if self.mode != "record":
+            hit = self._entries.get(key)
+            if hit is not None:
+                self.played.append(key)
+                return ReplayedResponse(hit)
+
+        if self.mode == "replay":
+            raise CassetteMiss(
+                f"no recording for this request on {self.path}.\n"
+                f"  digest: {key}\n"
+                f"  Re-run with mode='auto' (or --jev-record) to record it, and commit the "
+                f"updated cassette."
+            )
+
+        response = self._call_through(state, questions, model=model, **kwargs)
+        actual_model, answers, usage = _response_to_parts(response)
+        record = Record(
+            model=actual_model or model,
+            state=state,
+            questions=questions,
+            answers=answers,
+            usage=usage,
+            meta={"requested_model": model},
+        )
+        append_record(self.path, record)
+        self._entries[key] = record
+        # The live response is returned rather than the record: recording must
+        # not change what the code under test sees.
+        self.recorded.append(key)
+        return response
+
+    __call__ = system_one
+
+    def _call_through(self, state: Any, questions: dict[str, Any], **kwargs: Any) -> Any:
+        if self._system_one is None:
+            raise CassetteMiss(
+                f"cassette {self.path} is in mode {self.mode!r} and needs to call the API, "
+                f"but no client was supplied. Pass system_one= when constructing it."
+            )
+        return self._system_one(state, questions, **kwargs)

jevkit_pytest/plugin.py

@@ -0,0 +1,90 @@
+"""The pytest plugin: fixtures and the ``--jev-record`` flag."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Iterator
+
+import pytest
+
+from .assertions import assert_answer
+from .cassette import Cassette
+
+__all__ = ["jev_cassette", "jev_cassette_dir", "jev_client"]
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("jev", "TypeSafe Jev cassettes")
+    group.addoption(
+        "--jev-record", action="store_true", default=False,
+        help="record any request missing from its cassette (mode 'auto')",
+    )
+    group.addoption(
+        "--jev-rerecord", action="store_true", default=False,
+        help="re-record every request, replacing the cassettes (mode 'record')",
+    )
+    group.addoption(
+        "--jev-cassette-dir", action="store", default=None,
+        help="where cassettes live (default: tests/cassettes next to the test file)",
+    )
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line("markers", "jev_cassette(name): use a named cassette file")
+
+
+def _mode(config: pytest.Config) -> str:
+    if config.getoption("--jev-rerecord"):
+        return "record"
+    if config.getoption("--jev-record"):
+        return "auto"
+    return "replay"
+
+
+@pytest.fixture(scope="session")
+def jev_cassette_dir(request: pytest.FixtureRequest) -> Path:
+    """Directory holding cassettes. Override in a conftest to relocate them."""
+    configured = request.config.getoption("--jev-cassette-dir")
+    if configured:
+        return Path(configured)
+    return Path(str(request.config.rootpath)) / "tests" / "cassettes"
+
+
+@pytest.fixture
+def jev_client() -> Any:
+    """The live client, used only when recording.
+
+    Override this in your conftest to return something with a ``system_one``
+    method. Left as ``None`` a replay-only run still works; a recording run
+    fails with a message saying exactly this.
+    """
+    return None
+
+
+@pytest.fixture
+def jev_cassette(
+    request: pytest.FixtureRequest,
+    jev_cassette_dir: Path,
+    jev_client: Any,
+) -> Iterator[Cassette]:
+    """A cassette scoped to the current test.
+
+    The file is named after the test by default, so one test's recordings never
+    collide with another's. Override with ``@pytest.mark.jev_cassette("name")``
+    to share one cassette across several tests.
+    """
+    marker = request.node.get_closest_marker("jev_cassette")
+    name = marker.args[0] if marker and marker.args else request.node.name
+    safe = "".join(c if c.isalnum() or c in "-_." else "_" for c in str(name))
+
+    jev_cassette_dir.mkdir(parents=True, exist_ok=True)
+    path = jev_cassette_dir / f"{safe}.jevl"
+
+    system_one = getattr(jev_client, "system_one", None) if jev_client is not None else None
+    cassette = Cassette(path, system_one, mode=_mode(request.config))
+    yield cassette
+
+
+# Re-exported so ``from jevkit_pytest import assert_answer`` works after the
+# plugin is loaded as an entry point.
+__all__ += ["assert_answer"]

jevkit_pytest-0.1.0.dist-info/METADATA

@@ -0,0 +1,103 @@
+Metadata-Version: 2.5
+Name: jevkit-pytest
+Version: 0.1.0
+Summary: Record and replay TypeSafe Jev requests in pytest. Cassettes are .jevl files, so a test recording doubles as a drift golden set.
+Project-URL: Homepage, https://github.com/pjdurden/jevkit-py
+Project-URL: Issues, https://github.com/pjdurden/jevkit-py/issues
+Author: Prajjwal Chittori
+License-Expression: MIT
+Keywords: cassette,jev,pytest,system-one,testing,typesafe,vcr
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: jevkit-core>=0.2.0
+Requires-Dist: pytest>=7.0
+Description-Content-Type: text/markdown
+
+# jevkit-pytest
+
+Record and replay TypeSafe Jev requests in pytest.
+
+A model call in a test is slow, costs money, needs a key in CI, and can change
+its answer under you when the alias moves. The fix is the one VCR established for
+HTTP: record real responses once, replay them forever, re-record on purpose.
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+pip install jevkit-pytest
+```
+
+## Use
+
+```python
+def test_routes_billing_questions(jev_cassette):
+    response = jev_cassette.system_one(
+        "I was charged twice for the same order",
+        {"team": {"type": "choice", "instructions": "Which team should handle this",
+                  "criteria": {"billing": "Payment issues", "technical": "Bugs",
+                               "unknown": "None apply"}}},
+    )
+    assert response.answers["team"]["choice"] == "billing"
+```
+
+Record the first time, then never again:
+
+```bash
+pytest --jev-record       # record anything missing
+pytest                    # replay only; a miss is a failure
+pytest --jev-rerecord     # replace every recording
+```
+
+To record you need a live client. Supply one by overriding the `jev_client`
+fixture in your `conftest.py`:
+
+```python
+import pytest
+from typesafe_sdk import TypeSafeClient
+
+@pytest.fixture
+def jev_client():
+    with TypeSafeClient() as client:
+        yield client
+```
+
+Replay-only runs need no client and no API key, which is the point: CI stays
+green without a secret.
+
+## Assertions that explain themselves
+
+```python
+from jevkit_pytest import assert_answer, assert_confident
+
+assert_answer(response.answers["team"], "billing", min_probability=0.6)
+assert_confident(response.answers["urgency"], 0.8)
+```
+
+On failure these print the whole distribution, because a 0.51/0.49 split and a
+0.99/0.01 split are different bugs and `assert x == y` cannot tell them apart.
+
+`assert_confident` uses the API's confidence for Choice and Score. A Noul carries
+none, so its distance from 0.5 is used and the message says which it used.
+
+## Cassettes are golden sets
+
+A cassette is a `.jevl` file, the same format `jevkit-drift`, `jevkit-bench` and
+`jevkit-calibrate` read. So the recordings your tests already make are a golden
+set you can replay against the next model version:
+
+```bash
+jevkit-drift tests/cassettes/test_routes_billing_questions.jevl candidate.jevl
+```
+
+That is the whole reason the format was defined before any of these packages.
+
+## License
+
+MIT

jevkit_pytest-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+jevkit_pytest/__init__.py,sha256=2pyniclDDTNchuMAzNNXofGIrC8RBBjiEnBF8Cehw8c,473
+jevkit_pytest/assertions.py,sha256=aWDpNZbQbAN8EiWheiIta3vhawgfb-OpKo6z3wd7skM,3111
+jevkit_pytest/cassette.py,sha256=v2MdFO-pmK5T01T2AMWTtbT8ZCI7HLIS3anD05V9zT8,7678
+jevkit_pytest/plugin.py,sha256=FlBhotK-a63LoPDmsuEnpNCqM4cB7ahnf7Y4d8ohMao,2998
+jevkit_pytest-0.1.0.dist-info/METADATA,sha256=Yp8bZ1iYgg1Z6jJDWhadBndE0_j2xtwQ6Z3fxDxYZL4,3357
+jevkit_pytest-0.1.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
+jevkit_pytest-0.1.0.dist-info/entry_points.txt,sha256=__pcKgSgXicCqUTdicXJmwYgvlfLz2BXirAgG2naacs,41
+jevkit_pytest-0.1.0.dist-info/RECORD,,

jevkit_pytest-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.3
+Root-Is-Purelib: true
+Tag: py3-none-any

jevkit_pytest-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+jevkit = jevkit_pytest.plugin