jevkit-drift 0.1.0__tar.gz

jevkit_drift-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.coverage
+htmlcov/
+.ruff_cache/
+uv.lock

jevkit_drift-0.1.0/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.5
+Name: jevkit-drift
+Version: 0.1.0
+Summary: Detect when a new TypeSafe Jev model version flips decisions your code depends on. Replays a golden set and diffs the answers.
+Project-URL: Homepage, https://github.com/pjdurden/jevkit-py
+Project-URL: Issues, https://github.com/pjdurden/jevkit-py/issues
+Author: Prajjwal Chittori
+License-Expression: MIT
+Keywords: drift,jev,regression,system-one,testing,typesafe
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: jevkit-core>=0.2.0
+Description-Content-Type: text/markdown
+
+# jevkit-drift
+
+TypeSafe's docs warn that `jev-latest` moves when a new version ships, and that
+confidence thresholds tuned against one version do not automatically hold on the
+next. `jevkit-drift` answers the question that warning implies: for the requests
+you actually care about, what changed?
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+pip install jevkit-drift
+```
+
+## Two kinds of change
+
+A **flip** is a changed decision. Your code takes a different branch. This is
+what breaks things.
+
+A **shift** is movement in the probability distribution with the same decision
+still on top. Harmless on its own, but it is what walks an answer toward a
+threshold, so a large shift is an early warning before anything has flipped.
+
+Both are reported separately, because conflating them is how a drift report ends
+up either too noisy to read or too quiet to help.
+
+## Use
+
+```python
+from jevkit_core import read_records, write_records
+from jevkit_drift import compare_sets, replay
+
+baseline = list(read_records("golden.jevl"))
+candidate = replay(baseline, lambda s, q: client.system_one(s, q, model="jev-1.14.0"))
+write_records("candidate.jevl", candidate)
+
+report = compare_sets(baseline, candidate)
+print(report.summary())
+
+for request_id, delta in report.flips:
+    print(request_id, delta.describe())
+```
+
+## CLI
+
+```bash
+jevkit-drift golden.jevl candidate.jevl
+jevkit-drift golden.jevl candidate.jevl --max-flips 3
+jevkit-drift golden.jevl candidate.jevl --max-shift 0.15
+jevkit-drift golden.jevl candidate.jevl --format json
+```
+
+Exit codes: `0` within tolerance, `1` drifted, `2` bad usage.
+
+## How records are paired
+
+On `request_id`, the digest of state plus questions with the model deliberately
+excluded. That is the whole reason the record format keeps two digests: the full
+`id` changes when the model changes, so it cannot pair a baseline with its
+replay, while `request_id` can.
+
+Change the state or a question and the records stop pairing, which is correct.
+They are no longer the same request, and comparing them would be meaningless.
+The CLI says so rather than reporting zero drift.
+
+## License
+
+MIT

jevkit_drift-0.1.0/README.md

@@ -0,0 +1,67 @@
+# jevkit-drift
+
+TypeSafe's docs warn that `jev-latest` moves when a new version ships, and that
+confidence thresholds tuned against one version do not automatically hold on the
+next. `jevkit-drift` answers the question that warning implies: for the requests
+you actually care about, what changed?
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+pip install jevkit-drift
+```
+
+## Two kinds of change
+
+A **flip** is a changed decision. Your code takes a different branch. This is
+what breaks things.
+
+A **shift** is movement in the probability distribution with the same decision
+still on top. Harmless on its own, but it is what walks an answer toward a
+threshold, so a large shift is an early warning before anything has flipped.
+
+Both are reported separately, because conflating them is how a drift report ends
+up either too noisy to read or too quiet to help.
+
+## Use
+
+```python
+from jevkit_core import read_records, write_records
+from jevkit_drift import compare_sets, replay
+
+baseline = list(read_records("golden.jevl"))
+candidate = replay(baseline, lambda s, q: client.system_one(s, q, model="jev-1.14.0"))
+write_records("candidate.jevl", candidate)
+
+report = compare_sets(baseline, candidate)
+print(report.summary())
+
+for request_id, delta in report.flips:
+    print(request_id, delta.describe())
+```
+
+## CLI
+
+```bash
+jevkit-drift golden.jevl candidate.jevl
+jevkit-drift golden.jevl candidate.jevl --max-flips 3
+jevkit-drift golden.jevl candidate.jevl --max-shift 0.15
+jevkit-drift golden.jevl candidate.jevl --format json
+```
+
+Exit codes: `0` within tolerance, `1` drifted, `2` bad usage.
+
+## How records are paired
+
+On `request_id`, the digest of state plus questions with the model deliberately
+excluded. That is the whole reason the record format keeps two digests: the full
+`id` changes when the model changes, so it cannot pair a baseline with its
+replay, while `request_id` can.
+
+Change the state or a question and the records stop pairing, which is correct.
+They are no longer the same request, and comparing them would be meaningless.
+The CLI says so rather than reporting zero drift.
+
+## License
+
+MIT

jevkit_drift-0.1.0/pyproject.toml

@@ -0,0 +1,33 @@
+[project]
+name = "jevkit-drift"
+version = "0.1.0"
+description = "Detect when a new TypeSafe Jev model version flips decisions your code depends on. Replays a golden set and diffs the answers."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "Prajjwal Chittori" }]
+keywords = ["jev", "typesafe", "system-one", "drift", "regression", "testing"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = ["jevkit-core>=0.2.0"]
+
+[project.scripts]
+jevkit-drift = "jevkit_drift.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/pjdurden/jevkit-py"
+Issues = "https://github.com/pjdurden/jevkit-py/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/jevkit_drift"]

jevkit_drift-0.1.0/src/jevkit_drift/__init__.py

@@ -0,0 +1,17 @@
+"""Detect when a new Jev model version changes decisions you depend on.
+
+TypeSafe's docs warn that ``jev-latest`` moves and that tuned confidence
+thresholds move with it. This package replays a golden set against a new version
+and reports what actually changed, separating flipped decisions from probability
+shifts that have not flipped anything yet.
+"""
+
+from .compare import (DriftReport, QuestionDelta, RecordDelta, compare_answers,
+                      compare_records, compare_sets, total_variation)
+from .replay import replay
+
+__version__ = "0.1.0"
+
+__all__ = ["compare_sets", "compare_records", "compare_answers", "replay",
+           "DriftReport", "RecordDelta", "QuestionDelta", "total_variation",
+           "__version__"]

jevkit_drift-0.1.0/src/jevkit_drift/cli.py

@@ -0,0 +1,70 @@
+"""``jevkit-drift`` command line interface."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from jevkit_core import RecordFormatError, read_records
+
+from .compare import compare_sets
+
+EXIT_OK = 0
+EXIT_DRIFTED = 1
+EXIT_USAGE = 2
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="jevkit-drift",
+        description="Compare two .jevl runs of the same requests and report which "
+                    "decisions flipped. Never calls the API.",
+    )
+    parser.add_argument("baseline", help="the golden set")
+    parser.add_argument("candidate", help="the same requests answered by another model")
+    parser.add_argument("--max-flips", type=int, default=0,
+                        help="tolerated number of flipped decisions (default 0)")
+    parser.add_argument("--max-shift", type=float, default=None,
+                        help="fail if any distribution moves more than this (0..1)")
+    parser.add_argument("--format", choices=("text", "json"), default="text")
+    parser.add_argument("--quiet", action="store_true", help="only print the summary")
+    args = parser.parse_args(argv)
+
+    try:
+        before = list(read_records(args.baseline))
+        after = list(read_records(args.candidate))
+    except (OSError, RecordFormatError) as exc:
+        print(f"jevkit-drift: {exc}", file=sys.stderr)
+        return EXIT_USAGE
+
+    try:
+        report = compare_sets(before, after)
+    except ValueError as exc:
+        print(f"jevkit-drift: {exc}", file=sys.stderr)
+        return EXIT_USAGE
+
+    if args.format == "json":
+        print(json.dumps(report.to_dict(), indent=2))
+    else:
+        print(report.summary())
+        if report.flips and not args.quiet:
+            print("\nflipped decisions:")
+            for request_id, delta in report.flips:
+                print(f"  {request_id[7:19]}  {delta.describe()}")
+
+    if not report.deltas and (before or after):
+        print("jevkit-drift: no requests matched between the two files. Records pair on "
+              "state and questions, so a change to either makes them incomparable.",
+              file=sys.stderr)
+        return EXIT_USAGE
+
+    if len(report.flips) > args.max_flips:
+        return EXIT_DRIFTED
+    if args.max_shift is not None and report.max_shift > args.max_shift:
+        return EXIT_DRIFTED
+    return EXIT_OK
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

jevkit_drift-0.1.0/src/jevkit_drift/compare.py

@@ -0,0 +1,236 @@
+"""Comparing two answers to the same request.
+
+TypeSafe's docs warn that ``jev-latest`` moves when a new version ships, and
+that confidence thresholds tuned against one version do not automatically hold
+on the next. This module answers the question that warning implies: for a set of
+requests you already care about, what actually changed?
+
+Two kinds of change are tracked separately, because they mean different things:
+
+- A **flip** is a changed decision. Your code takes a different branch. This is
+  what breaks things.
+- A **shift** is movement in the probability distribution with the same decision
+  on top. Harmless on its own, but it is what moves an answer toward a threshold,
+  so a large shift is an early warning even when nothing flipped yet.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+from jevkit_core import Answer, Record, parse_answers
+
+__all__ = ["QuestionDelta", "RecordDelta", "DriftReport", "compare_answers",
+           "compare_records", "compare_sets"]
+
+
+def total_variation(a: dict[str, float], b: dict[str, float]) -> float:
+    """Total variation distance between two distributions, on 0..1.
+
+    Outcomes present in one distribution and not the other count in full, which
+    is what makes this meaningful when a new model version changes the option
+    set. Distributions are not renormalized: if the API returns something that
+    does not sum to 1, that is reported rather than hidden.
+    """
+    keys = set(a) | set(b)
+    return sum(abs(a.get(k, 0.0) - b.get(k, 0.0)) for k in keys) / 2.0
+
+
+@dataclass
+class QuestionDelta:
+    """What changed for one question between two runs."""
+
+    question_id: str
+    type: str
+    before: Any
+    after: Any
+    distribution_shift: float
+    confidence_before: float | None
+    confidence_after: float | None
+    score_before: float | None = None
+    score_after: float | None = None
+
+    @property
+    def flipped(self) -> bool:
+        """The selected outcome changed."""
+        return self.before != self.after
+
+    @property
+    def confidence_delta(self) -> float | None:
+        if self.confidence_before is None or self.confidence_after is None:
+            return None
+        return self.confidence_after - self.confidence_before
+
+    def describe(self) -> str:
+        # JSON rendering rather than repr(), so the Python and JavaScript CLIs
+        # emit byte-identical output. repr() quotes with ', JSON with ".
+        if self.flipped:
+            head = (f"{self.question_id}: FLIP {json.dumps(self.before)} -> "
+                    f"{json.dumps(self.after)}")
+        else:
+            head = f"{self.question_id}: stable ({json.dumps(self.before)})"
+        parts = [f"shift={self.distribution_shift:.3f}"]
+        delta = self.confidence_delta
+        if delta is not None:
+            parts.append(f"conf {self.confidence_before:.3f} -> "
+                         f"{self.confidence_after:.3f} ({delta:+.3f})")
+        return f"{head}  [{', '.join(parts)}]"
+
+
+@dataclass
+class RecordDelta:
+    """What changed for one request between two runs."""
+
+    request_id: str
+    model_before: str
+    model_after: str
+    questions: list[QuestionDelta] = field(default_factory=list)
+
+    @property
+    def flips(self) -> list[QuestionDelta]:
+        return [q for q in self.questions if q.flipped]
+
+    @property
+    def max_shift(self) -> float:
+        return max((q.distribution_shift for q in self.questions), default=0.0)
+
+
+def compare_answers(qid: str, before: Answer, after: Answer) -> QuestionDelta:
+    if before.type != after.type:
+        raise ValueError(
+            f"question {qid!r}: type changed from {before.type!r} to {after.type!r}. "
+            f"These are not the same question and cannot be compared."
+        )
+    return QuestionDelta(
+        question_id=qid,
+        type=before.type,
+        before=before.predicted(),
+        after=after.predicted(),
+        distribution_shift=total_variation(before.probabilities, after.probabilities),
+        confidence_before=before.confidence,
+        confidence_after=after.confidence,
+        score_before=before.score,
+        score_after=after.score,
+    )
+
+
+def compare_records(before: Record, after: Record) -> RecordDelta:
+    if before.request_id != after.request_id:
+        raise ValueError(
+            "records describe different requests and cannot be compared "
+            f"({before.request_id} vs {after.request_id})"
+        )
+    old = parse_answers(before.answers)
+    new = parse_answers(after.answers)
+
+    missing = sorted(set(old) - set(new))
+    added = sorted(set(new) - set(old))
+    if missing or added:
+        detail = []
+        if missing:
+            detail.append(f"missing in the new run: {', '.join(missing)}")
+        if added:
+            detail.append(f"only in the new run: {', '.join(added)}")
+        raise ValueError(
+            f"answer sets differ for request {before.request_id}: {'; '.join(detail)}"
+        )
+
+    return RecordDelta(
+        request_id=before.request_id,
+        model_before=before.model,
+        model_after=after.model,
+        questions=[compare_answers(qid, old[qid], new[qid]) for qid in sorted(old)],
+    )
+
+
+@dataclass
+class DriftReport:
+    """Drift across a whole golden set."""
+
+    deltas: list[RecordDelta] = field(default_factory=list)
+    unmatched_before: list[str] = field(default_factory=list)
+    unmatched_after: list[str] = field(default_factory=list)
+
+    @property
+    def total_questions(self) -> int:
+        return sum(len(d.questions) for d in self.deltas)
+
+    @property
+    def flips(self) -> list[tuple[str, QuestionDelta]]:
+        return [(d.request_id, q) for d in self.deltas for q in d.flips]
+
+    @property
+    def flip_rate(self) -> float:
+        total = self.total_questions
+        return len(self.flips) / total if total else 0.0
+
+    @property
+    def max_shift(self) -> float:
+        return max((d.max_shift for d in self.deltas), default=0.0)
+
+    @property
+    def mean_shift(self) -> float:
+        shifts = [q.distribution_shift for d in self.deltas for q in d.questions]
+        return sum(shifts) / len(shifts) if shifts else 0.0
+
+    def models(self) -> tuple[set[str], set[str]]:
+        return ({d.model_before for d in self.deltas}, {d.model_after for d in self.deltas})
+
+    def summary(self) -> str:
+        before, after = self.models()
+        lines = [
+            f"compared {len(self.deltas)} request(s), {self.total_questions} question(s)",
+            f"  {', '.join(sorted(before)) or '?'} -> {', '.join(sorted(after)) or '?'}",
+            f"  flips:      {len(self.flips)} ({self.flip_rate:.1%})",
+            f"  mean shift: {self.mean_shift:.4f}",
+            f"  max shift:  {self.max_shift:.4f}",
+        ]
+        if self.unmatched_before:
+            lines.append(f"  {len(self.unmatched_before)} record(s) in the baseline had no "
+                         f"counterpart and were not compared")
+        if self.unmatched_after:
+            lines.append(f"  {len(self.unmatched_after)} record(s) in the new run had no "
+                         f"counterpart and were not compared")
+        return "\n".join(lines)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "requests": len(self.deltas),
+            "questions": self.total_questions,
+            "flips": len(self.flips),
+            "flip_rate": self.flip_rate,
+            "mean_shift": self.mean_shift,
+            "max_shift": self.max_shift,
+            "unmatched_before": self.unmatched_before,
+            "unmatched_after": self.unmatched_after,
+            "details": [
+                {
+                    "request_id": rid,
+                    "question_id": q.question_id,
+                    "before": q.before,
+                    "after": q.after,
+                    "distribution_shift": q.distribution_shift,
+                    "confidence_delta": q.confidence_delta,
+                }
+                for rid, q in self.flips
+            ],
+        }
+
+
+def compare_sets(before: list[Record], after: list[Record]) -> DriftReport:
+    """Pair two sets of records by request id and compare each pair.
+
+    Records are paired on ``request_id``, the digest of state plus questions with
+    the model excluded, which is exactly what makes a baseline comparable to its
+    replay on a different model version.
+    """
+    old = {r.request_id: r for r in before}
+    new = {r.request_id: r for r in after}
+    shared = sorted(set(old) & set(new))
+    return DriftReport(
+        deltas=[compare_records(old[rid], new[rid]) for rid in shared],
+        unmatched_before=sorted(set(old) - set(new)),
+        unmatched_after=sorted(set(new) - set(old)),
+    )

jevkit_drift-0.1.0/src/jevkit_drift/replay.py

@@ -0,0 +1,72 @@
+"""Replaying a golden set against a model."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Iterable
+
+from jevkit_core import Record
+
+__all__ = ["replay", "SystemOneCallable"]
+
+SystemOneCallable = Callable[[Any, dict[str, Any]], Any]
+
+
+def _extract(response: Any) -> tuple[str, dict[str, Any], dict[str, Any] | None]:
+    """Pull (model, answers, usage) out of an SDK response or a plain dict."""
+    if isinstance(response, dict):
+        model = response.get("model", "")
+        answers = response.get("answers", {})
+        usage = response.get("usage")
+    else:
+        model = getattr(response, "model", "")
+        answers = getattr(response, "answers", {})
+        usage = getattr(response, "usage", None)
+
+    plain: dict[str, Any] = {}
+    for qid, answer in (answers or {}).items():
+        if isinstance(answer, dict):
+            plain[qid] = answer
+        elif hasattr(answer, "model_dump"):
+            plain[qid] = answer.model_dump()
+        elif hasattr(answer, "__dict__"):
+            plain[qid] = {k: v for k, v in vars(answer).items() if not k.startswith("_")}
+        else:
+            raise TypeError(f"answer {qid!r}: cannot convert {type(answer).__name__} to a dict")
+
+    if usage is not None and not isinstance(usage, dict):
+        if hasattr(usage, "model_dump"):
+            usage = usage.model_dump()
+        elif hasattr(usage, "__dict__"):
+            usage = {k: v for k, v in vars(usage).items() if not k.startswith("_")}
+
+    return str(model), plain, usage
+
+
+def replay(
+    records: Iterable[Record],
+    system_one: SystemOneCallable,
+    *,
+    tags: Iterable[str] | None = None,
+) -> list[Record]:
+    """Re-send each record's request and return the new answers as new records.
+
+    ``system_one`` is any callable taking ``(state, questions)`` and returning a
+    response, which is the shape both official SDKs already expose. Labels and
+    tags carry over from the baseline so a replayed set stays scoreable.
+    """
+    extra_tags = list(tags or [])
+    out: list[Record] = []
+    for record in records:
+        response = system_one(record.state, record.questions)
+        model, answers, usage = _extract(response)
+        out.append(Record(
+            model=model or record.model,
+            state=record.state,
+            questions=record.questions,
+            answers=answers,
+            usage=usage,
+            label=record.label,
+            tags=list(record.tags) + extra_tags,
+            meta={**record.meta, "replayed_from": record.id},
+        ))
+    return out

jevkit_drift-0.1.0/tests/test_drift.py

@@ -0,0 +1,114 @@
+import pytest
+from jevkit_core import Record
+from jevkit_drift import compare_records, compare_sets, replay, total_variation
+
+QUESTIONS = {"team": {"type": "choice", "instructions": "Which team"}}
+
+
+def rec(model, choice, probs, confidence=0.5, state="s"):
+    return Record(model=model, state=state, questions=QUESTIONS,
+                  answers={"team": {"type": "choice", "choice": choice,
+                                    "probabilities": probs, "confidence": confidence}})
+
+
+def test_total_variation_of_identical_distributions_is_zero():
+    assert total_variation({"a": 0.5, "b": 0.5}, {"a": 0.5, "b": 0.5}) == 0.0
+
+
+def test_total_variation_of_disjoint_distributions_is_one():
+    assert total_variation({"a": 1.0}, {"b": 1.0}) == pytest.approx(1.0)
+
+
+def test_total_variation_counts_outcomes_missing_from_one_side():
+    assert total_variation({"a": 1.0}, {"a": 0.5, "b": 0.5}) == pytest.approx(0.5)
+
+
+def test_a_changed_decision_is_a_flip():
+    delta = compare_records(
+        rec("jev-1.13.0", "billing", {"billing": 0.7, "technical": 0.3}),
+        rec("jev-1.14.0", "technical", {"billing": 0.3, "technical": 0.7}),
+    )
+    assert delta.flips and delta.questions[0].flipped
+    assert delta.questions[0].before == "billing"
+    assert delta.questions[0].after == "technical"
+
+
+def test_a_moved_distribution_with_the_same_decision_is_not_a_flip():
+    delta = compare_records(
+        rec("jev-1.13.0", "billing", {"billing": 0.9, "technical": 0.1}),
+        rec("jev-1.14.0", "billing", {"billing": 0.6, "technical": 0.4}),
+    )
+    assert not delta.flips
+    assert delta.max_shift == pytest.approx(0.3)
+
+
+def test_confidence_delta_is_reported():
+    delta = compare_records(
+        rec("jev-1.13.0", "billing", {"billing": 0.9}, confidence=0.8),
+        rec("jev-1.14.0", "billing", {"billing": 0.9}, confidence=0.5),
+    )
+    assert delta.questions[0].confidence_delta == pytest.approx(-0.3)
+
+
+def test_records_for_different_requests_cannot_be_compared():
+    with pytest.raises(ValueError, match="different requests"):
+        compare_records(rec("m", "a", {"a": 1.0}, state="one"),
+                        rec("m", "a", {"a": 1.0}, state="two"))
+
+
+def test_a_changed_answer_type_is_refused():
+    before = Record(model="m", state="s", questions=QUESTIONS,
+                    answers={"team": {"type": "choice", "choice": "a", "probabilities": {"a": 1.0}}})
+    after = Record(model="m", state="s", questions=QUESTIONS,
+                   answers={"team": {"type": "noul", "noul": 1.0}})
+    with pytest.raises(ValueError, match="type changed"):
+        compare_records(before, after)
+
+
+def test_a_missing_answer_is_refused_rather_than_ignored():
+    before = rec("m", "billing", {"billing": 1.0})
+    after = Record(model="m", state="s", questions=QUESTIONS, answers={})
+    with pytest.raises(ValueError, match="missing in the new run"):
+        compare_records(before, after)
+
+
+def test_sets_pair_on_request_id_across_model_versions():
+    report = compare_sets(
+        [rec("jev-1.13.0", "billing", {"billing": 0.7, "technical": 0.3})],
+        [rec("jev-1.14.0", "technical", {"billing": 0.3, "technical": 0.7})],
+    )
+    assert len(report.deltas) == 1
+    assert len(report.flips) == 1
+    assert report.flip_rate == 1.0
+
+
+def test_unmatched_records_are_reported_not_silently_dropped():
+    report = compare_sets(
+        [rec("m", "a", {"a": 1.0}, state="one")],
+        [rec("m", "a", {"a": 1.0}, state="two")],
+    )
+    assert report.deltas == []
+    assert len(report.unmatched_before) == 1
+    assert len(report.unmatched_after) == 1
+
+
+def test_replay_carries_labels_and_calls_through():
+    baseline = [Record(model="jev-1.13.0", state="s", questions=QUESTIONS,
+                       answers={"team": {"type": "choice", "choice": "billing",
+                                         "probabilities": {"billing": 1.0}}},
+                       label={"team": "billing"}, tags=["routing"])]
+    seen = []
+
+    def fake(state, questions):
+        seen.append((state, questions))
+        return {"model": "jev-1.14.0",
+                "answers": {"team": {"type": "choice", "choice": "technical",
+                                     "probabilities": {"technical": 1.0}}},
+                "usage": {"input_tokens": 5}}
+
+    out = replay(baseline, fake)
+    assert len(seen) == 1
+    assert out[0].model == "jev-1.14.0"
+    assert out[0].label == {"team": "billing"}
+    assert "routing" in out[0].tags
+    assert out[0].request_id == baseline[0].request_id