jevkit-core 0.2.0__tar.gz

jevkit_core-0.2.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.coverage
+htmlcov/
+.ruff_cache/
+uv.lock

jevkit_core-0.2.0/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.5
+Name: jevkit-core
+Version: 0.2.0
+Summary: Shared substrate for jevkit: the .jevl record format, canonical request digests, question normalization, and token budgets for TypeSafe's Jev.
+Project-URL: Homepage, https://github.com/pjdurden/jevkit-py
+Project-URL: Issues, https://github.com/pjdurden/jevkit-py/issues
+Author: Prajjwal Chittori
+License-Expression: MIT
+Keywords: calibration,jev,llm,system-one,typesafe
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# jevkit-core
+
+Shared substrate for [jevkit](https://github.com/pjdurden/jevkit-py), a set of
+tools for building on TypeSafe's Jev. Nothing in this package calls the API.
+
+- **`.jevl` record format** — one JSON Lines format that serves as test
+  cassette, drift golden set, benchmark instance, and calibration observation.
+  See [the spec](../../docs/specs/record-format-v1.md).
+- **Canonical digests** — RFC 8785 canonicalization, so the same request always
+  hashes the same way. `record_id()` includes the model; `request_id()` does
+  not, which is what lets you pair a golden record with its replay on a new
+  model version.
+- **Question normalization** — one view over SDK objects and plain dicts alike.
+- **Token budgets** — estimates against Jev's two documented limits: 64k for
+  state plus all questions, 32k for state plus the longest single question.
+
+> Unofficial and unaffiliated with TypeSafe.
+
+## Install
+
+```bash
+pip install jevkit-core
+```
+
+## License
+
+MIT

jevkit_core-0.2.0/README.md

@@ -0,0 +1,27 @@
+# jevkit-core
+
+Shared substrate for [jevkit](https://github.com/pjdurden/jevkit-py), a set of
+tools for building on TypeSafe's Jev. Nothing in this package calls the API.
+
+- **`.jevl` record format** — one JSON Lines format that serves as test
+  cassette, drift golden set, benchmark instance, and calibration observation.
+  See [the spec](../../docs/specs/record-format-v1.md).
+- **Canonical digests** — RFC 8785 canonicalization, so the same request always
+  hashes the same way. `record_id()` includes the model; `request_id()` does
+  not, which is what lets you pair a golden record with its replay on a new
+  model version.
+- **Question normalization** — one view over SDK objects and plain dicts alike.
+- **Token budgets** — estimates against Jev's two documented limits: 64k for
+  state plus all questions, 32k for state plus the longest single question.
+
+> Unofficial and unaffiliated with TypeSafe.
+
+## Install
+
+```bash
+pip install jevkit-core
+```
+
+## License
+
+MIT

jevkit_core-0.2.0/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "jevkit-core"
+version = "0.2.0"
+description = "Shared substrate for jevkit: the .jevl record format, canonical request digests, question normalization, and token budgets for TypeSafe's Jev."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "Prajjwal Chittori" }]
+keywords = ["jev", "typesafe", "system-one", "llm", "calibration"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/pjdurden/jevkit-py"
+Issues = "https://github.com/pjdurden/jevkit-py/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/jevkit_core"]

jevkit_core-0.2.0/src/jevkit_core/__init__.py

@@ -0,0 +1,28 @@
+"""Shared substrate for the jevkit packages.
+
+Nothing here calls the jev API. This package holds the pieces every other
+jevkit tool needs: canonical digests, the `.jevl` record format, a uniform
+view of a question, and token budget estimation.
+"""
+
+from .answers import NOUL_THRESHOLD, Answer, parse_answer, parse_answers
+from .canonical import canonical_json, digest, record_id, request_id
+from .question import (Question, flatten_text, normalize_question,
+                       normalize_questions)
+from .record import (FORMAT_VERSION, Record, RecordFormatError, append_record,
+                     load_cassette, read_records, write_records)
+from .tokens import (STATE_BUDGET, TOTAL_BUDGET, BudgetReport, check_budget,
+                     estimate_tokens)
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "Answer", "parse_answer", "parse_answers", "NOUL_THRESHOLD",
+    "canonical_json", "digest", "record_id", "request_id",
+    "Question", "flatten_text", "normalize_question", "normalize_questions",
+    "Record", "RecordFormatError", "FORMAT_VERSION",
+    "read_records", "write_records", "append_record", "load_cassette",
+    "estimate_tokens", "check_budget", "BudgetReport",
+    "TOTAL_BUDGET", "STATE_BUDGET",
+    "__version__",
+]

jevkit_core-0.2.0/src/jevkit_core/answers.py

@@ -0,0 +1,151 @@
+"""A uniform view of a jev answer, and how it compares to a ground-truth label.
+
+`drift`, `calibrate`, and `bench` all need the same three things from an answer:
+what it predicted, how much probability sat on a given outcome, and whether it
+matched a label. Those live here so the three packages agree on the definitions
+rather than each inventing its own.
+
+The vocabulary is deliberately narrow:
+
+- **predicted** is the outcome the answer selects. For a Choice it is the option
+  key. For a Noul it is ``True`` when ``noul`` clears the threshold. For a Score
+  it is the index of the most probable level, which is *not* the same as
+  rounding ``score``.
+- **confidence** is what the API returned, and Nouls do not have one. A Noul's
+  distance from 0.5 is a different quantity and is exposed separately as
+  ``decisiveness`` rather than pretending it is the same number.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+__all__ = ["Answer", "parse_answer", "parse_answers", "NOUL_THRESHOLD"]
+
+NOUL_THRESHOLD = 0.5
+
+
+@dataclass
+class Answer:
+    """Normalized view over one answer from the API."""
+
+    id: str
+    type: str
+    raw: dict[str, Any]
+
+    # -- what it said ------------------------------------------------------
+
+    @property
+    def probabilities(self) -> dict[str, float]:
+        """Outcome -> probability.
+
+        Choice returns its options. Score returns its level indices as strings.
+        Noul has no distribution from the API, so the two-outcome distribution
+        it implies is synthesized here.
+        """
+        if self.type == "noul":
+            p = float(self.raw.get("noul", 0.0))
+            return {"true": p, "false": 1.0 - p}
+        probs = self.raw.get("probabilities") or {}
+        return {str(k): float(v) for k, v in probs.items()}
+
+    @property
+    def confidence(self) -> float | None:
+        """As returned by the API. ``None`` for a Noul, which carries none."""
+        value = self.raw.get("confidence")
+        return None if value is None else float(value)
+
+    @property
+    def decisiveness(self) -> float:
+        """How far from maximally uncertain this answer is, on 0..1.
+
+        For a Noul this is ``|noul - 0.5| * 2``. This is *not* confidence and is
+        not comparable to the API's confidence across question types; it exists
+        so Nouls can be thresholded on something with a defined meaning.
+        """
+        if self.type == "noul":
+            return abs(float(self.raw.get("noul", 0.0)) - NOUL_THRESHOLD) * 2.0
+        conf = self.confidence
+        return 0.0 if conf is None else conf
+
+    def predicted(self, *, noul_threshold: float = NOUL_THRESHOLD) -> Any:
+        """The outcome this answer selects."""
+        if self.type == "noul":
+            return float(self.raw.get("noul", 0.0)) >= noul_threshold
+        if self.type == "choice":
+            return self.raw.get("choice")
+        if self.type == "score":
+            probs = self.probabilities
+            if not probs:
+                return None
+            best = max(probs, key=lambda k: probs[k])
+            try:
+                return int(best)
+            except ValueError:
+                return best
+        return None
+
+    @property
+    def score(self) -> float | None:
+        """The probability-weighted score, for a Score answer."""
+        if self.type != "score":
+            return None
+        value = self.raw.get("score")
+        return None if value is None else float(value)
+
+    # -- how it compares to a label ---------------------------------------
+
+    def _label_key(self, label: Any) -> str:
+        if self.type == "noul":
+            return "true" if bool(label) else "false"
+        return str(label)
+
+    def probability_of(self, label: Any) -> float:
+        """Probability this answer assigned to ``label``.
+
+        Returns 0.0 for an outcome the question never offered, which is the
+        honest reading: the model could not have selected it.
+        """
+        return self.probabilities.get(self._label_key(label), 0.0)
+
+    def is_correct(self, label: Any, *, noul_threshold: float = NOUL_THRESHOLD) -> bool:
+        predicted = self.predicted(noul_threshold=noul_threshold)
+        if self.type == "noul":
+            return bool(predicted) is bool(label)
+        if self.type == "score":
+            try:
+                return int(predicted) == int(label)  # type: ignore[arg-type]
+            except (TypeError, ValueError):
+                return predicted == label
+        return predicted == label
+
+    @property
+    def top_probability(self) -> float:
+        """Probability mass on the predicted outcome."""
+        probs = self.probabilities
+        return max(probs.values()) if probs else 0.0
+
+
+def parse_answer(qid: str, raw: Any) -> Answer:
+    if not isinstance(raw, dict):
+        raise ValueError(f"answer {qid!r}: expected an object, got {type(raw).__name__}")
+    atype = raw.get("type")
+    if atype not in ("choice", "score", "noul"):
+        # Infer from shape when the API response omits the discriminator.
+        if "choice" in raw:
+            atype = "choice"
+        elif "noul" in raw:
+            atype = "noul"
+        elif "score" in raw:
+            atype = "score"
+        else:
+            raise ValueError(
+                f"answer {qid!r}: cannot determine type; expected a 'type' field or one "
+                f"of 'choice'/'score'/'noul'"
+            )
+    return Answer(id=qid, type=atype, raw=raw)
+
+
+def parse_answers(answers: dict[str, Any]) -> dict[str, Answer]:
+    return {qid: parse_answer(qid, raw) for qid, raw in answers.items()}

jevkit_core-0.2.0/src/jevkit_core/canonical.py

@@ -0,0 +1,57 @@
+"""RFC 8785 JSON Canonicalization, and the digests jevkit builds on it."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from typing import Any
+
+__all__ = ["canonical_json", "digest", "request_id", "record_id"]
+
+
+def _check_finite(value: Any) -> None:
+    if isinstance(value, float) and not math.isfinite(value):
+        raise ValueError(f"cannot canonicalize non-finite number: {value!r}")
+    if isinstance(value, dict):
+        for v in value.values():
+            _check_finite(v)
+    elif isinstance(value, (list, tuple)):
+        for v in value:
+            _check_finite(v)
+
+
+def canonical_json(value: Any) -> bytes:
+    """Serialize to RFC 8785 canonical form.
+
+    Object keys sorted by code point, no insignificant whitespace, UTF-8.
+    NaN and Infinity are rejected rather than emitted as invalid JSON.
+    """
+    _check_finite(value)
+    return json.dumps(
+        value,
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+        allow_nan=False,
+    ).encode("utf-8")
+
+
+def digest(value: Any) -> str:
+    """`sha256:<hex>` over the canonical form of ``value``."""
+    return "sha256:" + hashlib.sha256(canonical_json(value)).hexdigest()
+
+
+def request_id(state: Any, questions: dict[str, Any]) -> str:
+    """Digest of state + questions, with no model.
+
+    This is what pairs a golden-set record with its replay on a different
+    model version, since the full record id includes the model and would
+    therefore differ.
+    """
+    return digest({"questions": questions, "state": state})
+
+
+def record_id(model: str, state: Any, questions: dict[str, Any]) -> str:
+    """Digest of model + state + questions. The record's ``id`` field."""
+    return digest({"model": model, "questions": questions, "state": state})

jevkit_core-0.2.0/src/jevkit_core/question.py

@@ -0,0 +1,124 @@
+"""A uniform view of a jev question.
+
+Questions reach jevkit in three shapes: plain dicts as sent to the HTTP API,
+SDK objects (``Choice``/``Score``/``Noul``), and whatever a user's own helper
+produces. Every jevkit tool wants the same few facts out of them, so they are
+normalized once here rather than re-sniffed in each package.
+
+``instructions`` and ``criteria`` both accept JSON structure, not just
+strings, so the normalized form keeps the raw value and exposes the flattened
+text separately for anything doing textual analysis.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Iterator
+
+__all__ = ["Question", "QuestionType", "normalize_question", "normalize_questions",
+           "flatten_text"]
+
+QuestionType = str  # "choice" | "score" | "noul"
+
+_VALID_TYPES = ("choice", "score", "noul")
+
+
+def flatten_text(value: Any) -> str:
+    """Collapse a string / dict / list into one string for textual analysis.
+
+    Dict keys are included: in a Choice, the keys are the option names and
+    carry meaning the model sees.
+    """
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (int, float, bool)):
+        return str(value)
+    if isinstance(value, dict):
+        parts: list[str] = []
+        for k, v in value.items():
+            parts.append(str(k))
+            parts.append(flatten_text(v))
+        return " ".join(p for p in parts if p)
+    if isinstance(value, (list, tuple)):
+        return " ".join(p for p in (flatten_text(v) for v in value) if p)
+    return str(value)
+
+
+@dataclass
+class Question:
+    """Normalized question. ``raw`` is always the untouched original."""
+
+    id: str
+    type: QuestionType
+    instructions: Any
+    criteria: Any
+    raw: Any = field(repr=False, default=None)
+
+    @property
+    def instructions_text(self) -> str:
+        return flatten_text(self.instructions)
+
+    @property
+    def criteria_text(self) -> str:
+        return flatten_text(self.criteria)
+
+    @property
+    def text(self) -> str:
+        """Everything the model reads, as one string."""
+        return f"{self.instructions_text} {self.criteria_text}".strip()
+
+    @property
+    def options(self) -> list[str]:
+        """Choice option keys, or Score level descriptions. Empty otherwise."""
+        if self.type == "choice" and isinstance(self.criteria, dict):
+            return [str(k) for k in self.criteria.keys()]
+        if self.type == "score" and isinstance(self.criteria, (list, tuple)):
+            return [flatten_text(v) for v in self.criteria]
+        if self.type == "score" and isinstance(self.criteria, dict):
+            return [flatten_text(v) for v in self.criteria.values()]
+        return []
+
+    def option_descriptions(self) -> Iterator[tuple[str, str]]:
+        """(option key, description) pairs for a Choice."""
+        if self.type == "choice" and isinstance(self.criteria, dict):
+            for k, v in self.criteria.items():
+                yield str(k), flatten_text(v)
+
+
+def _get(obj: Any, name: str) -> Any:
+    if isinstance(obj, dict):
+        return obj.get(name)
+    return getattr(obj, name, None)
+
+
+def _infer_type(obj: Any) -> str | None:
+    declared = _get(obj, "type")
+    if isinstance(declared, str) and declared.lower() in _VALID_TYPES:
+        return declared.lower()
+    # SDK objects carry no "type" field; fall back to the class name.
+    cls = type(obj).__name__.lower()
+    if cls in _VALID_TYPES:
+        return cls
+    return None
+
+
+def normalize_question(qid: str, obj: Any) -> Question:
+    qtype = _infer_type(obj)
+    if qtype is None:
+        raise ValueError(
+            f"question {qid!r}: cannot determine type. Expected a 'type' key of "
+            f"{_VALID_TYPES}, or a Choice/Score/Noul object."
+        )
+    return Question(
+        id=qid,
+        type=qtype,
+        instructions=_get(obj, "instructions"),
+        criteria=_get(obj, "criteria"),
+        raw=obj,
+    )
+
+
+def normalize_questions(questions: dict[str, Any]) -> list[Question]:
+    return [normalize_question(qid, obj) for qid, obj in questions.items()]

jevkit_core-0.2.0/src/jevkit_core/record.py

@@ -0,0 +1,164 @@
+"""Reading and writing `.jevl` records. See docs/specs/record-format-v1.md."""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, IO, Iterable, Iterator
+
+from .canonical import record_id, request_id
+
+__all__ = ["Record", "FORMAT_VERSION", "RecordFormatError",
+           "read_records", "write_records", "append_record", "load_cassette"]
+
+FORMAT_VERSION = 1
+
+_REQUIRED = ("v", "id", "ts", "model", "request", "answers")
+
+
+class RecordFormatError(ValueError):
+    """A `.jevl` line is malformed, or its version is unreadable."""
+
+
+def _utcnow() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+@dataclass
+class Record:
+    model: str
+    state: Any
+    questions: dict[str, Any]
+    answers: dict[str, Any]
+    id: str = ""
+    ts: str = field(default_factory=_utcnow)
+    usage: dict[str, Any] | None = None
+    label: dict[str, Any] | None = None
+    tags: list[str] = field(default_factory=list)
+    meta: dict[str, Any] = field(default_factory=dict)
+    extra: dict[str, Any] = field(default_factory=dict, repr=False)
+
+    def __post_init__(self) -> None:
+        if not self.id:
+            self.id = record_id(self.model, self.state, self.questions)
+
+    @property
+    def request_id(self) -> str:
+        """Model-independent digest, used to pair across model versions."""
+        return request_id(self.state, self.questions)
+
+    def to_dict(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "v": FORMAT_VERSION,
+            "id": self.id,
+            "ts": self.ts,
+            "model": self.model,
+            "request": {"state": self.state, "questions": self.questions},
+            "answers": self.answers,
+        }
+        if self.usage is not None:
+            out["usage"] = self.usage
+        if self.label is not None:
+            out["label"] = self.label
+        if self.tags:
+            out["tags"] = list(self.tags)
+        if self.meta:
+            out["meta"] = dict(self.meta)
+        # Unknown keys survive a read/write round trip.
+        for k, v in self.extra.items():
+            out.setdefault(k, v)
+        return out
+
+    @classmethod
+    def from_dict(cls, raw: dict[str, Any], *, source: str = "<memory>",
+                  line_no: int = 0) -> "Record":
+        where = f"{source}:{line_no}" if line_no else source
+        if not isinstance(raw, dict):
+            raise RecordFormatError(f"{where}: record must be a JSON object")
+
+        version = raw.get("v")
+        if version is None:
+            raise RecordFormatError(f"{where}: missing required key 'v'")
+        if not isinstance(version, int):
+            raise RecordFormatError(f"{where}: 'v' must be an integer, got {version!r}")
+        if version > FORMAT_VERSION:
+            raise RecordFormatError(
+                f"{where}: record format v{version} is newer than this reader "
+                f"(v{FORMAT_VERSION}). Upgrade jevkit-core rather than guessing."
+            )
+
+        missing = [k for k in _REQUIRED if k not in raw]
+        if missing:
+            raise RecordFormatError(f"{where}: missing required key(s): {', '.join(missing)}")
+
+        request = raw["request"]
+        if not isinstance(request, dict) or "questions" not in request:
+            raise RecordFormatError(f"{where}: 'request' must be an object with 'questions'")
+
+        known = set(_REQUIRED) | {"usage", "label", "tags", "meta"}
+        return cls(
+            model=raw["model"],
+            state=request.get("state"),
+            questions=request["questions"],
+            answers=raw["answers"],
+            id=raw["id"],
+            ts=raw["ts"],
+            usage=raw.get("usage"),
+            label=raw.get("label"),
+            tags=list(raw.get("tags") or []),
+            meta=dict(raw.get("meta") or {}),
+            extra={k: v for k, v in raw.items() if k not in known},
+        )
+
+
+def read_records(path: str | os.PathLike[str] | IO[str]) -> Iterator[Record]:
+    """Yield records from a `.jevl` file.
+
+    A line that fails to parse raises. Skipping silently would let a truncated
+    golden set look like a passing one.
+    """
+    if hasattr(path, "read"):
+        yield from _read_stream(path, getattr(path, "name", "<stream>"))  # type: ignore[arg-type]
+        return
+    with open(path, "r", encoding="utf-8") as fh:
+        yield from _read_stream(fh, str(path))
+
+
+def _read_stream(fh: IO[str], source: str) -> Iterator[Record]:
+    for line_no, line in enumerate(fh, start=1):
+        stripped = line.strip()
+        if not stripped:
+            continue
+        try:
+            raw = json.loads(stripped)
+        except json.JSONDecodeError as exc:
+            raise RecordFormatError(f"{source}:{line_no}: invalid JSON: {exc}") from exc
+        yield Record.from_dict(raw, source=source, line_no=line_no)
+
+
+def write_records(path: str | os.PathLike[str], records: Iterable[Record]) -> int:
+    count = 0
+    with open(path, "w", encoding="utf-8") as fh:
+        for record in records:
+            fh.write(json.dumps(record.to_dict(), ensure_ascii=False) + "\n")
+            count += 1
+    return count
+
+
+def append_record(path: str | os.PathLike[str], record: Record) -> None:
+    with open(path, "a", encoding="utf-8") as fh:
+        fh.write(json.dumps(record.to_dict(), ensure_ascii=False) + "\n")
+
+
+def load_cassette(path: str | os.PathLike[str]) -> dict[str, Record]:
+    """Map record id -> Record, last occurrence winning.
+
+    Re-recording a request appends rather than rewrites, so the last line for
+    an id is the current answer.
+    """
+    table: dict[str, Record] = {}
+    for record in read_records(path):
+        table[record.id] = record
+    return table

jevkit_core-0.2.0/src/jevkit_core/tokens.py

@@ -0,0 +1,87 @@
+"""Token budget estimation for jev requests.
+
+Jev ingests the state once and evaluates every question against it, so two
+separate budgets apply. Both are documented on the model card:
+
+    64k  state + every question combined
+    32k  state + the single longest question
+
+These are *estimates*. jevkit deliberately does not bundle a tokenizer: the
+dependency is heavy, the vendor does not publish which tokenizer Jev uses,
+and a wrong tokenizer is more misleading than an honest approximation. The
+estimator is intentionally conservative so that a request jevkit calls safe
+is very unlikely to be rejected.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .canonical import canonical_json
+
+__all__ = ["estimate_tokens", "BudgetReport", "check_budget",
+           "TOTAL_BUDGET", "STATE_BUDGET"]
+
+TOTAL_BUDGET = 64_000
+STATE_BUDGET = 32_000
+
+# Bytes per token. Real-world English on BPE-family tokenizers runs ~4.0;
+# 3.5 buys headroom for punctuation-dense JSON without being absurd.
+_BYTES_PER_TOKEN = 3.5
+
+
+def estimate_tokens(value: Any) -> int:
+    """Conservative token estimate for any JSON-serializable value."""
+    if isinstance(value, str):
+        payload = value.encode("utf-8")
+    else:
+        payload = canonical_json(value)
+    return max(1, int(len(payload) / _BYTES_PER_TOKEN) + 1)
+
+
+class BudgetReport:
+    """Where a request sits against both documented budgets."""
+
+    __slots__ = ("state_tokens", "question_tokens", "longest_question_id")
+
+    def __init__(
+        self,
+        state_tokens: int,
+        question_tokens: dict[str, int],
+        longest_question_id: str | None,
+    ) -> None:
+        self.state_tokens = state_tokens
+        self.question_tokens = question_tokens
+        self.longest_question_id = longest_question_id
+
+    @property
+    def total(self) -> int:
+        return self.state_tokens + sum(self.question_tokens.values())
+
+    @property
+    def longest_pair(self) -> int:
+        """state + the single longest question."""
+        if self.longest_question_id is None:
+            return self.state_tokens
+        return self.state_tokens + self.question_tokens[self.longest_question_id]
+
+    @property
+    def over_total(self) -> bool:
+        return self.total > TOTAL_BUDGET
+
+    @property
+    def over_state(self) -> bool:
+        return self.longest_pair > STATE_BUDGET
+
+    def __repr__(self) -> str:  # pragma: no cover - debug aid
+        return (
+            f"BudgetReport(total={self.total}, longest_pair={self.longest_pair}, "
+            f"over_total={self.over_total}, over_state={self.over_state})"
+        )
+
+
+def check_budget(state: Any, questions: dict[str, Any]) -> BudgetReport:
+    state_tokens = estimate_tokens(state)
+    per_question = {qid: estimate_tokens(q) for qid, q in questions.items()}
+    longest = max(per_question, key=lambda k: per_question[k]) if per_question else None
+    return BudgetReport(state_tokens, per_question, longest)

jevkit_core-0.2.0/tests/test_answers.py

@@ -0,0 +1,66 @@
+import pytest
+from jevkit_core import parse_answer, parse_answers
+
+CHOICE = {"type": "choice", "choice": "billing",
+          "probabilities": {"billing": 0.7, "technical": 0.3}, "confidence": 0.55}
+SCORE = {"type": "score", "score": 1.03,
+         "probabilities": {"0": 0.1, "1": 0.8, "2": 0.1}, "confidence": 0.84}
+NOUL = {"type": "noul", "noul": 0.99}
+
+
+def test_choice_predicts_its_selection():
+    a = parse_answer("q", CHOICE)
+    assert a.predicted() == "billing"
+    assert a.probability_of("billing") == 0.7
+    assert a.is_correct("billing") and not a.is_correct("technical")
+
+
+def test_noul_predicts_a_bool_against_the_threshold():
+    assert parse_answer("q", NOUL).predicted() is True
+    assert parse_answer("q", {"type": "noul", "noul": 0.2}).predicted() is False
+    assert parse_answer("q", {"type": "noul", "noul": 0.5}).predicted() is True
+
+
+def test_noul_synthesizes_a_two_outcome_distribution():
+    assert parse_answer("q", NOUL).probabilities == {"true": 0.99, "false": pytest.approx(0.01)}
+
+
+def test_noul_has_no_confidence_but_has_decisiveness():
+    a = parse_answer("q", NOUL)
+    assert a.confidence is None
+    assert a.decisiveness == pytest.approx(0.98)
+
+
+def test_score_predicts_the_most_probable_level_not_the_rounded_score():
+    # score is 1.03 but the mass could sit elsewhere; predicted follows the mass.
+    a = parse_answer("q", {"type": "score", "score": 1.03,
+                           "probabilities": {"0": 0.45, "1": 0.1, "2": 0.45}})
+    assert a.predicted() == 0
+    assert parse_answer("q", SCORE).predicted() == 1
+
+
+def test_score_is_correct_against_an_integer_label():
+    assert parse_answer("q", SCORE).is_correct(1)
+    assert not parse_answer("q", SCORE).is_correct(2)
+
+
+def test_probability_of_an_unoffered_outcome_is_zero():
+    assert parse_answer("q", CHOICE).probability_of("sales") == 0.0
+
+
+def test_top_probability_is_the_mass_on_the_prediction():
+    assert parse_answer("q", CHOICE).top_probability == 0.7
+
+
+def test_type_is_inferred_when_the_discriminator_is_missing():
+    assert parse_answer("q", {"choice": "a", "probabilities": {"a": 1.0}}).type == "choice"
+    assert parse_answer("q", {"noul": 0.4}).type == "noul"
+
+
+def test_unrecognisable_answer_raises():
+    with pytest.raises(ValueError, match="cannot determine type"):
+        parse_answer("q", {"mystery": 1})
+
+
+def test_parse_answers_maps_every_question():
+    assert set(parse_answers({"a": CHOICE, "b": NOUL})) == {"a", "b"}

jevkit_core-0.2.0/tests/test_canonical.py

@@ -0,0 +1,40 @@
+import json
+import pytest
+from jevkit_core import canonical_json, digest, record_id, request_id
+
+
+def test_key_order_does_not_change_canonical_form():
+    assert canonical_json({"b": 1, "a": 2}) == canonical_json({"a": 2, "b": 1})
+
+
+def test_canonical_form_has_no_insignificant_whitespace():
+    assert canonical_json({"a": [1, 2]}) == b'{"a":[1,2]}'
+
+
+def test_canonical_form_is_utf8_not_escaped():
+    assert canonical_json({"k": "café"}) == '{"k":"café"}'.encode("utf-8")
+
+
+def test_non_finite_numbers_are_rejected():
+    for bad in (float("nan"), float("inf"), float("-inf")):
+        with pytest.raises(ValueError):
+            canonical_json({"x": bad})
+    with pytest.raises(ValueError):
+        canonical_json({"nested": {"deep": [float("nan")]}})
+
+
+def test_digest_is_stable_and_prefixed():
+    d = digest({"a": 1})
+    assert d.startswith("sha256:") and len(d) == 71
+    assert d == digest({"a": 1})
+
+
+def test_request_id_ignores_model_but_record_id_does_not():
+    state, questions = "hello", {"q": {"type": "noul", "instructions": "ok?"}}
+    assert record_id("jev-1.13.0", state, questions) != record_id("jev-1.14.0", state, questions)
+    assert request_id(state, questions) == request_id(state, questions)
+
+
+def test_record_id_changes_when_state_changes():
+    q = {"q": {"type": "noul", "instructions": "ok?"}}
+    assert record_id("m", "a", q) != record_id("m", "b", q)

jevkit_core-0.2.0/tests/test_record.py

@@ -0,0 +1,91 @@
+import io
+import json
+import pytest
+from jevkit_core import (FORMAT_VERSION, Record, RecordFormatError, load_cassette,
+                         read_records, write_records)
+
+
+def make(model="jev-1.13.0", state="s", answers=None):
+    return Record(model=model, state=state,
+                  questions={"q": {"type": "noul", "instructions": "ok?"}},
+                  answers=answers or {"q": {"type": "noul", "noul": 0.9}})
+
+
+def test_id_is_derived_when_absent():
+    assert make().id.startswith("sha256:")
+
+
+def test_round_trip_preserves_fields(tmp_path):
+    path = tmp_path / "a.jevl"
+    original = make()
+    original.tags = ["x"]
+    original.usage = {"input_tokens": 10}
+    assert write_records(path, [original]) == 1
+    (restored,) = list(read_records(path))
+    assert restored.id == original.id
+    assert restored.model == original.model
+    assert restored.tags == ["x"]
+    assert restored.usage == {"input_tokens": 10}
+
+
+def test_unknown_keys_survive_round_trip(tmp_path):
+    path = tmp_path / "a.jevl"
+    raw = make().to_dict()
+    raw["future_field"] = {"kept": True}
+    path.write_text(json.dumps(raw) + "\n")
+    (record,) = list(read_records(path))
+    assert record.extra["future_field"] == {"kept": True}
+    assert record.to_dict()["future_field"] == {"kept": True}
+
+
+def test_newer_format_version_is_refused(tmp_path):
+    path = tmp_path / "a.jevl"
+    raw = make().to_dict()
+    raw["v"] = FORMAT_VERSION + 1
+    path.write_text(json.dumps(raw) + "\n")
+    with pytest.raises(RecordFormatError, match="newer than this reader"):
+        list(read_records(path))
+
+
+def test_malformed_line_raises_rather_than_skipping(tmp_path):
+    path = tmp_path / "a.jevl"
+    path.write_text(json.dumps(make().to_dict()) + "\nnot json\n")
+    with pytest.raises(RecordFormatError, match="invalid JSON"):
+        list(read_records(path))
+
+
+def test_missing_required_key_raises(tmp_path):
+    path = tmp_path / "a.jevl"
+    raw = make().to_dict()
+    del raw["answers"]
+    path.write_text(json.dumps(raw) + "\n")
+    with pytest.raises(RecordFormatError, match="missing required key"):
+        list(read_records(path))
+
+
+def test_blank_lines_are_skipped(tmp_path):
+    path = tmp_path / "a.jevl"
+    path.write_text("\n" + json.dumps(make().to_dict()) + "\n\n")
+    assert len(list(read_records(path))) == 1
+
+
+def test_cassette_last_record_wins(tmp_path):
+    path = tmp_path / "a.jevl"
+    first = make(answers={"q": {"type": "noul", "noul": 0.1}})
+    second = make(answers={"q": {"type": "noul", "noul": 0.9}})
+    assert first.id == second.id
+    write_records(path, [first, second])
+    table = load_cassette(path)
+    assert len(table) == 1
+    assert table[first.id].answers["q"]["noul"] == 0.9
+
+
+def test_request_id_pairs_records_across_model_versions():
+    a, b = make(model="jev-1.13.0"), make(model="jev-1.14.0")
+    assert a.id != b.id
+    assert a.request_id == b.request_id
+
+
+def test_reads_from_a_stream():
+    text = json.dumps(make().to_dict()) + "\n"
+    assert len(list(read_records(io.StringIO(text)))) == 1

jevkit_core-0.2.0/tests/test_tokens.py

@@ -0,0 +1,39 @@
+from jevkit_core import STATE_BUDGET, TOTAL_BUDGET, check_budget, estimate_tokens
+
+
+def test_estimate_grows_with_length():
+    assert estimate_tokens("a" * 1000) > estimate_tokens("a" * 10)
+
+
+def test_estimate_is_never_zero():
+    assert estimate_tokens("") >= 1
+
+
+def test_small_request_is_within_both_budgets():
+    report = check_budget("short state", {"q": {"type": "noul", "instructions": "ok?"}})
+    assert not report.over_state and not report.over_total
+
+
+def test_oversized_state_trips_the_state_budget():
+    report = check_budget("x" * (STATE_BUDGET * 4), {"q": {"type": "noul", "instructions": "ok?"}})
+    assert report.over_state
+
+
+def test_many_questions_trip_the_total_budget():
+    questions = {f"q{i}": {"type": "noul", "instructions": "y" * 2000} for i in range(200)}
+    report = check_budget("s", questions)
+    assert report.over_total
+
+
+def test_longest_question_is_identified():
+    report = check_budget("s", {
+        "small": {"type": "noul", "instructions": "a"},
+        "big": {"type": "noul", "instructions": "a" * 500},
+    })
+    assert report.longest_question_id == "big"
+
+
+def test_empty_questions_does_not_crash():
+    report = check_budget("s", {})
+    assert report.longest_question_id is None
+    assert report.longest_pair == report.state_tokens