jevcheck 0.2.0__py3-none-any.whl

jevcheck/__init__.py

@@ -0,0 +1,57 @@
+"""jevcheck — pytest for Jev behavior."""
+
+from jevcheck.answers import (
+    ChoiceAnswer,
+    JevResponse,
+    NoulAnswer,
+    ScoreAnswer,
+    Usage,
+    mapped_confidence,
+)
+from jevcheck.client import AUTH_ENV, JevClient
+from jevcheck.compare import compare, record_answers
+from jevcheck.contract import Case, Contract, FieldExpect, load_contract, load_replay, write_replay
+from jevcheck.eval import CaseResult, EvalReport, Outcome, evaluate, evaluate_case
+from jevcheck.gate import Action, Decision, Gate
+from jevcheck.pinning import (
+    ModelIdentityError,
+    UnpinnedModelError,
+    is_unpinned,
+    require_pinned,
+    require_response_identity,
+)
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "AUTH_ENV",
+    "Action",
+    "Case",
+    "CaseResult",
+    "ChoiceAnswer",
+    "Contract",
+    "Decision",
+    "EvalReport",
+    "compare",
+    "FieldExpect",
+    "Gate",
+    "JevClient",
+    "JevResponse",
+    "ModelIdentityError",
+    "NoulAnswer",
+    "Outcome",
+    "ScoreAnswer",
+    "UnpinnedModelError",
+    "Usage",
+    "evaluate",
+    "evaluate_case",
+    "is_unpinned",
+    "load_contract",
+    "load_replay",
+    "record_answers",
+    "write_replay",
+    "mapped_confidence",
+    "require_pinned",
+    "require_response_identity",
+    "__version__",
+]

jevcheck/__main__.py

@@ -0,0 +1,3 @@
+from jevcheck.cli import main
+
+raise SystemExit(main())

jevcheck/answers.py

@@ -0,0 +1,228 @@
+"""Verified System One answer models. Fields match docs/jev-api.md only."""
+
+from __future__ import annotations
+
+import math
+from typing import Annotated, Any, Literal, Mapping
+
+from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, field_validator
+
+from jevcheck.questions import JSONContent
+
+# Probability maps must be finite, nonnegative, and sum to 1 within this
+# absolute tolerance. JSON/IEEE floats will not hit 1.0 exactly for many
+# hand-written distributions; 1e-6 is the agreed check, not a fixture-format
+# change. Maps that miss this are rejected rather than silently renormalized.
+PROBABILITY_SUM_TOLERANCE = 1e-6
+
+
+class Usage(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    input_tokens: StrictInt | None = None
+    output_tokens: StrictInt | None = None
+
+
+class ChoiceAnswer(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: Literal["choice"] = "choice"
+    choice: StrictStr
+    confidence: float = Field(ge=0.0, le=1.0)
+    probabilities: dict[str, float]
+
+    @field_validator("confidence")
+    @classmethod
+    def _finite_confidence(cls, value: float) -> float:
+        return _require_finite(value, "choice confidence")
+
+    @field_validator("probabilities")
+    @classmethod
+    def _valid_probabilities(cls, value: dict[str, float]) -> dict[str, float]:
+        return validate_probability_map(value, what="choice")
+
+
+class NoulAnswer(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: Literal["noul"] = "noul"
+    noul: float = Field(ge=0.0, le=1.0)
+
+    @field_validator("noul")
+    @classmethod
+    def _finite_noul(cls, value: float) -> float:
+        return _require_finite(value, "noul")
+
+
+class ScoreAnswer(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: Literal["score"] = "score"
+    score: float
+    confidence: float = Field(ge=0.0, le=1.0)
+    legend: dict[str, JSONContent]
+    probabilities: dict[str, float]
+
+    @field_validator("score", "confidence")
+    @classmethod
+    def _finite_score_fields(cls, value: float) -> float:
+        return _require_finite(value, "score field")
+
+    @field_validator("legend", "probabilities", mode="before")
+    @classmethod
+    def _stringify_int_keys(cls, value: Any) -> Any:
+        """SDK 0.7.0 score maps use integer keys; coerce to str without inventing fields."""
+        if isinstance(value, Mapping):
+            return {str(key): item for key, item in value.items()}
+        return value
+
+    @field_validator("legend")
+    @classmethod
+    def _valid_legend(cls, value: dict[str, JSONContent]) -> dict[str, JSONContent]:
+        if not value:
+            raise ValueError("score legend must be a nonempty mapping")
+        return value
+
+    @field_validator("probabilities")
+    @classmethod
+    def _valid_probabilities(cls, value: dict[str, float]) -> dict[str, float]:
+        return validate_probability_map(value, what="score")
+
+
+Answer = Annotated[
+    NoulAnswer | ChoiceAnswer | ScoreAnswer,
+    Field(discriminator="type"),
+]
+
+
+class JevResponse(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    model: StrictStr
+    usage: Usage = Field(default_factory=Usage)
+    answers: dict[str, Answer] = Field(default_factory=dict)
+
+    @field_validator("model")
+    @classmethod
+    def _nonempty_model(cls, value: str) -> str:
+        if not value.strip():
+            raise ValueError("response.model must be a nonempty string")
+        return value
+
+
+def mapped_confidence(answer: NoulAnswer | ChoiceAnswer | ScoreAnswer) -> float:
+    """Scalar used for floors, regressions, and the optional Gate helper."""
+    if isinstance(answer, NoulAnswer):
+        return answer.noul
+    return answer.confidence
+
+
+def adapt_answer(raw: Any) -> NoulAnswer | ChoiceAnswer | ScoreAnswer:
+    """Accept SDK models, namespaces, or verified dicts.
+
+    Missing choice or score probabilities are rejected (not replaced with ``{}``).
+    Score ``legend`` is required and nonempty, matching SDK-shaped responses.
+    Score ``legend`` / ``probabilities`` integer keys from typesafe-sdk 0.7.0
+    are stringified so they match this schema.
+    """
+    if isinstance(raw, (NoulAnswer, ChoiceAnswer, ScoreAnswer)):
+        return raw
+    data = _to_mapping(raw)
+    kind = data.get("type")
+    if kind == "noul" or ("noul" in data and "choice" not in data and "score" not in data):
+        return NoulAnswer.model_validate({"type": "noul", "noul": data["noul"]})
+    if kind == "choice" or "choice" in data:
+        payload: dict[str, Any] = {
+            "type": "choice",
+            "choice": data["choice"],
+            "confidence": data["confidence"],
+        }
+        if "probabilities" in data:
+            payload["probabilities"] = data["probabilities"]
+        return ChoiceAnswer.model_validate(payload)
+    if kind == "score" or "score" in data:
+        payload = {
+            "type": "score",
+            "score": data["score"],
+            "confidence": data["confidence"],
+        }
+        if "legend" in data:
+            payload["legend"] = data["legend"]
+        if "probabilities" in data:
+            payload["probabilities"] = data["probabilities"]
+        return ScoreAnswer.model_validate(payload)
+    raise ValueError(f"unsupported answer object: {raw!r}")
+
+
+def adapt_response(raw: Any) -> JevResponse:
+    if isinstance(raw, JevResponse):
+        return raw
+    data = _to_mapping(raw)
+    model = data.get("model", _MISSING)
+    if model is _MISSING:
+        raise ValueError("response.model is required")
+    if model is None:
+        raise ValueError("response.model must not be null")
+    if not isinstance(model, str):
+        raise ValueError(f"response.model must be a string, not {type(model).__name__}")
+    answers_raw = data.get("answers") or {}
+    if not isinstance(answers_raw, Mapping):
+        raise TypeError("response.answers must be a mapping")
+    usage_raw = data.get("usage") or {}
+    return JevResponse(
+        model=model,
+        usage=Usage.model_validate(_to_mapping(usage_raw) if usage_raw is not None else {}),
+        answers={str(name): adapt_answer(answer) for name, answer in answers_raw.items()},
+    )
+
+
+def validate_probability_map(value: Any, *, what: str) -> dict[str, float]:
+    if not isinstance(value, Mapping) or not value:
+        raise ValueError(f"{what} probabilities must be a nonempty mapping")
+    cleaned: dict[str, float] = {}
+    for key, raw in value.items():
+        name = str(key)
+        try:
+            number = float(raw)
+        except (TypeError, ValueError) as exc:
+            raise ValueError(f"{what} probability {name!r} must be a finite number") from exc
+        if not math.isfinite(number):
+            raise ValueError(f"{what} probability {name!r} must be finite")
+        if number < 0.0:
+            raise ValueError(f"{what} probability {name!r} must be >= 0")
+        cleaned[name] = number
+    total = math.fsum(cleaned.values())
+    if abs(total - 1.0) > PROBABILITY_SUM_TOLERANCE:
+        raise ValueError(
+            f"{what} probabilities must sum to 1 ± {PROBABILITY_SUM_TOLERANCE}; got {total}"
+        )
+    return cleaned
+
+
+def _require_finite(value: float, what: str) -> float:
+    if not math.isfinite(value):
+        raise ValueError(f"{what} must be a finite number")
+    return value
+
+
+def _to_mapping(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, Mapping):
+        return dict(raw)
+    if hasattr(raw, "model_dump"):
+        dumped = raw.model_dump()
+        if isinstance(dumped, Mapping):
+            return dict(dumped)
+    skip = {"model_config", "model_fields", "model_computed_fields"}
+    if hasattr(raw, "__dict__"):
+        return {k: v for k, v in vars(raw).items() if not k.startswith("_") and k not in skip}
+    public = {
+        name: getattr(raw, name)
+        for name in dir(raw)
+        if not name.startswith("_") and not callable(getattr(raw, name, None))
+    }
+    if public:
+        return public
+    raise TypeError(f"cannot adapt {type(raw).__name__} to a mapping")
+
+
+_MISSING = object()

jevcheck/cli.py

@@ -0,0 +1,356 @@
+"""jevcheck CLI — eval a candidate model against a pinned contract."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from typesafe_sdk import (
+    TypeSafeAPIError,
+    TypeSafeAPIResponseValidationError,
+    TypeSafeAuthenticationError,
+    TypeSafeError,
+    TypeSafeInternalServerError,
+    TypeSafeRateLimitError,
+)
+
+from jevcheck.answers import JevResponse
+from jevcheck.client import AUTH_ENV, JevClient
+from jevcheck.compare import compare, record_answers
+from jevcheck.contract import Case, Contract, load_contract, load_replay, write_replay
+from jevcheck.eval import Fetch, evaluate
+from jevcheck.pinning import (
+    ModelIdentityError,
+    UnpinnedModelError,
+    format_resolved_model,
+    require_pinned,
+)
+
+# Exit 0: compatible. Exit 1: behavioral contract failure.
+# Exit 2: usage / input / identity. Exit 3: operational (HTTP / invalid API body).
+EXIT_OK = 0
+EXIT_BREAKING = 1
+EXIT_USAGE = 2
+EXIT_OPS = 3
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="jevcheck",
+        description="Pin Jev production contracts and eval candidate model upgrades.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    eval_cmd = sub.add_parser("eval", help="compare a candidate model to a contract")
+    eval_cmd.add_argument("contract", type=Path, help="JSON contract or JSONL fixtures")
+    eval_cmd.add_argument(
+        "--candidate-model",
+        required=True,
+        help="explicit candidate model (upgrades must be intentional)",
+    )
+    eval_cmd.add_argument(
+        "--baseline-model",
+        default=None,
+        help="override contract baseline_model (required for JSONL without _meta)",
+    )
+    eval_cmd.add_argument(
+        "--answers",
+        type=Path,
+        default=None,
+        help="replay file of mocked System One responses keyed by case id",
+    )
+    eval_cmd.add_argument(
+        "--allow-unpinned",
+        action="store_true",
+        help=(
+            "opt in to floating names such as jev-latest and jev-preview; "
+            "accept a concrete resolved response model (not another alias)"
+        ),
+    )
+
+    record_cmd = sub.add_parser(
+        "record",
+        help="fetch baseline answers and write a replay JSON (v0.2)",
+    )
+    record_cmd.add_argument("contract", type=Path, help="JSON contract or JSONL fixtures")
+    record_cmd.add_argument(
+        "--out",
+        type=Path,
+        required=True,
+        help="path to write a replay file usable by eval --answers / compare --from",
+    )
+    record_cmd.add_argument(
+        "--baseline-model",
+        default=None,
+        help="override contract baseline_model (required for JSONL without _meta)",
+    )
+    record_cmd.add_argument(
+        "--answers",
+        type=Path,
+        default=None,
+        help="replay file of mocked baseline responses (CI / tests; no live key)",
+    )
+    record_cmd.add_argument(
+        "--allow-unpinned",
+        action="store_true",
+        help=(
+            "opt in to floating names such as jev-latest and jev-preview; "
+            "accept a concrete resolved response model (not another alias)"
+        ),
+    )
+
+    compare_cmd = sub.add_parser(
+        "compare",
+        help="evaluate a candidate against recorded or live baseline answers (v0.2)",
+    )
+    compare_cmd.add_argument("contract", type=Path, help="JSON contract or JSONL fixtures")
+    source = compare_cmd.add_mutually_exclusive_group(required=True)
+    source.add_argument(
+        "--from",
+        dest="from_replay",
+        type=Path,
+        default=None,
+        help="recorded baseline replay JSON (output of jevcheck record)",
+    )
+    source.add_argument(
+        "--from-model",
+        dest="from_model",
+        default=None,
+        help="live baseline model (fetches both models when --answers is omitted)",
+    )
+    compare_cmd.add_argument(
+        "--to",
+        dest="to_model",
+        required=True,
+        help="explicit candidate model (same pinning rules as eval --candidate-model)",
+    )
+    compare_cmd.add_argument(
+        "--baseline-model",
+        default=None,
+        help="override contract baseline_model (required for JSONL without _meta)",
+    )
+    compare_cmd.add_argument(
+        "--answers",
+        type=Path,
+        default=None,
+        help="replay file of mocked candidate responses keyed by case id",
+    )
+    compare_cmd.add_argument(
+        "--allow-unpinned",
+        action="store_true",
+        help=(
+            "opt in to floating names such as jev-latest and jev-preview; "
+            "accept a concrete resolved response model (not another alias)"
+        ),
+    )
+
+    args = parser.parse_args(list(argv) if argv is not None else None)
+    handlers = {
+        "eval": _run_eval,
+        "record": _run_record,
+        "compare": _run_compare,
+    }
+    if args.command not in handlers:
+        parser.error("unknown command")
+
+    try:
+        return handlers[args.command](args)
+    except UnpinnedModelError as exc:
+        _print_error(exc)
+        return EXIT_USAGE
+    except ModelIdentityError as exc:
+        _print_error(exc)
+        return EXIT_USAGE
+    except TypeSafeAuthenticationError as exc:
+        _print_error(f"authentication failed ({exc.status}): {exc}")
+        return EXIT_OPS
+    except TypeSafeRateLimitError as exc:
+        _print_error(f"rate limited ({exc.status}): {exc}")
+        return EXIT_OPS
+    except TypeSafeInternalServerError as exc:
+        _print_error(f"server error ({exc.status}): {exc}")
+        return EXIT_OPS
+    except TypeSafeAPIResponseValidationError as exc:
+        _print_error(f"invalid API response: {exc}")
+        return EXIT_OPS
+    except TypeSafeAPIError as exc:
+        _print_error(f"API error ({exc.status}): {exc}")
+        return EXIT_OPS
+    except TypeSafeError as exc:
+        _print_error(exc)
+        return EXIT_USAGE
+    except (OSError, TypeError, ValueError, KeyError, RuntimeError) as exc:
+        _print_error(exc)
+        return EXIT_USAGE
+
+
+def _run_eval(args: argparse.Namespace) -> int:
+    allow = bool(args.allow_unpinned)
+    candidate = require_pinned(args.candidate_model, allow_unpinned=allow)
+    contract: Contract = load_contract(
+        args.contract,
+        baseline_model=args.baseline_model,
+        allow_unpinned=allow,
+    )
+    fetch = _answers_or_live_fetch(
+        contract,
+        args.answers,
+        model=candidate,
+        allow=allow,
+        live_need="live eval",
+    )
+    report = evaluate(
+        contract,
+        fetch,
+        candidate_model=candidate,
+        allow_unpinned=allow,
+    )
+    sys.stdout.write(report.summary())
+    return EXIT_BREAKING if report.breaking else EXIT_OK
+
+
+def _run_record(args: argparse.Namespace) -> int:
+    allow = bool(args.allow_unpinned)
+    contract = load_contract(
+        args.contract,
+        baseline_model=args.baseline_model,
+        allow_unpinned=allow,
+    )
+    baseline = require_pinned(contract.baseline_model, allow_unpinned=allow)
+    fetch = _answers_or_live_fetch(
+        contract,
+        args.answers,
+        model=baseline,
+        allow=allow,
+        live_need="live record",
+    )
+    recorded = record_answers(
+        contract,
+        fetch,
+        baseline_model=baseline,
+        allow_unpinned=allow,
+    )
+    write_replay(args.out, recorded)
+    title = contract.name or "contract"
+    resolved_models = list(dict.fromkeys(response.model for response in recorded.values()))
+    if len(resolved_models) == 1:
+        resolved = resolved_models[0]
+    elif resolved_models:
+        resolved = ", ".join(resolved_models)
+    else:
+        resolved = baseline
+    sys.stdout.write(
+        f"jevcheck record: {title}\n"
+        f"baseline: {format_resolved_model(baseline, resolved)}\n"
+        f"cases: {len(recorded)}\n"
+        f"wrote: {args.out}\n"
+    )
+    return EXIT_OK
+
+
+def _run_compare(args: argparse.Namespace) -> int:
+    allow = bool(args.allow_unpinned)
+    if args.from_model and args.baseline_model:
+        from_pin = require_pinned(args.from_model, allow_unpinned=allow)
+        base_pin = require_pinned(args.baseline_model, allow_unpinned=allow)
+        if from_pin != base_pin:
+            raise ValueError("--from-model and --baseline-model disagree")
+    baseline_override = args.from_model or args.baseline_model
+    contract = load_contract(
+        args.contract,
+        baseline_model=baseline_override,
+        allow_unpinned=allow,
+    )
+    candidate = require_pinned(args.to_model, allow_unpinned=allow)
+    if args.from_replay is not None:
+        baseline = load_replay(args.from_replay)
+    else:
+        live_baseline = require_pinned(contract.baseline_model, allow_unpinned=allow)
+        baseline_fetch = _answers_or_live_fetch(
+            contract,
+            None,
+            model=live_baseline,
+            allow=allow,
+            live_need="live compare",
+        )
+        baseline = record_answers(
+            contract,
+            baseline_fetch,
+            baseline_model=live_baseline,
+            allow_unpinned=allow,
+        )
+    fetch = _answers_or_live_fetch(
+        contract,
+        args.answers,
+        model=candidate,
+        allow=allow,
+        live_need="live compare",
+    )
+    report = compare(
+        contract,
+        baseline,
+        fetch,
+        candidate_model=candidate,
+        allow_unpinned=allow,
+    )
+    sys.stdout.write(report.summary())
+    return EXIT_BREAKING if report.breaking else EXIT_OK
+
+
+def _answers_or_live_fetch(
+    contract: Contract,
+    answers: Path | None,
+    *,
+    model: str,
+    allow: bool,
+    live_need: str,
+) -> Fetch:
+    if answers is not None:
+        replay = load_replay(answers)
+        missing = [case.id for case in contract.cases if case.id not in replay]
+        if missing:
+            raise KeyError(f"replay file missing cases: {missing}")
+
+        def fetch(case: Case) -> JevResponse:
+            return replay[case.id]
+
+        return fetch
+
+    client = JevClient(model=model, allow_unpinned=allow)
+
+    def fetch(case: Case) -> JevResponse:
+        return client.system_one(state=case.state, questions=case.questions, model=model)
+
+    if not client.api_key:
+        raise RuntimeError(f"{live_need} needs {AUTH_ENV} or a --answers replay file")
+    return fetch
+
+
+def redact_secrets(text: str, *secrets: str | None) -> str:
+    """Remove known credentials from user-visible error text."""
+    redacted = text
+    for secret in secrets:
+        if secret:
+            redacted = redacted.replace(secret, "***")
+    env_key = os.environ.get(AUTH_ENV)
+    if env_key:
+        redacted = redacted.replace(env_key, "***")
+    redacted = re.sub(r"(?i)(bearer\s+)\S+", r"\1***", redacted)
+    redacted = re.sub(
+        r"(?i)(authorization\s*[:=]\s*)(\S+)",
+        r"\1***",
+        redacted,
+    )
+    return redacted
+
+
+def _print_error(exc: object) -> None:
+    print(f"jevcheck: {redact_secrets(str(exc))}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())