jevkit-core 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jevkit_core/__init__.py +28 -0
- jevkit_core/answers.py +151 -0
- jevkit_core/canonical.py +57 -0
- jevkit_core/question.py +124 -0
- jevkit_core/record.py +164 -0
- jevkit_core/tokens.py +87 -0
- jevkit_core-0.2.0.dist-info/METADATA +46 -0
- jevkit_core-0.2.0.dist-info/RECORD +9 -0
- jevkit_core-0.2.0.dist-info/WHEEL +4 -0
jevkit_core/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Shared substrate for the jevkit packages.
|
|
2
|
+
|
|
3
|
+
Nothing here calls the jev API. This package holds the pieces every other
|
|
4
|
+
jevkit tool needs: canonical digests, the `.jevl` record format, a uniform
|
|
5
|
+
view of a question, and token budget estimation.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .answers import NOUL_THRESHOLD, Answer, parse_answer, parse_answers
|
|
9
|
+
from .canonical import canonical_json, digest, record_id, request_id
|
|
10
|
+
from .question import (Question, flatten_text, normalize_question,
|
|
11
|
+
normalize_questions)
|
|
12
|
+
from .record import (FORMAT_VERSION, Record, RecordFormatError, append_record,
|
|
13
|
+
load_cassette, read_records, write_records)
|
|
14
|
+
from .tokens import (STATE_BUDGET, TOTAL_BUDGET, BudgetReport, check_budget,
|
|
15
|
+
estimate_tokens)
|
|
16
|
+
|
|
17
|
+
__version__ = "0.2.0"
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"Answer", "parse_answer", "parse_answers", "NOUL_THRESHOLD",
|
|
21
|
+
"canonical_json", "digest", "record_id", "request_id",
|
|
22
|
+
"Question", "flatten_text", "normalize_question", "normalize_questions",
|
|
23
|
+
"Record", "RecordFormatError", "FORMAT_VERSION",
|
|
24
|
+
"read_records", "write_records", "append_record", "load_cassette",
|
|
25
|
+
"estimate_tokens", "check_budget", "BudgetReport",
|
|
26
|
+
"TOTAL_BUDGET", "STATE_BUDGET",
|
|
27
|
+
"__version__",
|
|
28
|
+
]
|
jevkit_core/answers.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""A uniform view of a jev answer, and how it compares to a ground-truth label.
|
|
2
|
+
|
|
3
|
+
`drift`, `calibrate`, and `bench` all need the same three things from an answer:
|
|
4
|
+
what it predicted, how much probability sat on a given outcome, and whether it
|
|
5
|
+
matched a label. Those live here so the three packages agree on the definitions
|
|
6
|
+
rather than each inventing its own.
|
|
7
|
+
|
|
8
|
+
The vocabulary is deliberately narrow:
|
|
9
|
+
|
|
10
|
+
- **predicted** is the outcome the answer selects. For a Choice it is the option
|
|
11
|
+
key. For a Noul it is ``True`` when ``noul`` clears the threshold. For a Score
|
|
12
|
+
it is the index of the most probable level, which is *not* the same as
|
|
13
|
+
rounding ``score``.
|
|
14
|
+
- **confidence** is what the API returned, and Nouls do not have one. A Noul's
|
|
15
|
+
distance from 0.5 is a different quantity and is exposed separately as
|
|
16
|
+
``decisiveness`` rather than pretending it is the same number.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
__all__ = ["Answer", "parse_answer", "parse_answers", "NOUL_THRESHOLD"]
|
|
25
|
+
|
|
26
|
+
NOUL_THRESHOLD = 0.5
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Answer:
|
|
31
|
+
"""Normalized view over one answer from the API."""
|
|
32
|
+
|
|
33
|
+
id: str
|
|
34
|
+
type: str
|
|
35
|
+
raw: dict[str, Any]
|
|
36
|
+
|
|
37
|
+
# -- what it said ------------------------------------------------------
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def probabilities(self) -> dict[str, float]:
|
|
41
|
+
"""Outcome -> probability.
|
|
42
|
+
|
|
43
|
+
Choice returns its options. Score returns its level indices as strings.
|
|
44
|
+
Noul has no distribution from the API, so the two-outcome distribution
|
|
45
|
+
it implies is synthesized here.
|
|
46
|
+
"""
|
|
47
|
+
if self.type == "noul":
|
|
48
|
+
p = float(self.raw.get("noul", 0.0))
|
|
49
|
+
return {"true": p, "false": 1.0 - p}
|
|
50
|
+
probs = self.raw.get("probabilities") or {}
|
|
51
|
+
return {str(k): float(v) for k, v in probs.items()}
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def confidence(self) -> float | None:
|
|
55
|
+
"""As returned by the API. ``None`` for a Noul, which carries none."""
|
|
56
|
+
value = self.raw.get("confidence")
|
|
57
|
+
return None if value is None else float(value)
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def decisiveness(self) -> float:
|
|
61
|
+
"""How far from maximally uncertain this answer is, on 0..1.
|
|
62
|
+
|
|
63
|
+
For a Noul this is ``|noul - 0.5| * 2``. This is *not* confidence and is
|
|
64
|
+
not comparable to the API's confidence across question types; it exists
|
|
65
|
+
so Nouls can be thresholded on something with a defined meaning.
|
|
66
|
+
"""
|
|
67
|
+
if self.type == "noul":
|
|
68
|
+
return abs(float(self.raw.get("noul", 0.0)) - NOUL_THRESHOLD) * 2.0
|
|
69
|
+
conf = self.confidence
|
|
70
|
+
return 0.0 if conf is None else conf
|
|
71
|
+
|
|
72
|
+
def predicted(self, *, noul_threshold: float = NOUL_THRESHOLD) -> Any:
|
|
73
|
+
"""The outcome this answer selects."""
|
|
74
|
+
if self.type == "noul":
|
|
75
|
+
return float(self.raw.get("noul", 0.0)) >= noul_threshold
|
|
76
|
+
if self.type == "choice":
|
|
77
|
+
return self.raw.get("choice")
|
|
78
|
+
if self.type == "score":
|
|
79
|
+
probs = self.probabilities
|
|
80
|
+
if not probs:
|
|
81
|
+
return None
|
|
82
|
+
best = max(probs, key=lambda k: probs[k])
|
|
83
|
+
try:
|
|
84
|
+
return int(best)
|
|
85
|
+
except ValueError:
|
|
86
|
+
return best
|
|
87
|
+
return None
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def score(self) -> float | None:
|
|
91
|
+
"""The probability-weighted score, for a Score answer."""
|
|
92
|
+
if self.type != "score":
|
|
93
|
+
return None
|
|
94
|
+
value = self.raw.get("score")
|
|
95
|
+
return None if value is None else float(value)
|
|
96
|
+
|
|
97
|
+
# -- how it compares to a label ---------------------------------------
|
|
98
|
+
|
|
99
|
+
def _label_key(self, label: Any) -> str:
|
|
100
|
+
if self.type == "noul":
|
|
101
|
+
return "true" if bool(label) else "false"
|
|
102
|
+
return str(label)
|
|
103
|
+
|
|
104
|
+
def probability_of(self, label: Any) -> float:
|
|
105
|
+
"""Probability this answer assigned to ``label``.
|
|
106
|
+
|
|
107
|
+
Returns 0.0 for an outcome the question never offered, which is the
|
|
108
|
+
honest reading: the model could not have selected it.
|
|
109
|
+
"""
|
|
110
|
+
return self.probabilities.get(self._label_key(label), 0.0)
|
|
111
|
+
|
|
112
|
+
def is_correct(self, label: Any, *, noul_threshold: float = NOUL_THRESHOLD) -> bool:
|
|
113
|
+
predicted = self.predicted(noul_threshold=noul_threshold)
|
|
114
|
+
if self.type == "noul":
|
|
115
|
+
return bool(predicted) is bool(label)
|
|
116
|
+
if self.type == "score":
|
|
117
|
+
try:
|
|
118
|
+
return int(predicted) == int(label) # type: ignore[arg-type]
|
|
119
|
+
except (TypeError, ValueError):
|
|
120
|
+
return predicted == label
|
|
121
|
+
return predicted == label
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def top_probability(self) -> float:
|
|
125
|
+
"""Probability mass on the predicted outcome."""
|
|
126
|
+
probs = self.probabilities
|
|
127
|
+
return max(probs.values()) if probs else 0.0
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def parse_answer(qid: str, raw: Any) -> Answer:
|
|
131
|
+
if not isinstance(raw, dict):
|
|
132
|
+
raise ValueError(f"answer {qid!r}: expected an object, got {type(raw).__name__}")
|
|
133
|
+
atype = raw.get("type")
|
|
134
|
+
if atype not in ("choice", "score", "noul"):
|
|
135
|
+
# Infer from shape when the API response omits the discriminator.
|
|
136
|
+
if "choice" in raw:
|
|
137
|
+
atype = "choice"
|
|
138
|
+
elif "noul" in raw:
|
|
139
|
+
atype = "noul"
|
|
140
|
+
elif "score" in raw:
|
|
141
|
+
atype = "score"
|
|
142
|
+
else:
|
|
143
|
+
raise ValueError(
|
|
144
|
+
f"answer {qid!r}: cannot determine type; expected a 'type' field or one "
|
|
145
|
+
f"of 'choice'/'score'/'noul'"
|
|
146
|
+
)
|
|
147
|
+
return Answer(id=qid, type=atype, raw=raw)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def parse_answers(answers: dict[str, Any]) -> dict[str, Answer]:
|
|
151
|
+
return {qid: parse_answer(qid, raw) for qid, raw in answers.items()}
|
jevkit_core/canonical.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""RFC 8785 JSON Canonicalization, and the digests jevkit builds on it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
__all__ = ["canonical_json", "digest", "request_id", "record_id"]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _check_finite(value: Any) -> None:
|
|
14
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
15
|
+
raise ValueError(f"cannot canonicalize non-finite number: {value!r}")
|
|
16
|
+
if isinstance(value, dict):
|
|
17
|
+
for v in value.values():
|
|
18
|
+
_check_finite(v)
|
|
19
|
+
elif isinstance(value, (list, tuple)):
|
|
20
|
+
for v in value:
|
|
21
|
+
_check_finite(v)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def canonical_json(value: Any) -> bytes:
|
|
25
|
+
"""Serialize to RFC 8785 canonical form.
|
|
26
|
+
|
|
27
|
+
Object keys sorted by code point, no insignificant whitespace, UTF-8.
|
|
28
|
+
NaN and Infinity are rejected rather than emitted as invalid JSON.
|
|
29
|
+
"""
|
|
30
|
+
_check_finite(value)
|
|
31
|
+
return json.dumps(
|
|
32
|
+
value,
|
|
33
|
+
sort_keys=True,
|
|
34
|
+
separators=(",", ":"),
|
|
35
|
+
ensure_ascii=False,
|
|
36
|
+
allow_nan=False,
|
|
37
|
+
).encode("utf-8")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def digest(value: Any) -> str:
|
|
41
|
+
"""`sha256:<hex>` over the canonical form of ``value``."""
|
|
42
|
+
return "sha256:" + hashlib.sha256(canonical_json(value)).hexdigest()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def request_id(state: Any, questions: dict[str, Any]) -> str:
|
|
46
|
+
"""Digest of state + questions, with no model.
|
|
47
|
+
|
|
48
|
+
This is what pairs a golden-set record with its replay on a different
|
|
49
|
+
model version, since the full record id includes the model and would
|
|
50
|
+
therefore differ.
|
|
51
|
+
"""
|
|
52
|
+
return digest({"questions": questions, "state": state})
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def record_id(model: str, state: Any, questions: dict[str, Any]) -> str:
|
|
56
|
+
"""Digest of model + state + questions. The record's ``id`` field."""
|
|
57
|
+
return digest({"model": model, "questions": questions, "state": state})
|
jevkit_core/question.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""A uniform view of a jev question.
|
|
2
|
+
|
|
3
|
+
Questions reach jevkit in three shapes: plain dicts as sent to the HTTP API,
|
|
4
|
+
SDK objects (``Choice``/``Score``/``Noul``), and whatever a user's own helper
|
|
5
|
+
produces. Every jevkit tool wants the same few facts out of them, so they are
|
|
6
|
+
normalized once here rather than re-sniffed in each package.
|
|
7
|
+
|
|
8
|
+
``instructions`` and ``criteria`` both accept JSON structure, not just
|
|
9
|
+
strings, so the normalized form keeps the raw value and exposes the flattened
|
|
10
|
+
text separately for anything doing textual analysis.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Any, Iterator
|
|
17
|
+
|
|
18
|
+
__all__ = ["Question", "QuestionType", "normalize_question", "normalize_questions",
|
|
19
|
+
"flatten_text"]
|
|
20
|
+
|
|
21
|
+
QuestionType = str # "choice" | "score" | "noul"
|
|
22
|
+
|
|
23
|
+
_VALID_TYPES = ("choice", "score", "noul")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def flatten_text(value: Any) -> str:
|
|
27
|
+
"""Collapse a string / dict / list into one string for textual analysis.
|
|
28
|
+
|
|
29
|
+
Dict keys are included: in a Choice, the keys are the option names and
|
|
30
|
+
carry meaning the model sees.
|
|
31
|
+
"""
|
|
32
|
+
if value is None:
|
|
33
|
+
return ""
|
|
34
|
+
if isinstance(value, str):
|
|
35
|
+
return value
|
|
36
|
+
if isinstance(value, (int, float, bool)):
|
|
37
|
+
return str(value)
|
|
38
|
+
if isinstance(value, dict):
|
|
39
|
+
parts: list[str] = []
|
|
40
|
+
for k, v in value.items():
|
|
41
|
+
parts.append(str(k))
|
|
42
|
+
parts.append(flatten_text(v))
|
|
43
|
+
return " ".join(p for p in parts if p)
|
|
44
|
+
if isinstance(value, (list, tuple)):
|
|
45
|
+
return " ".join(p for p in (flatten_text(v) for v in value) if p)
|
|
46
|
+
return str(value)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class Question:
|
|
51
|
+
"""Normalized question. ``raw`` is always the untouched original."""
|
|
52
|
+
|
|
53
|
+
id: str
|
|
54
|
+
type: QuestionType
|
|
55
|
+
instructions: Any
|
|
56
|
+
criteria: Any
|
|
57
|
+
raw: Any = field(repr=False, default=None)
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def instructions_text(self) -> str:
|
|
61
|
+
return flatten_text(self.instructions)
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def criteria_text(self) -> str:
|
|
65
|
+
return flatten_text(self.criteria)
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def text(self) -> str:
|
|
69
|
+
"""Everything the model reads, as one string."""
|
|
70
|
+
return f"{self.instructions_text} {self.criteria_text}".strip()
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def options(self) -> list[str]:
|
|
74
|
+
"""Choice option keys, or Score level descriptions. Empty otherwise."""
|
|
75
|
+
if self.type == "choice" and isinstance(self.criteria, dict):
|
|
76
|
+
return [str(k) for k in self.criteria.keys()]
|
|
77
|
+
if self.type == "score" and isinstance(self.criteria, (list, tuple)):
|
|
78
|
+
return [flatten_text(v) for v in self.criteria]
|
|
79
|
+
if self.type == "score" and isinstance(self.criteria, dict):
|
|
80
|
+
return [flatten_text(v) for v in self.criteria.values()]
|
|
81
|
+
return []
|
|
82
|
+
|
|
83
|
+
def option_descriptions(self) -> Iterator[tuple[str, str]]:
|
|
84
|
+
"""(option key, description) pairs for a Choice."""
|
|
85
|
+
if self.type == "choice" and isinstance(self.criteria, dict):
|
|
86
|
+
for k, v in self.criteria.items():
|
|
87
|
+
yield str(k), flatten_text(v)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _get(obj: Any, name: str) -> Any:
|
|
91
|
+
if isinstance(obj, dict):
|
|
92
|
+
return obj.get(name)
|
|
93
|
+
return getattr(obj, name, None)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _infer_type(obj: Any) -> str | None:
|
|
97
|
+
declared = _get(obj, "type")
|
|
98
|
+
if isinstance(declared, str) and declared.lower() in _VALID_TYPES:
|
|
99
|
+
return declared.lower()
|
|
100
|
+
# SDK objects carry no "type" field; fall back to the class name.
|
|
101
|
+
cls = type(obj).__name__.lower()
|
|
102
|
+
if cls in _VALID_TYPES:
|
|
103
|
+
return cls
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def normalize_question(qid: str, obj: Any) -> Question:
|
|
108
|
+
qtype = _infer_type(obj)
|
|
109
|
+
if qtype is None:
|
|
110
|
+
raise ValueError(
|
|
111
|
+
f"question {qid!r}: cannot determine type. Expected a 'type' key of "
|
|
112
|
+
f"{_VALID_TYPES}, or a Choice/Score/Noul object."
|
|
113
|
+
)
|
|
114
|
+
return Question(
|
|
115
|
+
id=qid,
|
|
116
|
+
type=qtype,
|
|
117
|
+
instructions=_get(obj, "instructions"),
|
|
118
|
+
criteria=_get(obj, "criteria"),
|
|
119
|
+
raw=obj,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def normalize_questions(questions: dict[str, Any]) -> list[Question]:
|
|
124
|
+
return [normalize_question(qid, obj) for qid, obj in questions.items()]
|
jevkit_core/record.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Reading and writing `.jevl` records. See docs/specs/record-format-v1.md."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
|
+
from typing import Any, IO, Iterable, Iterator
|
|
10
|
+
|
|
11
|
+
from .canonical import record_id, request_id
|
|
12
|
+
|
|
13
|
+
__all__ = ["Record", "FORMAT_VERSION", "RecordFormatError",
|
|
14
|
+
"read_records", "write_records", "append_record", "load_cassette"]
|
|
15
|
+
|
|
16
|
+
FORMAT_VERSION = 1
|
|
17
|
+
|
|
18
|
+
_REQUIRED = ("v", "id", "ts", "model", "request", "answers")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class RecordFormatError(ValueError):
|
|
22
|
+
"""A `.jevl` line is malformed, or its version is unreadable."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _utcnow() -> str:
|
|
26
|
+
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Record:
|
|
31
|
+
model: str
|
|
32
|
+
state: Any
|
|
33
|
+
questions: dict[str, Any]
|
|
34
|
+
answers: dict[str, Any]
|
|
35
|
+
id: str = ""
|
|
36
|
+
ts: str = field(default_factory=_utcnow)
|
|
37
|
+
usage: dict[str, Any] | None = None
|
|
38
|
+
label: dict[str, Any] | None = None
|
|
39
|
+
tags: list[str] = field(default_factory=list)
|
|
40
|
+
meta: dict[str, Any] = field(default_factory=dict)
|
|
41
|
+
extra: dict[str, Any] = field(default_factory=dict, repr=False)
|
|
42
|
+
|
|
43
|
+
def __post_init__(self) -> None:
|
|
44
|
+
if not self.id:
|
|
45
|
+
self.id = record_id(self.model, self.state, self.questions)
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def request_id(self) -> str:
|
|
49
|
+
"""Model-independent digest, used to pair across model versions."""
|
|
50
|
+
return request_id(self.state, self.questions)
|
|
51
|
+
|
|
52
|
+
def to_dict(self) -> dict[str, Any]:
|
|
53
|
+
out: dict[str, Any] = {
|
|
54
|
+
"v": FORMAT_VERSION,
|
|
55
|
+
"id": self.id,
|
|
56
|
+
"ts": self.ts,
|
|
57
|
+
"model": self.model,
|
|
58
|
+
"request": {"state": self.state, "questions": self.questions},
|
|
59
|
+
"answers": self.answers,
|
|
60
|
+
}
|
|
61
|
+
if self.usage is not None:
|
|
62
|
+
out["usage"] = self.usage
|
|
63
|
+
if self.label is not None:
|
|
64
|
+
out["label"] = self.label
|
|
65
|
+
if self.tags:
|
|
66
|
+
out["tags"] = list(self.tags)
|
|
67
|
+
if self.meta:
|
|
68
|
+
out["meta"] = dict(self.meta)
|
|
69
|
+
# Unknown keys survive a read/write round trip.
|
|
70
|
+
for k, v in self.extra.items():
|
|
71
|
+
out.setdefault(k, v)
|
|
72
|
+
return out
|
|
73
|
+
|
|
74
|
+
@classmethod
|
|
75
|
+
def from_dict(cls, raw: dict[str, Any], *, source: str = "<memory>",
|
|
76
|
+
line_no: int = 0) -> "Record":
|
|
77
|
+
where = f"{source}:{line_no}" if line_no else source
|
|
78
|
+
if not isinstance(raw, dict):
|
|
79
|
+
raise RecordFormatError(f"{where}: record must be a JSON object")
|
|
80
|
+
|
|
81
|
+
version = raw.get("v")
|
|
82
|
+
if version is None:
|
|
83
|
+
raise RecordFormatError(f"{where}: missing required key 'v'")
|
|
84
|
+
if not isinstance(version, int):
|
|
85
|
+
raise RecordFormatError(f"{where}: 'v' must be an integer, got {version!r}")
|
|
86
|
+
if version > FORMAT_VERSION:
|
|
87
|
+
raise RecordFormatError(
|
|
88
|
+
f"{where}: record format v{version} is newer than this reader "
|
|
89
|
+
f"(v{FORMAT_VERSION}). Upgrade jevkit-core rather than guessing."
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
missing = [k for k in _REQUIRED if k not in raw]
|
|
93
|
+
if missing:
|
|
94
|
+
raise RecordFormatError(f"{where}: missing required key(s): {', '.join(missing)}")
|
|
95
|
+
|
|
96
|
+
request = raw["request"]
|
|
97
|
+
if not isinstance(request, dict) or "questions" not in request:
|
|
98
|
+
raise RecordFormatError(f"{where}: 'request' must be an object with 'questions'")
|
|
99
|
+
|
|
100
|
+
known = set(_REQUIRED) | {"usage", "label", "tags", "meta"}
|
|
101
|
+
return cls(
|
|
102
|
+
model=raw["model"],
|
|
103
|
+
state=request.get("state"),
|
|
104
|
+
questions=request["questions"],
|
|
105
|
+
answers=raw["answers"],
|
|
106
|
+
id=raw["id"],
|
|
107
|
+
ts=raw["ts"],
|
|
108
|
+
usage=raw.get("usage"),
|
|
109
|
+
label=raw.get("label"),
|
|
110
|
+
tags=list(raw.get("tags") or []),
|
|
111
|
+
meta=dict(raw.get("meta") or {}),
|
|
112
|
+
extra={k: v for k, v in raw.items() if k not in known},
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def read_records(path: str | os.PathLike[str] | IO[str]) -> Iterator[Record]:
|
|
117
|
+
"""Yield records from a `.jevl` file.
|
|
118
|
+
|
|
119
|
+
A line that fails to parse raises. Skipping silently would let a truncated
|
|
120
|
+
golden set look like a passing one.
|
|
121
|
+
"""
|
|
122
|
+
if hasattr(path, "read"):
|
|
123
|
+
yield from _read_stream(path, getattr(path, "name", "<stream>")) # type: ignore[arg-type]
|
|
124
|
+
return
|
|
125
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
126
|
+
yield from _read_stream(fh, str(path))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _read_stream(fh: IO[str], source: str) -> Iterator[Record]:
|
|
130
|
+
for line_no, line in enumerate(fh, start=1):
|
|
131
|
+
stripped = line.strip()
|
|
132
|
+
if not stripped:
|
|
133
|
+
continue
|
|
134
|
+
try:
|
|
135
|
+
raw = json.loads(stripped)
|
|
136
|
+
except json.JSONDecodeError as exc:
|
|
137
|
+
raise RecordFormatError(f"{source}:{line_no}: invalid JSON: {exc}") from exc
|
|
138
|
+
yield Record.from_dict(raw, source=source, line_no=line_no)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def write_records(path: str | os.PathLike[str], records: Iterable[Record]) -> int:
|
|
142
|
+
count = 0
|
|
143
|
+
with open(path, "w", encoding="utf-8") as fh:
|
|
144
|
+
for record in records:
|
|
145
|
+
fh.write(json.dumps(record.to_dict(), ensure_ascii=False) + "\n")
|
|
146
|
+
count += 1
|
|
147
|
+
return count
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def append_record(path: str | os.PathLike[str], record: Record) -> None:
|
|
151
|
+
with open(path, "a", encoding="utf-8") as fh:
|
|
152
|
+
fh.write(json.dumps(record.to_dict(), ensure_ascii=False) + "\n")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def load_cassette(path: str | os.PathLike[str]) -> dict[str, Record]:
|
|
156
|
+
"""Map record id -> Record, last occurrence winning.
|
|
157
|
+
|
|
158
|
+
Re-recording a request appends rather than rewrites, so the last line for
|
|
159
|
+
an id is the current answer.
|
|
160
|
+
"""
|
|
161
|
+
table: dict[str, Record] = {}
|
|
162
|
+
for record in read_records(path):
|
|
163
|
+
table[record.id] = record
|
|
164
|
+
return table
|
jevkit_core/tokens.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Token budget estimation for jev requests.
|
|
2
|
+
|
|
3
|
+
Jev ingests the state once and evaluates every question against it, so two
|
|
4
|
+
separate budgets apply. Both are documented on the model card:
|
|
5
|
+
|
|
6
|
+
64k state + every question combined
|
|
7
|
+
32k state + the single longest question
|
|
8
|
+
|
|
9
|
+
These are *estimates*. jevkit deliberately does not bundle a tokenizer: the
|
|
10
|
+
dependency is heavy, the vendor does not publish which tokenizer Jev uses,
|
|
11
|
+
and a wrong tokenizer is more misleading than an honest approximation. The
|
|
12
|
+
estimator is intentionally conservative so that a request jevkit calls safe
|
|
13
|
+
is very unlikely to be rejected.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from .canonical import canonical_json
|
|
21
|
+
|
|
22
|
+
__all__ = ["estimate_tokens", "BudgetReport", "check_budget",
|
|
23
|
+
"TOTAL_BUDGET", "STATE_BUDGET"]
|
|
24
|
+
|
|
25
|
+
TOTAL_BUDGET = 64_000
|
|
26
|
+
STATE_BUDGET = 32_000
|
|
27
|
+
|
|
28
|
+
# Bytes per token. Real-world English on BPE-family tokenizers runs ~4.0;
|
|
29
|
+
# 3.5 buys headroom for punctuation-dense JSON without being absurd.
|
|
30
|
+
_BYTES_PER_TOKEN = 3.5
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def estimate_tokens(value: Any) -> int:
|
|
34
|
+
"""Conservative token estimate for any JSON-serializable value."""
|
|
35
|
+
if isinstance(value, str):
|
|
36
|
+
payload = value.encode("utf-8")
|
|
37
|
+
else:
|
|
38
|
+
payload = canonical_json(value)
|
|
39
|
+
return max(1, int(len(payload) / _BYTES_PER_TOKEN) + 1)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class BudgetReport:
|
|
43
|
+
"""Where a request sits against both documented budgets."""
|
|
44
|
+
|
|
45
|
+
__slots__ = ("state_tokens", "question_tokens", "longest_question_id")
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
state_tokens: int,
|
|
50
|
+
question_tokens: dict[str, int],
|
|
51
|
+
longest_question_id: str | None,
|
|
52
|
+
) -> None:
|
|
53
|
+
self.state_tokens = state_tokens
|
|
54
|
+
self.question_tokens = question_tokens
|
|
55
|
+
self.longest_question_id = longest_question_id
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def total(self) -> int:
|
|
59
|
+
return self.state_tokens + sum(self.question_tokens.values())
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def longest_pair(self) -> int:
|
|
63
|
+
"""state + the single longest question."""
|
|
64
|
+
if self.longest_question_id is None:
|
|
65
|
+
return self.state_tokens
|
|
66
|
+
return self.state_tokens + self.question_tokens[self.longest_question_id]
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def over_total(self) -> bool:
|
|
70
|
+
return self.total > TOTAL_BUDGET
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def over_state(self) -> bool:
|
|
74
|
+
return self.longest_pair > STATE_BUDGET
|
|
75
|
+
|
|
76
|
+
def __repr__(self) -> str: # pragma: no cover - debug aid
|
|
77
|
+
return (
|
|
78
|
+
f"BudgetReport(total={self.total}, longest_pair={self.longest_pair}, "
|
|
79
|
+
f"over_total={self.over_total}, over_state={self.over_state})"
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def check_budget(state: Any, questions: dict[str, Any]) -> BudgetReport:
|
|
84
|
+
state_tokens = estimate_tokens(state)
|
|
85
|
+
per_question = {qid: estimate_tokens(q) for qid, q in questions.items()}
|
|
86
|
+
longest = max(per_question, key=lambda k: per_question[k]) if per_question else None
|
|
87
|
+
return BudgetReport(state_tokens, per_question, longest)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: jevkit-core
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Shared substrate for jevkit: the .jevl record format, canonical request digests, question normalization, and token budgets for TypeSafe's Jev.
|
|
5
|
+
Project-URL: Homepage, https://github.com/pjdurden/jevkit-py
|
|
6
|
+
Project-URL: Issues, https://github.com/pjdurden/jevkit-py/issues
|
|
7
|
+
Author: Prajjwal Chittori
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Keywords: calibration,jev,llm,system-one,typesafe
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# jevkit-core
|
|
21
|
+
|
|
22
|
+
Shared substrate for [jevkit](https://github.com/pjdurden/jevkit-py), a set of
|
|
23
|
+
tools for building on TypeSafe's Jev. Nothing in this package calls the API.
|
|
24
|
+
|
|
25
|
+
- **`.jevl` record format** — one JSON Lines format that serves as test
|
|
26
|
+
cassette, drift golden set, benchmark instance, and calibration observation.
|
|
27
|
+
See [the spec](../../docs/specs/record-format-v1.md).
|
|
28
|
+
- **Canonical digests** — RFC 8785 canonicalization, so the same request always
|
|
29
|
+
hashes the same way. `record_id()` includes the model; `request_id()` does
|
|
30
|
+
not, which is what lets you pair a golden record with its replay on a new
|
|
31
|
+
model version.
|
|
32
|
+
- **Question normalization** — one view over SDK objects and plain dicts alike.
|
|
33
|
+
- **Token budgets** — estimates against Jev's two documented limits: 64k for
|
|
34
|
+
state plus all questions, 32k for state plus the longest single question.
|
|
35
|
+
|
|
36
|
+
> Unofficial and unaffiliated with TypeSafe.
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install jevkit-core
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## License
|
|
45
|
+
|
|
46
|
+
MIT
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
jevkit_core/__init__.py,sha256=DQ3rcTlVvaUUPBhmmtOEqf6qyayAKvNMF64Jaq6XkUU,1234
|
|
2
|
+
jevkit_core/answers.py,sha256=chhwo6SjDi9Y0AWZXUf6nIVJyJVHRbA8sXMVuTQ0Sb4,5622
|
|
3
|
+
jevkit_core/canonical.py,sha256=R27fXBIAhjF_5AsqL6ZT-hps_I2Hf7EnjEZBjjgOJMY,1794
|
|
4
|
+
jevkit_core/question.py,sha256=Xv4LHE8OXdJkhb2dKLIodaEZ3f3Mhsz8YTg_Nr7JRHQ,4133
|
|
5
|
+
jevkit_core/record.py,sha256=1vp7nwA-7QnlGVukPEzGtcDCBIjSyPobCBxzVX4B6k8,5759
|
|
6
|
+
jevkit_core/tokens.py,sha256=KW7HHA-OmHUOq0gsHd-H3Pdz-OD3jUrGVKVA_rAd9T0,2942
|
|
7
|
+
jevkit_core-0.2.0.dist-info/METADATA,sha256=8oI9PqymqfEEDt6eZ2x3iImvEokybCgXrLm8jtfOjl0,1826
|
|
8
|
+
jevkit_core-0.2.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
|
|
9
|
+
jevkit_core-0.2.0.dist-info/RECORD,,
|