jevify 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jevify/__init__.py +7 -0
- jevify/__main__.py +9 -0
- jevify/adapters/__init__.py +1 -0
- jevify/adapters/embedding/__init__.py +1 -0
- jevify/adapters/embedding/adapter.py +203 -0
- jevify/adapters/embedding/local.py +82 -0
- jevify/adapters/encoder/__init__.py +1 -0
- jevify/adapters/encoder/adapter.py +165 -0
- jevify/adapters/encoder/local.py +77 -0
- jevify/adapters/endpoint/__init__.py +1 -0
- jevify/adapters/endpoint/adapter.py +281 -0
- jevify/adapters/endpoint/client.py +114 -0
- jevify/adapters/endpoint/dialects.py +166 -0
- jevify/adapters/endpoint/probe.py +589 -0
- jevify/adapters/endpoint/readout.py +244 -0
- jevify/adapters/rerank/__init__.py +1 -0
- jevify/adapters/rerank/adapter.py +152 -0
- jevify/api/__init__.py +1 -0
- jevify/api/app.py +228 -0
- jevify/api/schema.py +123 -0
- jevify/api/translate.py +239 -0
- jevify/cli.py +392 -0
- jevify/compose.py +82 -0
- jevify/domain/__init__.py +1 -0
- jevify/domain/distribution.py +103 -0
- jevify/domain/engine.py +152 -0
- jevify/domain/questions.py +121 -0
- jevify/evaluation/__init__.py +1 -0
- jevify/evaluation/calibrate.py +186 -0
- jevify/evaluation/controls.py +78 -0
- jevify/evaluation/metrics.py +91 -0
- jevify/evaluation/scorecard.py +385 -0
- jevify/evaluation/suites.py +166 -0
- jevify/ports/__init__.py +1 -0
- jevify/ports/backend.py +162 -0
- jevify/recipes/__init__.py +21 -0
- jevify/recipes/probe_case.yaml +72 -0
- jevify/recipes/render.py +198 -0
- jevify/recipes/schema.py +204 -0
- jevify/recipes/store.py +81 -0
- jevify-0.1.0.dist-info/METADATA +243 -0
- jevify-0.1.0.dist-info/RECORD +45 -0
- jevify-0.1.0.dist-info/WHEEL +4 -0
- jevify-0.1.0.dist-info/entry_points.txt +2 -0
- jevify-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
"""The readout ladder (ADR-0002 D4): how a distribution over the answer set is obtained.
|
|
2
|
+
|
|
3
|
+
Each rung knows whether it is available under a capability list, how to
|
|
4
|
+
shape the request, and how to read the answer tokens out of the response.
|
|
5
|
+
The token match is id-first, text second (ADR-0002 D2, plan decision 4).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from jevify.adapters.endpoint.dialects import Entry, Parsed, grammar_for, neutral_samplers
|
|
15
|
+
from jevify.ports.backend import BackendError, Capabilities, Dialect, Rung
|
|
16
|
+
from jevify.recipes.render import RenderedPart
|
|
17
|
+
from jevify.recipes.schema import Token
|
|
18
|
+
|
|
19
|
+
_PIECE_MARKS = {"Ġ": " ", "Ċ": "\n", "▁": " "}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def normalize_piece(text: str) -> str:
|
|
23
|
+
for mark, plain in _PIECE_MARKS.items():
|
|
24
|
+
text = text.replace(mark, plain)
|
|
25
|
+
return text
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def matches(entry: Entry, token: Token) -> bool:
|
|
29
|
+
if entry.id is not None and token.id is not None:
|
|
30
|
+
return entry.id == token.id
|
|
31
|
+
return normalize_piece(entry.text) == normalize_piece(token.text)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class Readout:
|
|
36
|
+
logprobs: dict[str, float]
|
|
37
|
+
off_menu_mass: float | None
|
|
38
|
+
missing: tuple[str, ...]
|
|
39
|
+
degraded: bool
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def read_answer_set(parsed: Parsed, question: RenderedPart, *, full_softmax: bool) -> Readout:
|
|
43
|
+
"""Collect each label's mass from the entries; a label's tokens are distinct events."""
|
|
44
|
+
per_label: dict[str, float] = {}
|
|
45
|
+
answer_mass = 0.0
|
|
46
|
+
missing: list[str] = []
|
|
47
|
+
for label in question.labels:
|
|
48
|
+
found = [
|
|
49
|
+
entry.logprob
|
|
50
|
+
for entry in parsed.entries
|
|
51
|
+
if any(matches(entry, token) for token in question.tokens[label])
|
|
52
|
+
]
|
|
53
|
+
if not found:
|
|
54
|
+
missing.append(label)
|
|
55
|
+
continue
|
|
56
|
+
peak = max(found)
|
|
57
|
+
logsum = peak + math.log(sum(math.exp(lp - peak) for lp in found))
|
|
58
|
+
per_label[label] = logsum
|
|
59
|
+
answer_mass += math.exp(logsum)
|
|
60
|
+
off_menu = max(0.0, 1.0 - answer_mass) if full_softmax and not missing else None
|
|
61
|
+
return Readout(per_label, off_menu, tuple(missing), degraded=bool(missing))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class MissingLabelsError(BackendError):
|
|
65
|
+
"""A rung that read the answer slot but not every label. On the auto ladder the next
|
|
66
|
+
proven rung takes the question; a pinned rung lets it surface."""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class TopKRung:
|
|
70
|
+
"""Rung 3: ask for the top-k list unbiased; exact when every answer token is present."""
|
|
71
|
+
|
|
72
|
+
rung = Rung.TOP_K
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def available(capabilities: Capabilities | None) -> bool:
|
|
76
|
+
return True # every OpenAI-compatible server lists top logprobs
|
|
77
|
+
|
|
78
|
+
@staticmethod
|
|
79
|
+
def shape(body: dict[str, Any], question: RenderedPart, top_k: int, dialect: Dialect) -> None:
|
|
80
|
+
if "messages" in body:
|
|
81
|
+
body["top_logprobs"] = top_k
|
|
82
|
+
else:
|
|
83
|
+
body["logprobs"] = top_k
|
|
84
|
+
|
|
85
|
+
@staticmethod
|
|
86
|
+
def read(parsed: Parsed, question: RenderedPart) -> Readout:
|
|
87
|
+
readout = read_answer_set(parsed, question, full_softmax=True)
|
|
88
|
+
if readout.missing:
|
|
89
|
+
raise MissingLabelsError(
|
|
90
|
+
f"labels {list(readout.missing)} not in the top-{len(parsed.entries)} logprobs; "
|
|
91
|
+
"raise readout.top_k or pin readout.rung to top_k_floor"
|
|
92
|
+
)
|
|
93
|
+
return readout
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class NamedTokenLogprobsRung:
|
|
97
|
+
"""Rung 1: ask vLLM for exactly the answer token ids; exact, off-menu mass visible."""
|
|
98
|
+
|
|
99
|
+
rung = Rung.NAMED_TOKEN_LOGPROBS
|
|
100
|
+
|
|
101
|
+
@staticmethod
|
|
102
|
+
def available(capabilities: Capabilities | None) -> bool:
|
|
103
|
+
return bool(capabilities and Rung.NAMED_TOKEN_LOGPROBS in capabilities.rungs)
|
|
104
|
+
|
|
105
|
+
@staticmethod
|
|
106
|
+
def shape(body: dict[str, Any], part: RenderedPart, top_k: int, dialect: Dialect) -> None:
|
|
107
|
+
ids = sorted({t.id for lbl in part.labels for t in part.tokens[lbl] if t.id is not None})
|
|
108
|
+
if not ids:
|
|
109
|
+
raise BackendError("named-token readout needs verified token ids; run the probe")
|
|
110
|
+
body["logprobs"] = True
|
|
111
|
+
body["top_logprobs"] = 1
|
|
112
|
+
body["logprob_token_ids"] = ids
|
|
113
|
+
body["return_as_token_id"] = True
|
|
114
|
+
|
|
115
|
+
@staticmethod
|
|
116
|
+
def read(parsed: Parsed, part: RenderedPart) -> Readout:
|
|
117
|
+
readout = read_answer_set(parsed, part, full_softmax=True)
|
|
118
|
+
if readout.missing:
|
|
119
|
+
raise BackendError(
|
|
120
|
+
f"labels {list(readout.missing)} missing from the named-token logprobs; "
|
|
121
|
+
"the backend may have ignored logprob_token_ids (re-run the probe)"
|
|
122
|
+
)
|
|
123
|
+
return readout
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _answer_texts(part: RenderedPart) -> list[str]:
|
|
127
|
+
return list(dict.fromkeys(t.text for lbl in part.labels for t in part.tokens[lbl]))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _answer_ids(part: RenderedPart) -> list[int]:
|
|
131
|
+
return sorted({t.id for lbl in part.labels for t in part.tokens[lbl] if t.id is not None})
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class GrammarRung:
|
|
135
|
+
"""Rung 2: a grammar masks the vocabulary to the answer spellings and the server reports
|
|
136
|
+
post-sampling probabilities renormalized over the survivors (llama.cpp)."""
|
|
137
|
+
|
|
138
|
+
rung = Rung.GRAMMAR
|
|
139
|
+
|
|
140
|
+
@staticmethod
|
|
141
|
+
def available(capabilities: Capabilities | None) -> bool:
|
|
142
|
+
return bool(capabilities and Rung.GRAMMAR in capabilities.rungs)
|
|
143
|
+
|
|
144
|
+
@staticmethod
|
|
145
|
+
def shape(body: dict[str, Any], part: RenderedPart, top_k: int, dialect: Dialect) -> None:
|
|
146
|
+
texts = _answer_texts(part)
|
|
147
|
+
body["grammar"] = grammar_for(texts)
|
|
148
|
+
neutral_samplers(body)
|
|
149
|
+
n = max(top_k, len(texts) + 4)
|
|
150
|
+
body["n_probs" if "n_predict" in body else "logprobs"] = n
|
|
151
|
+
if "messages" in body:
|
|
152
|
+
body["top_logprobs"] = n
|
|
153
|
+
|
|
154
|
+
FLOOR_LOGPROB = -40.0 # below anything a float32 softmax reports before underflow
|
|
155
|
+
|
|
156
|
+
@staticmethod
|
|
157
|
+
def read(parsed: Parsed, part: RenderedPart) -> Readout:
|
|
158
|
+
readout = read_answer_set(parsed, part, full_softmax=False)
|
|
159
|
+
if not readout.missing:
|
|
160
|
+
return readout
|
|
161
|
+
if not readout.logprobs:
|
|
162
|
+
raise BackendError(
|
|
163
|
+
"no answer label returned under the grammar; "
|
|
164
|
+
"the server may not honour grammar with post-sampling probabilities"
|
|
165
|
+
)
|
|
166
|
+
# Under a mask the reported list holds every answer with non-zero mass, so a label
|
|
167
|
+
# that is absent underflowed to zero: a certain answer, floored, not a degraded one.
|
|
168
|
+
logprobs = dict(readout.logprobs)
|
|
169
|
+
for label in readout.missing:
|
|
170
|
+
logprobs[label] = GrammarRung.FLOOR_LOGPROB
|
|
171
|
+
return Readout(logprobs, None, readout.missing, degraded=False)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class EqualBiasRung:
|
|
175
|
+
"""Rung 4: the same bias on every answer token lifts them into the reported list without
|
|
176
|
+
changing their ratios (softmax is shift-invariant within the biased set)."""
|
|
177
|
+
|
|
178
|
+
rung = Rung.EQUAL_BIAS
|
|
179
|
+
BIAS = 50
|
|
180
|
+
|
|
181
|
+
@staticmethod
|
|
182
|
+
def available(capabilities: Capabilities | None) -> bool:
|
|
183
|
+
return bool(capabilities and Rung.EQUAL_BIAS in capabilities.rungs)
|
|
184
|
+
|
|
185
|
+
@staticmethod
|
|
186
|
+
def shape(body: dict[str, Any], part: RenderedPart, top_k: int, dialect: Dialect) -> None:
|
|
187
|
+
ids = _answer_ids(part)
|
|
188
|
+
keys = [str(i) for i in ids] if ids else _answer_texts(part)
|
|
189
|
+
body["logit_bias"] = {k: EqualBiasRung.BIAS for k in keys}
|
|
190
|
+
if dialect is Dialect.LLAMACPP:
|
|
191
|
+
neutral_samplers(body)
|
|
192
|
+
n = max(top_k, len(keys) + 4)
|
|
193
|
+
if "messages" in body:
|
|
194
|
+
body["top_logprobs"] = n
|
|
195
|
+
else:
|
|
196
|
+
body["n_probs" if "n_predict" in body else "logprobs"] = n
|
|
197
|
+
|
|
198
|
+
@staticmethod
|
|
199
|
+
def read(parsed: Parsed, part: RenderedPart) -> Readout:
|
|
200
|
+
readout = read_answer_set(parsed, part, full_softmax=False)
|
|
201
|
+
if readout.missing:
|
|
202
|
+
raise BackendError(
|
|
203
|
+
f"labels {list(readout.missing)} absent after equal bias; "
|
|
204
|
+
"the server may report logprobs before bias is applied"
|
|
205
|
+
)
|
|
206
|
+
return readout
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class TopKFloorRung:
|
|
210
|
+
"""Rung 5: top-k, but a label that fell out of the list gets the observed floor minus a
|
|
211
|
+
margin, and the answer is marked degraded with the missing labels named."""
|
|
212
|
+
|
|
213
|
+
rung = Rung.TOP_K_FLOOR
|
|
214
|
+
MARGIN = 1.0
|
|
215
|
+
|
|
216
|
+
@staticmethod
|
|
217
|
+
def available(capabilities: Capabilities | None) -> bool:
|
|
218
|
+
return True
|
|
219
|
+
|
|
220
|
+
@staticmethod
|
|
221
|
+
def shape(body: dict[str, Any], part: RenderedPart, top_k: int, dialect: Dialect) -> None:
|
|
222
|
+
TopKRung.shape(body, part, top_k, dialect)
|
|
223
|
+
|
|
224
|
+
@staticmethod
|
|
225
|
+
def read(parsed: Parsed, part: RenderedPart) -> Readout:
|
|
226
|
+
readout = read_answer_set(parsed, part, full_softmax=True)
|
|
227
|
+
if not readout.missing:
|
|
228
|
+
return readout
|
|
229
|
+
if not parsed.entries:
|
|
230
|
+
raise BackendError("no logprobs returned; nothing to floor against")
|
|
231
|
+
floor = min(e.logprob for e in parsed.entries) - TopKFloorRung.MARGIN
|
|
232
|
+
logprobs = dict(readout.logprobs)
|
|
233
|
+
for label in readout.missing:
|
|
234
|
+
logprobs[label] = floor
|
|
235
|
+
return Readout(logprobs, None, readout.missing, degraded=True)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
RUNGS: dict[Rung, Any] = {
|
|
239
|
+
Rung.TOP_K_FLOOR: TopKFloorRung,
|
|
240
|
+
Rung.TOP_K: TopKRung,
|
|
241
|
+
Rung.NAMED_TOKEN_LOGPROBS: NamedTokenLogprobsRung,
|
|
242
|
+
Rung.GRAMMAR: GrammarRung,
|
|
243
|
+
Rung.EQUAL_BIAS: EqualBiasRung,
|
|
244
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The rerank kind: one independent relevance score per option from a rerank endpoint."""
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""RerankBackend: the backend port over a rerank endpoint (ADR-0002 D2, rerank kind).
|
|
2
|
+
|
|
3
|
+
Each option is scored as a document against the state as the query. Scores are
|
|
4
|
+
independent relevance judgments, so the answer is labeled `relevance`; the
|
|
5
|
+
engine's softmax over them is a ranking convenience, never a probability.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import datetime as dt
|
|
11
|
+
import hashlib
|
|
12
|
+
import math
|
|
13
|
+
import time
|
|
14
|
+
from collections.abc import Callable, Sequence
|
|
15
|
+
|
|
16
|
+
from jevify.adapters.endpoint.client import OpenAICompatibleClient
|
|
17
|
+
from jevify.domain.questions import ChoiceQuestion, NoulQuestion, Question, State
|
|
18
|
+
from jevify.ports.backend import (
|
|
19
|
+
BackendError,
|
|
20
|
+
Capabilities,
|
|
21
|
+
Dialect,
|
|
22
|
+
RawAnswer,
|
|
23
|
+
Rung,
|
|
24
|
+
StateHandle,
|
|
25
|
+
)
|
|
26
|
+
from jevify.recipes.render import RenderedPrefix
|
|
27
|
+
from jevify.recipes.schema import Recipe
|
|
28
|
+
|
|
29
|
+
Clock = Callable[[], float]
|
|
30
|
+
_FLOOR = 1e-9
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _fill(template: str, **fields: str) -> str:
|
|
34
|
+
for name, value in fields.items():
|
|
35
|
+
template = template.replace("{" + name + "}", value)
|
|
36
|
+
return template
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class RerankBackend:
|
|
40
|
+
def __init__(
|
|
41
|
+
self, client: OpenAICompatibleClient, recipe: Recipe, *, clock: Clock = time.perf_counter
|
|
42
|
+
) -> None:
|
|
43
|
+
if recipe.rerank is None or recipe.endpoint is None:
|
|
44
|
+
raise BackendError("a rerank backend needs rerank and endpoint sections")
|
|
45
|
+
self.client = client
|
|
46
|
+
self.recipe = recipe
|
|
47
|
+
self.spec = recipe.rerank
|
|
48
|
+
self.clock = clock
|
|
49
|
+
|
|
50
|
+
async def probe(self) -> Capabilities:
|
|
51
|
+
query = _fill(self.spec.query, state="probe")
|
|
52
|
+
try:
|
|
53
|
+
response = await self.client.post_v1(
|
|
54
|
+
self.spec.path,
|
|
55
|
+
{"model": self.recipe.model.name, "query": query, "documents": ["probe"]},
|
|
56
|
+
)
|
|
57
|
+
scores = self._scores(response, 1)
|
|
58
|
+
except BackendError as exc:
|
|
59
|
+
raise BackendError(f"rerank endpoint did not answer the probe: {exc}") from exc
|
|
60
|
+
notes = () if scores else ("rerank endpoint returned no scores for the probe",)
|
|
61
|
+
return Capabilities(
|
|
62
|
+
kind="rerank",
|
|
63
|
+
model=self.recipe.model.name,
|
|
64
|
+
dialect=Dialect(self.recipe.endpoint.dialect)
|
|
65
|
+
if self.recipe.endpoint and self.recipe.endpoint.dialect != "auto"
|
|
66
|
+
else None,
|
|
67
|
+
modalities=frozenset({"text"}),
|
|
68
|
+
rungs=frozenset(),
|
|
69
|
+
token_ids_verified=False,
|
|
70
|
+
probed_at=dt.datetime.now(dt.UTC).isoformat(timespec="seconds"),
|
|
71
|
+
notes=notes,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
async def warm(self, state: State) -> StateHandle:
|
|
75
|
+
if state.modalities - {"text"}:
|
|
76
|
+
raise BackendError("the rerank kind scores text only; the state carries image parts")
|
|
77
|
+
query = _fill(self.spec.query, state=state.text)
|
|
78
|
+
prefix = RenderedPrefix(mode="rerank", text=query)
|
|
79
|
+
state_id = hashlib.sha256(query.encode("utf-8")).hexdigest()
|
|
80
|
+
return StateHandle(state_id=state_id, prefix=prefix)
|
|
81
|
+
|
|
82
|
+
async def evaluate(self, handle: StateHandle, questions: Sequence[Question]) -> list[RawAnswer]:
|
|
83
|
+
return [await self._answer(handle, q) for q in questions]
|
|
84
|
+
|
|
85
|
+
def _documents(self, question: Question) -> tuple[list[str], list[str]]:
|
|
86
|
+
if isinstance(question, ChoiceQuestion):
|
|
87
|
+
labels = list(question.labels)
|
|
88
|
+
documents = [
|
|
89
|
+
_fill(
|
|
90
|
+
self.spec.document,
|
|
91
|
+
key=option.key,
|
|
92
|
+
description=(self.spec.description_sep + option.description)
|
|
93
|
+
if option.description
|
|
94
|
+
else "",
|
|
95
|
+
text=option.description or option.key,
|
|
96
|
+
)
|
|
97
|
+
for option in question.options
|
|
98
|
+
]
|
|
99
|
+
return labels, documents
|
|
100
|
+
if isinstance(question, NoulQuestion):
|
|
101
|
+
return ["true"], [question.instructions or ""]
|
|
102
|
+
raise BackendError(
|
|
103
|
+
"the rerank kind answers choice and noul questions only; a score question needs "
|
|
104
|
+
"a readout-capable recipe"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def _scores(self, response: dict, count: int) -> dict[int, float]:
|
|
108
|
+
results = response.get("results") or response.get("data") or []
|
|
109
|
+
scores: dict[int, float] = {}
|
|
110
|
+
for item in results:
|
|
111
|
+
index = item.get("index")
|
|
112
|
+
value = item.get(self.spec.score_field, item.get("score"))
|
|
113
|
+
if index is not None and value is not None:
|
|
114
|
+
scores[int(index)] = float(value)
|
|
115
|
+
return scores
|
|
116
|
+
|
|
117
|
+
async def _answer(self, handle: StateHandle, question: Question) -> RawAnswer:
|
|
118
|
+
labels, documents = self._documents(question)
|
|
119
|
+
body = {
|
|
120
|
+
"model": self.recipe.model.name,
|
|
121
|
+
"query": handle.prefix.text,
|
|
122
|
+
"documents": documents,
|
|
123
|
+
"top_n": len(documents),
|
|
124
|
+
}
|
|
125
|
+
if self.spec.instruction:
|
|
126
|
+
body["instruction"] = self.spec.instruction
|
|
127
|
+
started = self.clock()
|
|
128
|
+
response = await self.client.post_v1(self.spec.path, body)
|
|
129
|
+
latency_ms = (self.clock() - started) * 1000.0
|
|
130
|
+
scores = self._scores(response, len(documents))
|
|
131
|
+
if len(scores) != len(documents):
|
|
132
|
+
raise BackendError(
|
|
133
|
+
f"rerank endpoint scored {len(scores)} of {len(documents)} documents"
|
|
134
|
+
)
|
|
135
|
+
by_label = {label: scores[i] for i, label in enumerate(labels)}
|
|
136
|
+
if isinstance(question, NoulQuestion):
|
|
137
|
+
s = min(1.0, max(0.0, by_label["true"]))
|
|
138
|
+
logprobs = {"false": math.log(max(1.0 - s, _FLOOR)), "true": math.log(max(s, _FLOOR))}
|
|
139
|
+
else:
|
|
140
|
+
logprobs = {label: math.log(max(score, _FLOOR)) for label, score in by_label.items()}
|
|
141
|
+
usage = response.get("usage") or {}
|
|
142
|
+
return RawAnswer(
|
|
143
|
+
question_id=question.id,
|
|
144
|
+
logprobs=logprobs,
|
|
145
|
+
semantics="relevance",
|
|
146
|
+
rung=Rung.NATIVE,
|
|
147
|
+
latency_ms=latency_ms,
|
|
148
|
+
prompt_tokens=usage.get("prompt_tokens"),
|
|
149
|
+
calls=1,
|
|
150
|
+
composition="per_option",
|
|
151
|
+
raw={"scores": by_label, "path": self.spec.path},
|
|
152
|
+
)
|
jevify/api/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Jev's interface (ADR-0003): wire schema, translation to and from the domain, the server."""
|
jevify/api/app.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
"""The HTTP server: Jev's evaluate call, unchanged, over the library (ADR-0003 D1, D7).
|
|
2
|
+
|
|
3
|
+
Thin by design. Validation is explicit pydantic; errors use the FastAPI-style
|
|
4
|
+
`{"detail": [...]}` body the SDK understands; every response carries an
|
|
5
|
+
`x-typesafe-request-id` header because the SDK expects one.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import time
|
|
12
|
+
import uuid
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from pydantic import ValidationError
|
|
17
|
+
from starlette.applications import Starlette
|
|
18
|
+
from starlette.requests import Request
|
|
19
|
+
from starlette.responses import JSONResponse
|
|
20
|
+
from starlette.routing import Route
|
|
21
|
+
|
|
22
|
+
from jevify import __version__
|
|
23
|
+
from jevify.api.schema import HTTPValidationError, SystemOneRequest, SystemOneResponse
|
|
24
|
+
from jevify.api.translate import (
|
|
25
|
+
classify_results,
|
|
26
|
+
classify_to_systemone,
|
|
27
|
+
evaluation_to_jev,
|
|
28
|
+
jev_extension,
|
|
29
|
+
jev_request_to_domain,
|
|
30
|
+
)
|
|
31
|
+
from jevify.domain.engine import Engine
|
|
32
|
+
from jevify.ports.backend import BackendError, ContextLimitError
|
|
33
|
+
from jevify.recipes.schema import Recipe
|
|
34
|
+
from jevify.recipes.store import RecipeError, recipe_hash
|
|
35
|
+
|
|
36
|
+
REQUEST_ID_HEADER = "x-typesafe-request-id"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _validation_error(exc: ValidationError) -> JSONResponse:
|
|
40
|
+
detail = [
|
|
41
|
+
{"loc": list(item["loc"]), "msg": item["msg"], "type": item["type"]}
|
|
42
|
+
for item in exc.errors()
|
|
43
|
+
]
|
|
44
|
+
body = HTTPValidationError(detail=detail).model_dump(mode="json", exclude_none=True)
|
|
45
|
+
return JSONResponse(body, status_code=422)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _error(status: int, message: str) -> JSONResponse:
|
|
49
|
+
return JSONResponse({"error": {"message": message, "type": "jevify_error"}}, status_code=status)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class _HeldState:
|
|
54
|
+
handle: Any
|
|
55
|
+
expires_at: float
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class StateStore:
|
|
59
|
+
"""In-process handles for the optional ingest route (ADR-0003 D4): one process, a TTL,
|
|
60
|
+
and a loud 404 for an expired or unknown handle."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, ttl_s: float, clock=time.monotonic) -> None:
|
|
63
|
+
self.ttl_s = ttl_s
|
|
64
|
+
self.clock = clock
|
|
65
|
+
self._held: dict[str, _HeldState] = {}
|
|
66
|
+
|
|
67
|
+
def put(self, handle: Any) -> str:
|
|
68
|
+
self._sweep()
|
|
69
|
+
self._held[handle.state_id] = _HeldState(handle, self.clock() + self.ttl_s)
|
|
70
|
+
return handle.state_id
|
|
71
|
+
|
|
72
|
+
def get(self, state_id: str) -> Any | None:
|
|
73
|
+
self._sweep()
|
|
74
|
+
held = self._held.get(state_id)
|
|
75
|
+
return held.handle if held else None
|
|
76
|
+
|
|
77
|
+
def _sweep(self) -> None:
|
|
78
|
+
now = self.clock()
|
|
79
|
+
for key in [k for k, v in self._held.items() if v.expires_at <= now]:
|
|
80
|
+
del self._held[key]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def create_app(
|
|
84
|
+
engine: Engine, recipe: Recipe, *, api_key: str | None = None, state_ttl_s: float = 600.0
|
|
85
|
+
) -> Starlette:
|
|
86
|
+
digest = recipe_hash(recipe)
|
|
87
|
+
model_name = recipe.model.name
|
|
88
|
+
states = StateStore(state_ttl_s)
|
|
89
|
+
|
|
90
|
+
def authorized(request: Request) -> bool:
|
|
91
|
+
if api_key is None:
|
|
92
|
+
return True
|
|
93
|
+
header = request.headers.get("authorization", "")
|
|
94
|
+
return header == f"Bearer {api_key}"
|
|
95
|
+
|
|
96
|
+
async def systemone(request: Request) -> JSONResponse:
|
|
97
|
+
if not authorized(request):
|
|
98
|
+
return _error(401, "invalid api key")
|
|
99
|
+
try:
|
|
100
|
+
payload = await request.json()
|
|
101
|
+
except ValueError:
|
|
102
|
+
return _error(400, "body is not valid JSON")
|
|
103
|
+
return await evaluate_payload(payload)
|
|
104
|
+
|
|
105
|
+
async def evaluate_payload(payload: Any) -> JSONResponse:
|
|
106
|
+
"""The one evaluation path: systemone and classify both end here."""
|
|
107
|
+
try:
|
|
108
|
+
wire = SystemOneRequest.model_validate(payload)
|
|
109
|
+
except ValidationError as exc:
|
|
110
|
+
return _validation_error(exc)
|
|
111
|
+
try:
|
|
112
|
+
state, questions = jev_request_to_domain(wire)
|
|
113
|
+
except ValueError as exc:
|
|
114
|
+
return _error(422, str(exc))
|
|
115
|
+
try:
|
|
116
|
+
state_ref = (wire.x_jevify or {}).get("state_ref")
|
|
117
|
+
if state_ref:
|
|
118
|
+
handle = states.get(str(state_ref))
|
|
119
|
+
if handle is None:
|
|
120
|
+
return _error(404, f"state handle {state_ref!r} is expired or unknown")
|
|
121
|
+
evaluation = await engine.evaluate(handle, questions)
|
|
122
|
+
else:
|
|
123
|
+
evaluation = await engine.ask(state, questions)
|
|
124
|
+
except ContextLimitError as exc:
|
|
125
|
+
return _error(413, str(exc))
|
|
126
|
+
except (BackendError, RecipeError) as exc:
|
|
127
|
+
return _error(502, str(exc))
|
|
128
|
+
body: dict[str, Any] = evaluation_to_jev(evaluation, model=model_name, recipe_hash=digest)
|
|
129
|
+
body["x_jevify"] = jev_extension(
|
|
130
|
+
evaluation, {"recipe_hash": digest, "requested_model": wire.model}
|
|
131
|
+
)
|
|
132
|
+
SystemOneResponse.model_validate(body) # never emit a shape the SDK would reject
|
|
133
|
+
return JSONResponse(body, headers={REQUEST_ID_HEADER: f"req_{uuid.uuid4().hex}"})
|
|
134
|
+
|
|
135
|
+
async def create_state(request: Request) -> JSONResponse:
|
|
136
|
+
if not authorized(request):
|
|
137
|
+
return _error(401, "invalid api key")
|
|
138
|
+
try:
|
|
139
|
+
payload = await request.json()
|
|
140
|
+
except ValueError:
|
|
141
|
+
return _error(400, "body is not valid JSON")
|
|
142
|
+
if not isinstance(payload, dict) or payload.get("state") is None:
|
|
143
|
+
return _error(422, "a state is required")
|
|
144
|
+
try:
|
|
145
|
+
wire = SystemOneRequest.model_validate(
|
|
146
|
+
{
|
|
147
|
+
"state": payload["state"],
|
|
148
|
+
"model": model_name,
|
|
149
|
+
"questions": {"_": {"type": "noul"}},
|
|
150
|
+
}
|
|
151
|
+
)
|
|
152
|
+
state, _ = jev_request_to_domain(wire)
|
|
153
|
+
handle = await engine.warm(state)
|
|
154
|
+
except ValidationError as exc:
|
|
155
|
+
return _validation_error(exc)
|
|
156
|
+
except ValueError as exc:
|
|
157
|
+
return _error(422, str(exc))
|
|
158
|
+
except ContextLimitError as exc:
|
|
159
|
+
return _error(413, str(exc))
|
|
160
|
+
except (BackendError, RecipeError) as exc:
|
|
161
|
+
return _error(502, str(exc))
|
|
162
|
+
state_id = states.put(handle)
|
|
163
|
+
return JSONResponse(
|
|
164
|
+
{
|
|
165
|
+
"state_id": state_id,
|
|
166
|
+
"expires_in_s": state_ttl_s,
|
|
167
|
+
"warm_ms": handle.warm_ms,
|
|
168
|
+
"cached_tokens": handle.cached_tokens,
|
|
169
|
+
},
|
|
170
|
+
status_code=201,
|
|
171
|
+
headers={REQUEST_ID_HEADER: f"req_{uuid.uuid4().hex}"},
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
async def classify(request: Request) -> JSONResponse:
|
|
175
|
+
if not authorized(request):
|
|
176
|
+
return _error(401, "invalid api key")
|
|
177
|
+
try:
|
|
178
|
+
payload = await request.json()
|
|
179
|
+
except ValueError:
|
|
180
|
+
return _error(400, "body is not valid JSON")
|
|
181
|
+
try:
|
|
182
|
+
jev_request = classify_to_systemone(payload)
|
|
183
|
+
except ValueError as exc:
|
|
184
|
+
return _error(422, str(exc))
|
|
185
|
+
response = await evaluate_payload(jev_request)
|
|
186
|
+
if response.status_code != 200:
|
|
187
|
+
return response
|
|
188
|
+
body = json.loads(bytes(response.body))
|
|
189
|
+
mode = payload.get("mode", "boolean")
|
|
190
|
+
categories = (
|
|
191
|
+
list(jev_request["questions"])
|
|
192
|
+
if mode != "choice"
|
|
193
|
+
else list(jev_request["questions"]["category"]["criteria"])
|
|
194
|
+
)
|
|
195
|
+
body["results"] = classify_results(mode, categories, body["answers"])
|
|
196
|
+
body["mode"] = mode
|
|
197
|
+
return JSONResponse(
|
|
198
|
+
body, headers={REQUEST_ID_HEADER: response.headers.get(REQUEST_ID_HEADER, "")}
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
async def models(request: Request) -> JSONResponse:
|
|
202
|
+
if not authorized(request):
|
|
203
|
+
return _error(401, "invalid api key")
|
|
204
|
+
return JSONResponse(
|
|
205
|
+
{
|
|
206
|
+
"models": [
|
|
207
|
+
{
|
|
208
|
+
"name": model_name,
|
|
209
|
+
"description": f"jevify {__version__} recipe {digest[:12]}",
|
|
210
|
+
"release_date": recipe.provenance.created,
|
|
211
|
+
}
|
|
212
|
+
]
|
|
213
|
+
},
|
|
214
|
+
headers={REQUEST_ID_HEADER: f"req_{uuid.uuid4().hex}"},
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
async def health(request: Request) -> JSONResponse:
|
|
218
|
+
return JSONResponse({"status": "ok", "model": model_name, "recipe_hash": digest})
|
|
219
|
+
|
|
220
|
+
return Starlette(
|
|
221
|
+
routes=[
|
|
222
|
+
Route("/v1/systemone", systemone, methods=["POST"]),
|
|
223
|
+
Route("/v1/states", create_state, methods=["POST"]),
|
|
224
|
+
Route("/v1/classify", classify, methods=["POST"]),
|
|
225
|
+
Route("/v1/models", models, methods=["GET"]),
|
|
226
|
+
Route("/health", health, methods=["GET"]),
|
|
227
|
+
]
|
|
228
|
+
)
|