jevify 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jevify/__init__.py +7 -0
- jevify/__main__.py +9 -0
- jevify/adapters/__init__.py +1 -0
- jevify/adapters/embedding/__init__.py +1 -0
- jevify/adapters/embedding/adapter.py +203 -0
- jevify/adapters/embedding/local.py +82 -0
- jevify/adapters/encoder/__init__.py +1 -0
- jevify/adapters/encoder/adapter.py +165 -0
- jevify/adapters/encoder/local.py +77 -0
- jevify/adapters/endpoint/__init__.py +1 -0
- jevify/adapters/endpoint/adapter.py +281 -0
- jevify/adapters/endpoint/client.py +114 -0
- jevify/adapters/endpoint/dialects.py +166 -0
- jevify/adapters/endpoint/probe.py +589 -0
- jevify/adapters/endpoint/readout.py +244 -0
- jevify/adapters/rerank/__init__.py +1 -0
- jevify/adapters/rerank/adapter.py +152 -0
- jevify/api/__init__.py +1 -0
- jevify/api/app.py +228 -0
- jevify/api/schema.py +123 -0
- jevify/api/translate.py +239 -0
- jevify/cli.py +392 -0
- jevify/compose.py +82 -0
- jevify/domain/__init__.py +1 -0
- jevify/domain/distribution.py +103 -0
- jevify/domain/engine.py +152 -0
- jevify/domain/questions.py +121 -0
- jevify/evaluation/__init__.py +1 -0
- jevify/evaluation/calibrate.py +186 -0
- jevify/evaluation/controls.py +78 -0
- jevify/evaluation/metrics.py +91 -0
- jevify/evaluation/scorecard.py +385 -0
- jevify/evaluation/suites.py +166 -0
- jevify/ports/__init__.py +1 -0
- jevify/ports/backend.py +162 -0
- jevify/recipes/__init__.py +21 -0
- jevify/recipes/probe_case.yaml +72 -0
- jevify/recipes/render.py +198 -0
- jevify/recipes/schema.py +204 -0
- jevify/recipes/store.py +81 -0
- jevify-0.1.0.dist-info/METADATA +243 -0
- jevify-0.1.0.dist-info/RECORD +45 -0
- jevify-0.1.0.dist-info/WHEEL +4 -0
- jevify-0.1.0.dist-info/entry_points.txt +2 -0
- jevify-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,589 @@
|
|
|
1
|
+
"""The capability probe (ADR-0002 D3): a measurement, not a lookup table.
|
|
2
|
+
|
|
3
|
+
Every check sends real requests to the backend and records what came back.
|
|
4
|
+
The result is written into the recipe by `jevify probe`, and the adapter
|
|
5
|
+
picks its readout rung from it.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import datetime as dt
|
|
12
|
+
import time
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import yaml
|
|
18
|
+
|
|
19
|
+
from jevify.adapters.endpoint.client import OpenAICompatibleClient
|
|
20
|
+
from jevify.adapters.endpoint.dialects import (
|
|
21
|
+
base_request,
|
|
22
|
+
grammar_for,
|
|
23
|
+
neutral_samplers,
|
|
24
|
+
parse_response,
|
|
25
|
+
)
|
|
26
|
+
from jevify.adapters.endpoint.readout import matches, read_answer_set
|
|
27
|
+
from jevify.domain.questions import NoulQuestion, State
|
|
28
|
+
from jevify.ports.backend import (
|
|
29
|
+
BackendError,
|
|
30
|
+
CacheEvidence,
|
|
31
|
+
Capabilities,
|
|
32
|
+
Dialect,
|
|
33
|
+
FanoutEvidence,
|
|
34
|
+
Rung,
|
|
35
|
+
)
|
|
36
|
+
from jevify.recipes.render import RenderedPart, render_prefix, render_question
|
|
37
|
+
from jevify.recipes.schema import Recipe, Token
|
|
38
|
+
|
|
39
|
+
Clock = Callable[[], float]
|
|
40
|
+
|
|
41
|
+
_PROBE_CASE = Path(__file__).resolve().parents[2] / "recipes" / "probe_case.yaml"
|
|
42
|
+
_BLOCK_CANDIDATES = (1, 16, 32, 64, 128, 256, 512, 1024, 2048)
|
|
43
|
+
_MAX_REEVALUATED_TAIL = 8 # llama.cpp re-evaluates a few trailing tokens; still token-granular
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _one_pixel_png() -> str:
|
|
47
|
+
"""A 1x1 white PNG as a data URI, built here so no binary blob lives in the tree."""
|
|
48
|
+
import base64
|
|
49
|
+
import struct
|
|
50
|
+
import zlib
|
|
51
|
+
|
|
52
|
+
def chunk(tag: bytes, data: bytes) -> bytes:
|
|
53
|
+
return (
|
|
54
|
+
struct.pack(">I", len(data))
|
|
55
|
+
+ tag
|
|
56
|
+
+ data
|
|
57
|
+
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
png = (
|
|
61
|
+
b"\x89PNG\r\n\x1a\n"
|
|
62
|
+
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
|
|
63
|
+
+ chunk(b"IDAT", zlib.compress(b"\x00\xff\xff\xff", 9))
|
|
64
|
+
+ chunk(b"IEND", b"")
|
|
65
|
+
)
|
|
66
|
+
return "data:image/png;base64," + base64.b64encode(png).decode("ascii")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def load_probe_case() -> dict[str, Any]:
|
|
70
|
+
return yaml.safe_load(_PROBE_CASE.read_text(encoding="utf-8"))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
_CONTEXT_MARGIN = 160 # room for the template, the question and the answer slot
|
|
74
|
+
_CHARS_PER_TOKEN = 3.0 # conservative estimate when the backend cannot tokenize
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class Prober:
|
|
78
|
+
def __init__(
|
|
79
|
+
self, client: OpenAICompatibleClient, recipe: Recipe, clock: Clock = time.perf_counter
|
|
80
|
+
):
|
|
81
|
+
self.client = client
|
|
82
|
+
self.recipe = recipe
|
|
83
|
+
self.clock = clock
|
|
84
|
+
self.notes: list[str] = []
|
|
85
|
+
self.image_marker: str | None = None
|
|
86
|
+
|
|
87
|
+
async def run(self) -> Capabilities:
|
|
88
|
+
assert self.recipe.endpoint is not None
|
|
89
|
+
case = load_probe_case()
|
|
90
|
+
dialect, version = await self._dialect()
|
|
91
|
+
max_context = await self._max_context(dialect)
|
|
92
|
+
answer_tokens, multi, verified = await self._verify_tokens(dialect)
|
|
93
|
+
kwargs_honored = await self._thinking_off(dialect)
|
|
94
|
+
|
|
95
|
+
state = State.from_jev(await self._fit_state(case["state"], max_context, dialect))
|
|
96
|
+
prefix = render_prefix(self.recipe, state)
|
|
97
|
+
question = render_question(
|
|
98
|
+
self.recipe, NoulQuestion(id="probe", instructions=case["statement"])
|
|
99
|
+
)
|
|
100
|
+
part = question.parts[0]
|
|
101
|
+
|
|
102
|
+
parsed, entries_requested = await self._readout(
|
|
103
|
+
prefix, part, dialect, self.recipe.readout.top_k
|
|
104
|
+
)
|
|
105
|
+
top_k_cap = len(parsed.entries) if parsed.entries else 0
|
|
106
|
+
if top_k_cap == 0:
|
|
107
|
+
raise BackendError("no readout is possible on this backend: it returned no logprobs")
|
|
108
|
+
readout = read_answer_set(parsed, part, full_softmax=True)
|
|
109
|
+
if readout.missing:
|
|
110
|
+
self.notes.append(
|
|
111
|
+
f"probe case: labels {list(readout.missing)} outside the top-{top_k_cap}"
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
rungs = {Rung.TOP_K, Rung.TOP_K_FLOOR}
|
|
115
|
+
if await self._named_token_logprobs(prefix, part, dialect, answer_tokens):
|
|
116
|
+
rungs.add(Rung.NAMED_TOKEN_LOGPROBS)
|
|
117
|
+
post_bias = await self._bias_visible(prefix, part, dialect, parsed.entries, answer_tokens)
|
|
118
|
+
if post_bias:
|
|
119
|
+
rungs.add(Rung.EQUAL_BIAS)
|
|
120
|
+
if dialect is Dialect.LLAMACPP and await self._grammar_honoured(prefix, part, dialect):
|
|
121
|
+
rungs.add(Rung.GRAMMAR)
|
|
122
|
+
prefill = await self._prefill_honored(dialect) if prefix.mode == "messages" else None
|
|
123
|
+
cache = await self._cache(state, dialect)
|
|
124
|
+
slots = await self._slots(dialect)
|
|
125
|
+
fanout = await self._fanout(prefix, part, dialect, slots)
|
|
126
|
+
modalities = {"text"}
|
|
127
|
+
if await self._accepts_image(dialect):
|
|
128
|
+
modalities.add("image")
|
|
129
|
+
|
|
130
|
+
return Capabilities(
|
|
131
|
+
kind="endpoint",
|
|
132
|
+
model=self.recipe.model.name,
|
|
133
|
+
dialect=dialect,
|
|
134
|
+
backend_version=version,
|
|
135
|
+
max_context=max_context,
|
|
136
|
+
modalities=frozenset(modalities),
|
|
137
|
+
rungs=frozenset(rungs),
|
|
138
|
+
top_k_cap=top_k_cap if top_k_cap < entries_requested else None,
|
|
139
|
+
logprobs_post_bias=post_bias,
|
|
140
|
+
prefill_honored=prefill,
|
|
141
|
+
template_kwargs_honored=kwargs_honored,
|
|
142
|
+
image_marker=self.image_marker,
|
|
143
|
+
token_ids_verified=verified,
|
|
144
|
+
answer_tokens=answer_tokens,
|
|
145
|
+
multi_token_answers=tuple(multi),
|
|
146
|
+
cache=cache,
|
|
147
|
+
fanout=fanout,
|
|
148
|
+
slots=slots,
|
|
149
|
+
probed_at=dt.datetime.now(dt.UTC).isoformat(timespec="seconds"),
|
|
150
|
+
notes=tuple(self.notes),
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
# ------------------------------------------------------------------ identity
|
|
154
|
+
async def _dialect(self) -> tuple[Dialect, str | None]:
|
|
155
|
+
assert self.recipe.endpoint is not None
|
|
156
|
+
pinned = self.recipe.endpoint.dialect
|
|
157
|
+
for candidate, path, key in (
|
|
158
|
+
(Dialect.VLLM, "version", "version"),
|
|
159
|
+
(Dialect.LLAMACPP, "props", "build_info"),
|
|
160
|
+
):
|
|
161
|
+
if pinned not in ("auto", str(candidate)):
|
|
162
|
+
continue
|
|
163
|
+
try:
|
|
164
|
+
payload = await self.client.get_root(path)
|
|
165
|
+
except BackendError:
|
|
166
|
+
continue
|
|
167
|
+
if isinstance(payload, dict) and key in payload:
|
|
168
|
+
return candidate, str(payload[key])
|
|
169
|
+
if pinned not in ("auto", "generic"):
|
|
170
|
+
self.notes.append(f"dialect {pinned} pinned but its identity endpoint did not answer")
|
|
171
|
+
return Dialect(pinned), None
|
|
172
|
+
return Dialect.GENERIC, None
|
|
173
|
+
|
|
174
|
+
async def _max_context(self, dialect: Dialect) -> int | None:
|
|
175
|
+
"""The context the server actually serves: llama.cpp's per-slot n_ctx outranks the
|
|
176
|
+
model's trained context, which /v1/models reports as meta.n_ctx_train."""
|
|
177
|
+
if dialect is Dialect.LLAMACPP:
|
|
178
|
+
try:
|
|
179
|
+
props = await self.client.get_root("props")
|
|
180
|
+
n_ctx = (props.get("default_generation_settings") or {}).get("n_ctx")
|
|
181
|
+
if n_ctx:
|
|
182
|
+
return int(n_ctx)
|
|
183
|
+
except BackendError:
|
|
184
|
+
pass
|
|
185
|
+
try:
|
|
186
|
+
models = await self.client.get_v1("models")
|
|
187
|
+
except BackendError:
|
|
188
|
+
models = {}
|
|
189
|
+
for entry in models.get("data") or []:
|
|
190
|
+
if entry.get("max_model_len"):
|
|
191
|
+
return int(entry["max_model_len"])
|
|
192
|
+
meta = entry.get("meta") or {}
|
|
193
|
+
if meta.get("n_ctx_train"):
|
|
194
|
+
self.notes.append(
|
|
195
|
+
"max_context is the trained context; the served context is unknown"
|
|
196
|
+
)
|
|
197
|
+
return int(meta["n_ctx_train"])
|
|
198
|
+
return None
|
|
199
|
+
|
|
200
|
+
# ------------------------------------------------------------------ tokens
|
|
201
|
+
async def _tokenize(self, text: str, dialect: Dialect) -> list[int] | None:
|
|
202
|
+
try:
|
|
203
|
+
if dialect is Dialect.LLAMACPP:
|
|
204
|
+
payload = await self.client.post_root(
|
|
205
|
+
"tokenize", {"content": text, "add_special": False, "parse_special": False}
|
|
206
|
+
)
|
|
207
|
+
tokens = payload.get("tokens") or []
|
|
208
|
+
return [t["id"] if isinstance(t, dict) else int(t) for t in tokens]
|
|
209
|
+
payload = await self.client.post_root(
|
|
210
|
+
"tokenize",
|
|
211
|
+
{"model": self.recipe.model.name, "prompt": text, "add_special_tokens": False},
|
|
212
|
+
)
|
|
213
|
+
return [int(t) for t in payload.get("tokens") or []]
|
|
214
|
+
except BackendError:
|
|
215
|
+
return None
|
|
216
|
+
|
|
217
|
+
async def _verify_tokens(self, dialect: Dialect) -> tuple[dict[str, int], list[str], bool]:
|
|
218
|
+
groups: list[list[Token]] = [
|
|
219
|
+
*self.recipe.answers.noul.values(),
|
|
220
|
+
*self.recipe.answers.identifiers,
|
|
221
|
+
]
|
|
222
|
+
texts = list(dict.fromkeys(token.text for group in groups for token in group))
|
|
223
|
+
ids: dict[str, int] = {}
|
|
224
|
+
multi: list[str] = []
|
|
225
|
+
for text in texts:
|
|
226
|
+
tokens = await self._tokenize(text, dialect)
|
|
227
|
+
if tokens is None:
|
|
228
|
+
self.notes.append("no tokenize endpoint: answer token ids are unverified")
|
|
229
|
+
return {}, [], False
|
|
230
|
+
if len(tokens) == 1:
|
|
231
|
+
ids[text] = tokens[0]
|
|
232
|
+
else:
|
|
233
|
+
multi.append(text)
|
|
234
|
+
return ids, multi, True
|
|
235
|
+
|
|
236
|
+
async def _thinking_off(self, dialect: Dialect) -> bool | None:
|
|
237
|
+
"""Render the probe prompt through the server's own template and look for a think marker."""
|
|
238
|
+
if self.recipe.template.mode != "messages":
|
|
239
|
+
return None
|
|
240
|
+
messages: list[dict[str, Any]] = []
|
|
241
|
+
if self.recipe.template.system:
|
|
242
|
+
messages.append({"role": "system", "content": self.recipe.template.system})
|
|
243
|
+
messages.append({"role": "user", "content": "probe"})
|
|
244
|
+
rendered: str | None = None
|
|
245
|
+
try:
|
|
246
|
+
if dialect is Dialect.LLAMACPP:
|
|
247
|
+
payload = await self.client.post_root(
|
|
248
|
+
"apply-template",
|
|
249
|
+
{
|
|
250
|
+
"messages": messages,
|
|
251
|
+
"chat_template_kwargs": dict(self.recipe.template.kwargs),
|
|
252
|
+
},
|
|
253
|
+
)
|
|
254
|
+
rendered = str(payload.get("prompt", ""))
|
|
255
|
+
else:
|
|
256
|
+
payload = await self.client.post_root(
|
|
257
|
+
"tokenize",
|
|
258
|
+
{
|
|
259
|
+
"model": self.recipe.model.name,
|
|
260
|
+
"messages": messages,
|
|
261
|
+
"add_generation_prompt": True,
|
|
262
|
+
"return_token_strs": True,
|
|
263
|
+
"chat_template_kwargs": dict(self.recipe.template.kwargs),
|
|
264
|
+
},
|
|
265
|
+
)
|
|
266
|
+
rendered = "".join(payload.get("token_strs") or [])
|
|
267
|
+
except BackendError:
|
|
268
|
+
self.notes.append("could not render the template server-side; thinking state unknown")
|
|
269
|
+
return None
|
|
270
|
+
case = load_probe_case()
|
|
271
|
+
opens = case.get("think_markers") or []
|
|
272
|
+
closes = case.get("think_close_markers") or []
|
|
273
|
+
last_open = max((rendered.rfind(m) for m in opens), default=-1)
|
|
274
|
+
if last_open < 0:
|
|
275
|
+
return True
|
|
276
|
+
# An open block that the template closes again (an empty think block) is thinking off.
|
|
277
|
+
return any(rendered.find(c, last_open) >= 0 for c in closes)
|
|
278
|
+
|
|
279
|
+
# ------------------------------------------------------------------ readout checks
|
|
280
|
+
async def _readout(self, prefix, part: RenderedPart, dialect: Dialect, top_k: int):
|
|
281
|
+
path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
|
|
282
|
+
if "messages" in body:
|
|
283
|
+
body["top_logprobs"] = top_k
|
|
284
|
+
else:
|
|
285
|
+
body["logprobs"] = top_k
|
|
286
|
+
response = await self.client.post_v1(path, body)
|
|
287
|
+
return parse_response(path, response, dialect), top_k
|
|
288
|
+
|
|
289
|
+
async def _named_token_logprobs(self, prefix, part, dialect, answer_tokens) -> bool:
|
|
290
|
+
if dialect is not Dialect.VLLM or not answer_tokens:
|
|
291
|
+
return False
|
|
292
|
+
ids = sorted(
|
|
293
|
+
{
|
|
294
|
+
answer_tokens[t.text]
|
|
295
|
+
for lbl in part.labels
|
|
296
|
+
for t in part.tokens[lbl]
|
|
297
|
+
if t.text in answer_tokens
|
|
298
|
+
}
|
|
299
|
+
)
|
|
300
|
+
path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
|
|
301
|
+
body.update({"top_logprobs": 1, "logprob_token_ids": ids, "return_as_token_id": True})
|
|
302
|
+
try:
|
|
303
|
+
response = await self.client.post_v1(path, body)
|
|
304
|
+
except BackendError as exc:
|
|
305
|
+
self.notes.append(f"logprob_token_ids rejected: {exc}")
|
|
306
|
+
return False
|
|
307
|
+
parsed = parse_response(path, response, dialect)
|
|
308
|
+
returned = {e.id for e in parsed.entries if e.id is not None}
|
|
309
|
+
return set(ids) <= returned
|
|
310
|
+
|
|
311
|
+
async def _bias_visible(self, prefix, part, dialect, entries, answer_tokens) -> bool | None:
|
|
312
|
+
"""Bias the weakest answer token by +50 and see whether its reported logprob moves."""
|
|
313
|
+
candidates = [
|
|
314
|
+
e
|
|
315
|
+
for e in entries
|
|
316
|
+
if any(matches(e, t) for lbl in part.labels for t in part.tokens[lbl])
|
|
317
|
+
]
|
|
318
|
+
if not candidates:
|
|
319
|
+
return None
|
|
320
|
+
weakest = min(candidates, key=lambda e: e.logprob)
|
|
321
|
+
ident = weakest.id if weakest.id is not None else answer_tokens.get(weakest.text)
|
|
322
|
+
if ident is None:
|
|
323
|
+
return None
|
|
324
|
+
path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
|
|
325
|
+
if "messages" in body:
|
|
326
|
+
body["top_logprobs"] = self.recipe.readout.top_k
|
|
327
|
+
else:
|
|
328
|
+
body["logprobs"] = self.recipe.readout.top_k
|
|
329
|
+
body["logit_bias"] = {str(ident): 50}
|
|
330
|
+
if dialect is Dialect.LLAMACPP:
|
|
331
|
+
neutral_samplers(body)
|
|
332
|
+
try:
|
|
333
|
+
response = await self.client.post_v1(path, body)
|
|
334
|
+
except BackendError as exc:
|
|
335
|
+
self.notes.append(f"logit_bias rejected: {exc}")
|
|
336
|
+
return None
|
|
337
|
+
parsed = parse_response(path, response, dialect)
|
|
338
|
+
for entry in parsed.entries:
|
|
339
|
+
if (entry.id == ident) or (entry.id is None and entry.text == weakest.text):
|
|
340
|
+
return entry.logprob > weakest.logprob + 1.0
|
|
341
|
+
return False
|
|
342
|
+
|
|
343
|
+
async def _grammar_honoured(self, prefix, part, dialect: Dialect) -> bool:
|
|
344
|
+
"""Under a grammar over the answer spellings only those (or their prefixes) return."""
|
|
345
|
+
texts = list(dict.fromkeys(t.text for lbl in part.labels for t in part.tokens[lbl]))
|
|
346
|
+
path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
|
|
347
|
+
body["grammar"] = grammar_for(texts)
|
|
348
|
+
neutral_samplers(body)
|
|
349
|
+
body["logprobs"] = len(texts) + 4
|
|
350
|
+
try:
|
|
351
|
+
response = await self.client.post_v1(path, body)
|
|
352
|
+
except BackendError as exc:
|
|
353
|
+
self.notes.append(f"grammar rejected: {exc}")
|
|
354
|
+
return False
|
|
355
|
+
parsed = parse_response(path, response, dialect)
|
|
356
|
+
if not parsed.entries:
|
|
357
|
+
return False
|
|
358
|
+
if all(any(t.startswith(e.text) for t in texts) for e in parsed.entries):
|
|
359
|
+
return True
|
|
360
|
+
self.notes.append(
|
|
361
|
+
"grammar accepted but the reported probabilities are not constrained by it; "
|
|
362
|
+
"the grammar rung is not proven on this build"
|
|
363
|
+
)
|
|
364
|
+
return False
|
|
365
|
+
|
|
366
|
+
async def _prefill_honored(self, dialect: Dialect) -> bool | None:
|
|
367
|
+
if not self.recipe.template.prefill:
|
|
368
|
+
return None
|
|
369
|
+
messages: list[dict[str, Any]] = [{"role": "user", "content": "probe"}]
|
|
370
|
+
messages.append({"role": "assistant", "content": self.recipe.template.prefill})
|
|
371
|
+
try:
|
|
372
|
+
payload = await self.client.post_root(
|
|
373
|
+
"tokenize",
|
|
374
|
+
{
|
|
375
|
+
"model": self.recipe.model.name,
|
|
376
|
+
"messages": messages,
|
|
377
|
+
"add_generation_prompt": False,
|
|
378
|
+
"continue_final_message": True,
|
|
379
|
+
"return_token_strs": True,
|
|
380
|
+
},
|
|
381
|
+
)
|
|
382
|
+
except BackendError:
|
|
383
|
+
return None
|
|
384
|
+
rendered = "".join(payload.get("token_strs") or [])
|
|
385
|
+
return rendered.rstrip().endswith(self.recipe.template.prefill)
|
|
386
|
+
|
|
387
|
+
async def _fit_state(self, text: str, max_context: int | None, dialect: Dialect) -> str:
|
|
388
|
+
"""Cut the probe state so the full prompt fits the backend's context (INV-9 says the
|
|
389
|
+
adapter must never truncate a user's state; the probe's own case is ours to size)."""
|
|
390
|
+
if max_context is None:
|
|
391
|
+
return text
|
|
392
|
+
budget = max_context - _CONTEXT_MARGIN
|
|
393
|
+
if budget <= 0:
|
|
394
|
+
raise BackendError(f"context of {max_context} tokens is too small to probe")
|
|
395
|
+
tokens = await self._tokenize(text, dialect)
|
|
396
|
+
count = len(tokens) if tokens is not None else int(len(text) / _CHARS_PER_TOKEN)
|
|
397
|
+
if count <= budget:
|
|
398
|
+
return text
|
|
399
|
+
keep = int(len(text) * budget / count)
|
|
400
|
+
self.notes.append(f"probe state cut to about {budget} tokens to fit the context")
|
|
401
|
+
return text[:keep]
|
|
402
|
+
|
|
403
|
+
# ------------------------------------------------------------------ modalities
|
|
404
|
+
async def _accepts_image(self, dialect: Dialect) -> bool:
|
|
405
|
+
"""Send a one-pixel PNG the way this recipe would send any image, and the same
|
|
406
|
+
request without it. The modality is proven only when the image adds prompt tokens:
|
|
407
|
+
a server can accept an image and silently drop it (llama.cpp's native route does
|
|
408
|
+
when the recipe's media marker is not the marker the server generated for this
|
|
409
|
+
process), and an accepted-but-unseen image would answer from text alone."""
|
|
410
|
+
from jevify.domain.questions import ImagePart, State, TextPart
|
|
411
|
+
|
|
412
|
+
recipe = self.recipe
|
|
413
|
+
if dialect is Dialect.LLAMACPP:
|
|
414
|
+
try:
|
|
415
|
+
props = await self.client.get_root("props")
|
|
416
|
+
except BackendError:
|
|
417
|
+
props = {}
|
|
418
|
+
marker = props.get("media_marker")
|
|
419
|
+
if marker and recipe.template is not None:
|
|
420
|
+
self.image_marker = str(marker)
|
|
421
|
+
if str(marker) != recipe.template.image_marker:
|
|
422
|
+
self.notes.append(
|
|
423
|
+
f"server media marker {marker!r} written into template.image_marker; "
|
|
424
|
+
"llama.cpp draws a new one per process unless LLAMA_MEDIA_MARKER is "
|
|
425
|
+
"set at launch, so pin it or re-probe after every restart"
|
|
426
|
+
)
|
|
427
|
+
recipe = recipe.model_copy(
|
|
428
|
+
update={"template": recipe.template.model_copy(update={"image_marker": marker})}
|
|
429
|
+
)
|
|
430
|
+
with_image = State((TextPart("probe image"), ImagePart(_one_pixel_png())))
|
|
431
|
+
without = State((TextPart("probe image"),))
|
|
432
|
+
counts: list[int | None] = []
|
|
433
|
+
for state in (with_image, without):
|
|
434
|
+
try:
|
|
435
|
+
prefix = render_prefix(recipe, state)
|
|
436
|
+
part = render_question(
|
|
437
|
+
recipe, NoulQuestion(id="image", instructions="The image is blank.")
|
|
438
|
+
).parts[0]
|
|
439
|
+
path, body = base_request(recipe.model.name, prefix, part.text, dialect)
|
|
440
|
+
except ValueError as exc:
|
|
441
|
+
self.notes.append(f"images not sendable with this recipe: {exc}")
|
|
442
|
+
return False
|
|
443
|
+
body["logprobs"] = True if "messages" in body else 1
|
|
444
|
+
if "messages" in body:
|
|
445
|
+
body["top_logprobs"] = 1
|
|
446
|
+
post = self.client.post_root if path == "completion" else self.client.post_v1
|
|
447
|
+
try:
|
|
448
|
+
parsed = parse_response(path, await post(path, body), dialect)
|
|
449
|
+
except BackendError as exc:
|
|
450
|
+
self.notes.append(f"image part rejected: {exc}")
|
|
451
|
+
return False
|
|
452
|
+
if not parsed.entries:
|
|
453
|
+
return False
|
|
454
|
+
counts.append(parsed.prompt_tokens)
|
|
455
|
+
seen, text_only = counts
|
|
456
|
+
if seen is None or text_only is None:
|
|
457
|
+
self.notes.append(
|
|
458
|
+
"image accepted; the server reports no prompt token counts, so whether the "
|
|
459
|
+
"image is seen is unproven"
|
|
460
|
+
)
|
|
461
|
+
return False
|
|
462
|
+
if seen <= text_only:
|
|
463
|
+
self.notes.append(
|
|
464
|
+
f"image accepted but not seen: {seen} prompt tokens with it, {text_only} without"
|
|
465
|
+
)
|
|
466
|
+
return False
|
|
467
|
+
return True
|
|
468
|
+
|
|
469
|
+
# ------------------------------------------------------------------ fan-out
|
|
470
|
+
async def _slots(self, dialect: Dialect) -> int | None:
|
|
471
|
+
if dialect is not Dialect.LLAMACPP:
|
|
472
|
+
return None
|
|
473
|
+
try:
|
|
474
|
+
props = await self.client.get_root("props")
|
|
475
|
+
return int(props.get("total_slots") or 1)
|
|
476
|
+
except BackendError:
|
|
477
|
+
return None
|
|
478
|
+
|
|
479
|
+
async def _fanout(self, prefix, part: RenderedPart, dialect: Dialect, slots: int | None):
|
|
480
|
+
"""Four identical-prefix readouts sequentially, then concurrently; the concurrency the
|
|
481
|
+
recipe gets is the concurrent width only when it was measurably faster."""
|
|
482
|
+
width = 4 if slots is None else max(1, min(4, slots))
|
|
483
|
+
|
|
484
|
+
def request(slot: int | None):
|
|
485
|
+
path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
|
|
486
|
+
body["logprobs"] = True if "messages" in body else 1
|
|
487
|
+
if "messages" in body:
|
|
488
|
+
body["top_logprobs"] = 1
|
|
489
|
+
if slot is not None:
|
|
490
|
+
body["id_slot"] = slot
|
|
491
|
+
post = self.client.post_root if path == "completion" else self.client.post_v1
|
|
492
|
+
return post(path, body)
|
|
493
|
+
|
|
494
|
+
slot_of = (lambda i: i % width) if slots else (lambda i: None)
|
|
495
|
+
for i in range(width): # let every slot hold the prefix before timing
|
|
496
|
+
await request(slot_of(i))
|
|
497
|
+
t0 = self.clock()
|
|
498
|
+
for i in range(width):
|
|
499
|
+
await request(slot_of(i))
|
|
500
|
+
sequential_ms = (self.clock() - t0) * 1000.0
|
|
501
|
+
t0 = self.clock()
|
|
502
|
+
await asyncio.gather(*(request(slot_of(i)) for i in range(width)))
|
|
503
|
+
concurrent_ms = (self.clock() - t0) * 1000.0
|
|
504
|
+
concurrency = width if concurrent_ms < 0.7 * sequential_ms else 1
|
|
505
|
+
return FanoutEvidence(
|
|
506
|
+
concurrency=concurrency, concurrent_ms=concurrent_ms, sequential_ms=sequential_ms
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
# ------------------------------------------------------------------ cache
|
|
510
|
+
async def _cache(self, state: State, dialect: Dialect) -> CacheEvidence:
|
|
511
|
+
"""Send identical readouts over three prefix lengths; read the cached-token count."""
|
|
512
|
+
text = state.text
|
|
513
|
+
observations: list[tuple[int, int | None]] = []
|
|
514
|
+
first_ms = second_ms = None
|
|
515
|
+
for fraction in (0.35, 0.7, 1.0):
|
|
516
|
+
sub = State.from_jev(text[: int(len(text) * fraction)])
|
|
517
|
+
prefix = render_prefix(self.recipe, sub)
|
|
518
|
+
part = render_question(
|
|
519
|
+
self.recipe, NoulQuestion(id="cache", instructions="cache probe")
|
|
520
|
+
).parts[0]
|
|
521
|
+
path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
|
|
522
|
+
body["logprobs"] = True if "messages" in body else 1
|
|
523
|
+
if "messages" in body:
|
|
524
|
+
body["top_logprobs"] = 1
|
|
525
|
+
t0 = self.clock()
|
|
526
|
+
first = parse_response(path, await self.client.post_v1(path, body), dialect)
|
|
527
|
+
t1 = self.clock()
|
|
528
|
+
second = parse_response(path, await self.client.post_v1(path, body), dialect)
|
|
529
|
+
t2 = self.clock()
|
|
530
|
+
if fraction == 1.0:
|
|
531
|
+
first_ms, second_ms = (t1 - t0) * 1000.0, (t2 - t1) * 1000.0
|
|
532
|
+
if first.prompt_tokens is not None:
|
|
533
|
+
observations.append((first.prompt_tokens, second.cached_tokens))
|
|
534
|
+
last_cached = observations[-1][1] if observations else None
|
|
535
|
+
if last_cached is None:
|
|
536
|
+
self.notes.append(
|
|
537
|
+
"backend reports no cached-token count; cache evidence is timing only"
|
|
538
|
+
)
|
|
539
|
+
return CacheEvidence(None, first_ms, second_ms, None)
|
|
540
|
+
warm_prefix, warm_reuse = await self._warm_reuse(state, dialect)
|
|
541
|
+
if all(c is not None for _, c in observations):
|
|
542
|
+
tails = {n - c for n, c in observations} # type: ignore[operator]
|
|
543
|
+
if all(0 <= t <= _MAX_REEVALUATED_TAIL for t in tails):
|
|
544
|
+
if (tail := max(tails)) > 1:
|
|
545
|
+
self.notes.append(
|
|
546
|
+
f"token-granular cache; up to {tail} trailing prompt tokens are "
|
|
547
|
+
"re-evaluated on a call"
|
|
548
|
+
)
|
|
549
|
+
return CacheEvidence(1, first_ms, second_ms, last_cached, warm_prefix, warm_reuse)
|
|
550
|
+
consistent = [
|
|
551
|
+
b
|
|
552
|
+
for b in _BLOCK_CANDIDATES
|
|
553
|
+
if b > 1 and all((c is not None) and (c == b * (n // b)) for n, c in observations)
|
|
554
|
+
]
|
|
555
|
+
block = consistent[0] if consistent else None
|
|
556
|
+
if block is None:
|
|
557
|
+
self.notes.append(f"cached-token counts {observations} fit no block size candidate")
|
|
558
|
+
return CacheEvidence(block, first_ms, second_ms, last_cached, warm_prefix, warm_reuse)
|
|
559
|
+
|
|
560
|
+
async def _warm_reuse(self, state: State, dialect: Dialect) -> tuple[int | None, int | None]:
|
|
561
|
+
"""Send the warm form (prefix only), then a question over the same state, and read
|
|
562
|
+
how much of the prefix the question reused. Chat templates close the user turn after
|
|
563
|
+
the state, so a messages-mode warm may end at a prompt no question extends; a
|
|
564
|
+
recurrent model on llama.cpp can then resume nothing (it resumes from checkpoints at
|
|
565
|
+
the end of earlier prompts), while a KV-cache model still reuses the shared tokens."""
|
|
566
|
+
prefix = render_prefix(self.recipe, state)
|
|
567
|
+
path, body = base_request(self.recipe.model.name, prefix, "", dialect)
|
|
568
|
+
body["logprobs"] = True if "messages" in body else 1
|
|
569
|
+
if "messages" in body:
|
|
570
|
+
body["top_logprobs"] = 1
|
|
571
|
+
post = self.client.post_root if path == "completion" else self.client.post_v1
|
|
572
|
+
warm = parse_response(path, await post(path, body), dialect)
|
|
573
|
+
part = render_question(
|
|
574
|
+
self.recipe, NoulQuestion(id="reuse", instructions="warm reuse probe")
|
|
575
|
+
).parts[0]
|
|
576
|
+
path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
|
|
577
|
+
body["logprobs"] = True if "messages" in body else 1
|
|
578
|
+
if "messages" in body:
|
|
579
|
+
body["top_logprobs"] = 1
|
|
580
|
+
question = parse_response(path, await post(path, body), dialect)
|
|
581
|
+
if warm.prompt_tokens is None or question.cached_tokens is None:
|
|
582
|
+
return None, None
|
|
583
|
+
if warm.prompt_tokens > 0 and question.cached_tokens < warm.prompt_tokens // 2:
|
|
584
|
+
self.notes.append(
|
|
585
|
+
f"warm prefix is not reused by question requests ({question.cached_tokens} of "
|
|
586
|
+
f"{warm.prompt_tokens} prefix tokens); on a recurrent model use raw mode so the "
|
|
587
|
+
"warm prompt ends at the state boundary"
|
|
588
|
+
)
|
|
589
|
+
return warm.prompt_tokens, question.cached_tokens
|