jevify 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. jevify/__init__.py +7 -0
  2. jevify/__main__.py +9 -0
  3. jevify/adapters/__init__.py +1 -0
  4. jevify/adapters/embedding/__init__.py +1 -0
  5. jevify/adapters/embedding/adapter.py +203 -0
  6. jevify/adapters/embedding/local.py +82 -0
  7. jevify/adapters/encoder/__init__.py +1 -0
  8. jevify/adapters/encoder/adapter.py +165 -0
  9. jevify/adapters/encoder/local.py +77 -0
  10. jevify/adapters/endpoint/__init__.py +1 -0
  11. jevify/adapters/endpoint/adapter.py +281 -0
  12. jevify/adapters/endpoint/client.py +114 -0
  13. jevify/adapters/endpoint/dialects.py +166 -0
  14. jevify/adapters/endpoint/probe.py +589 -0
  15. jevify/adapters/endpoint/readout.py +244 -0
  16. jevify/adapters/rerank/__init__.py +1 -0
  17. jevify/adapters/rerank/adapter.py +152 -0
  18. jevify/api/__init__.py +1 -0
  19. jevify/api/app.py +228 -0
  20. jevify/api/schema.py +123 -0
  21. jevify/api/translate.py +239 -0
  22. jevify/cli.py +392 -0
  23. jevify/compose.py +82 -0
  24. jevify/domain/__init__.py +1 -0
  25. jevify/domain/distribution.py +103 -0
  26. jevify/domain/engine.py +152 -0
  27. jevify/domain/questions.py +121 -0
  28. jevify/evaluation/__init__.py +1 -0
  29. jevify/evaluation/calibrate.py +186 -0
  30. jevify/evaluation/controls.py +78 -0
  31. jevify/evaluation/metrics.py +91 -0
  32. jevify/evaluation/scorecard.py +385 -0
  33. jevify/evaluation/suites.py +166 -0
  34. jevify/ports/__init__.py +1 -0
  35. jevify/ports/backend.py +162 -0
  36. jevify/recipes/__init__.py +21 -0
  37. jevify/recipes/probe_case.yaml +72 -0
  38. jevify/recipes/render.py +198 -0
  39. jevify/recipes/schema.py +204 -0
  40. jevify/recipes/store.py +81 -0
  41. jevify-0.1.0.dist-info/METADATA +243 -0
  42. jevify-0.1.0.dist-info/RECORD +45 -0
  43. jevify-0.1.0.dist-info/WHEEL +4 -0
  44. jevify-0.1.0.dist-info/entry_points.txt +2 -0
  45. jevify-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,589 @@
1
+ """The capability probe (ADR-0002 D3): a measurement, not a lookup table.
2
+
3
+ Every check sends real requests to the backend and records what came back.
4
+ The result is written into the recipe by `jevify probe`, and the adapter
5
+ picks its readout rung from it.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ import datetime as dt
12
+ import time
13
+ from collections.abc import Callable
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ import yaml
18
+
19
+ from jevify.adapters.endpoint.client import OpenAICompatibleClient
20
+ from jevify.adapters.endpoint.dialects import (
21
+ base_request,
22
+ grammar_for,
23
+ neutral_samplers,
24
+ parse_response,
25
+ )
26
+ from jevify.adapters.endpoint.readout import matches, read_answer_set
27
+ from jevify.domain.questions import NoulQuestion, State
28
+ from jevify.ports.backend import (
29
+ BackendError,
30
+ CacheEvidence,
31
+ Capabilities,
32
+ Dialect,
33
+ FanoutEvidence,
34
+ Rung,
35
+ )
36
+ from jevify.recipes.render import RenderedPart, render_prefix, render_question
37
+ from jevify.recipes.schema import Recipe, Token
38
+
39
+ Clock = Callable[[], float]
40
+
41
+ _PROBE_CASE = Path(__file__).resolve().parents[2] / "recipes" / "probe_case.yaml"
42
+ _BLOCK_CANDIDATES = (1, 16, 32, 64, 128, 256, 512, 1024, 2048)
43
+ _MAX_REEVALUATED_TAIL = 8 # llama.cpp re-evaluates a few trailing tokens; still token-granular
44
+
45
+
46
+ def _one_pixel_png() -> str:
47
+ """A 1x1 white PNG as a data URI, built here so no binary blob lives in the tree."""
48
+ import base64
49
+ import struct
50
+ import zlib
51
+
52
+ def chunk(tag: bytes, data: bytes) -> bytes:
53
+ return (
54
+ struct.pack(">I", len(data))
55
+ + tag
56
+ + data
57
+ + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
58
+ )
59
+
60
+ png = (
61
+ b"\x89PNG\r\n\x1a\n"
62
+ + chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
63
+ + chunk(b"IDAT", zlib.compress(b"\x00\xff\xff\xff", 9))
64
+ + chunk(b"IEND", b"")
65
+ )
66
+ return "data:image/png;base64," + base64.b64encode(png).decode("ascii")
67
+
68
+
69
+ def load_probe_case() -> dict[str, Any]:
70
+ return yaml.safe_load(_PROBE_CASE.read_text(encoding="utf-8"))
71
+
72
+
73
+ _CONTEXT_MARGIN = 160 # room for the template, the question and the answer slot
74
+ _CHARS_PER_TOKEN = 3.0 # conservative estimate when the backend cannot tokenize
75
+
76
+
77
+ class Prober:
78
+ def __init__(
79
+ self, client: OpenAICompatibleClient, recipe: Recipe, clock: Clock = time.perf_counter
80
+ ):
81
+ self.client = client
82
+ self.recipe = recipe
83
+ self.clock = clock
84
+ self.notes: list[str] = []
85
+ self.image_marker: str | None = None
86
+
87
+ async def run(self) -> Capabilities:
88
+ assert self.recipe.endpoint is not None
89
+ case = load_probe_case()
90
+ dialect, version = await self._dialect()
91
+ max_context = await self._max_context(dialect)
92
+ answer_tokens, multi, verified = await self._verify_tokens(dialect)
93
+ kwargs_honored = await self._thinking_off(dialect)
94
+
95
+ state = State.from_jev(await self._fit_state(case["state"], max_context, dialect))
96
+ prefix = render_prefix(self.recipe, state)
97
+ question = render_question(
98
+ self.recipe, NoulQuestion(id="probe", instructions=case["statement"])
99
+ )
100
+ part = question.parts[0]
101
+
102
+ parsed, entries_requested = await self._readout(
103
+ prefix, part, dialect, self.recipe.readout.top_k
104
+ )
105
+ top_k_cap = len(parsed.entries) if parsed.entries else 0
106
+ if top_k_cap == 0:
107
+ raise BackendError("no readout is possible on this backend: it returned no logprobs")
108
+ readout = read_answer_set(parsed, part, full_softmax=True)
109
+ if readout.missing:
110
+ self.notes.append(
111
+ f"probe case: labels {list(readout.missing)} outside the top-{top_k_cap}"
112
+ )
113
+
114
+ rungs = {Rung.TOP_K, Rung.TOP_K_FLOOR}
115
+ if await self._named_token_logprobs(prefix, part, dialect, answer_tokens):
116
+ rungs.add(Rung.NAMED_TOKEN_LOGPROBS)
117
+ post_bias = await self._bias_visible(prefix, part, dialect, parsed.entries, answer_tokens)
118
+ if post_bias:
119
+ rungs.add(Rung.EQUAL_BIAS)
120
+ if dialect is Dialect.LLAMACPP and await self._grammar_honoured(prefix, part, dialect):
121
+ rungs.add(Rung.GRAMMAR)
122
+ prefill = await self._prefill_honored(dialect) if prefix.mode == "messages" else None
123
+ cache = await self._cache(state, dialect)
124
+ slots = await self._slots(dialect)
125
+ fanout = await self._fanout(prefix, part, dialect, slots)
126
+ modalities = {"text"}
127
+ if await self._accepts_image(dialect):
128
+ modalities.add("image")
129
+
130
+ return Capabilities(
131
+ kind="endpoint",
132
+ model=self.recipe.model.name,
133
+ dialect=dialect,
134
+ backend_version=version,
135
+ max_context=max_context,
136
+ modalities=frozenset(modalities),
137
+ rungs=frozenset(rungs),
138
+ top_k_cap=top_k_cap if top_k_cap < entries_requested else None,
139
+ logprobs_post_bias=post_bias,
140
+ prefill_honored=prefill,
141
+ template_kwargs_honored=kwargs_honored,
142
+ image_marker=self.image_marker,
143
+ token_ids_verified=verified,
144
+ answer_tokens=answer_tokens,
145
+ multi_token_answers=tuple(multi),
146
+ cache=cache,
147
+ fanout=fanout,
148
+ slots=slots,
149
+ probed_at=dt.datetime.now(dt.UTC).isoformat(timespec="seconds"),
150
+ notes=tuple(self.notes),
151
+ )
152
+
153
+ # ------------------------------------------------------------------ identity
154
+ async def _dialect(self) -> tuple[Dialect, str | None]:
155
+ assert self.recipe.endpoint is not None
156
+ pinned = self.recipe.endpoint.dialect
157
+ for candidate, path, key in (
158
+ (Dialect.VLLM, "version", "version"),
159
+ (Dialect.LLAMACPP, "props", "build_info"),
160
+ ):
161
+ if pinned not in ("auto", str(candidate)):
162
+ continue
163
+ try:
164
+ payload = await self.client.get_root(path)
165
+ except BackendError:
166
+ continue
167
+ if isinstance(payload, dict) and key in payload:
168
+ return candidate, str(payload[key])
169
+ if pinned not in ("auto", "generic"):
170
+ self.notes.append(f"dialect {pinned} pinned but its identity endpoint did not answer")
171
+ return Dialect(pinned), None
172
+ return Dialect.GENERIC, None
173
+
174
+ async def _max_context(self, dialect: Dialect) -> int | None:
175
+ """The context the server actually serves: llama.cpp's per-slot n_ctx outranks the
176
+ model's trained context, which /v1/models reports as meta.n_ctx_train."""
177
+ if dialect is Dialect.LLAMACPP:
178
+ try:
179
+ props = await self.client.get_root("props")
180
+ n_ctx = (props.get("default_generation_settings") or {}).get("n_ctx")
181
+ if n_ctx:
182
+ return int(n_ctx)
183
+ except BackendError:
184
+ pass
185
+ try:
186
+ models = await self.client.get_v1("models")
187
+ except BackendError:
188
+ models = {}
189
+ for entry in models.get("data") or []:
190
+ if entry.get("max_model_len"):
191
+ return int(entry["max_model_len"])
192
+ meta = entry.get("meta") or {}
193
+ if meta.get("n_ctx_train"):
194
+ self.notes.append(
195
+ "max_context is the trained context; the served context is unknown"
196
+ )
197
+ return int(meta["n_ctx_train"])
198
+ return None
199
+
200
+ # ------------------------------------------------------------------ tokens
201
+ async def _tokenize(self, text: str, dialect: Dialect) -> list[int] | None:
202
+ try:
203
+ if dialect is Dialect.LLAMACPP:
204
+ payload = await self.client.post_root(
205
+ "tokenize", {"content": text, "add_special": False, "parse_special": False}
206
+ )
207
+ tokens = payload.get("tokens") or []
208
+ return [t["id"] if isinstance(t, dict) else int(t) for t in tokens]
209
+ payload = await self.client.post_root(
210
+ "tokenize",
211
+ {"model": self.recipe.model.name, "prompt": text, "add_special_tokens": False},
212
+ )
213
+ return [int(t) for t in payload.get("tokens") or []]
214
+ except BackendError:
215
+ return None
216
+
217
+ async def _verify_tokens(self, dialect: Dialect) -> tuple[dict[str, int], list[str], bool]:
218
+ groups: list[list[Token]] = [
219
+ *self.recipe.answers.noul.values(),
220
+ *self.recipe.answers.identifiers,
221
+ ]
222
+ texts = list(dict.fromkeys(token.text for group in groups for token in group))
223
+ ids: dict[str, int] = {}
224
+ multi: list[str] = []
225
+ for text in texts:
226
+ tokens = await self._tokenize(text, dialect)
227
+ if tokens is None:
228
+ self.notes.append("no tokenize endpoint: answer token ids are unverified")
229
+ return {}, [], False
230
+ if len(tokens) == 1:
231
+ ids[text] = tokens[0]
232
+ else:
233
+ multi.append(text)
234
+ return ids, multi, True
235
+
236
+ async def _thinking_off(self, dialect: Dialect) -> bool | None:
237
+ """Render the probe prompt through the server's own template and look for a think marker."""
238
+ if self.recipe.template.mode != "messages":
239
+ return None
240
+ messages: list[dict[str, Any]] = []
241
+ if self.recipe.template.system:
242
+ messages.append({"role": "system", "content": self.recipe.template.system})
243
+ messages.append({"role": "user", "content": "probe"})
244
+ rendered: str | None = None
245
+ try:
246
+ if dialect is Dialect.LLAMACPP:
247
+ payload = await self.client.post_root(
248
+ "apply-template",
249
+ {
250
+ "messages": messages,
251
+ "chat_template_kwargs": dict(self.recipe.template.kwargs),
252
+ },
253
+ )
254
+ rendered = str(payload.get("prompt", ""))
255
+ else:
256
+ payload = await self.client.post_root(
257
+ "tokenize",
258
+ {
259
+ "model": self.recipe.model.name,
260
+ "messages": messages,
261
+ "add_generation_prompt": True,
262
+ "return_token_strs": True,
263
+ "chat_template_kwargs": dict(self.recipe.template.kwargs),
264
+ },
265
+ )
266
+ rendered = "".join(payload.get("token_strs") or [])
267
+ except BackendError:
268
+ self.notes.append("could not render the template server-side; thinking state unknown")
269
+ return None
270
+ case = load_probe_case()
271
+ opens = case.get("think_markers") or []
272
+ closes = case.get("think_close_markers") or []
273
+ last_open = max((rendered.rfind(m) for m in opens), default=-1)
274
+ if last_open < 0:
275
+ return True
276
+ # An open block that the template closes again (an empty think block) is thinking off.
277
+ return any(rendered.find(c, last_open) >= 0 for c in closes)
278
+
279
+ # ------------------------------------------------------------------ readout checks
280
+ async def _readout(self, prefix, part: RenderedPart, dialect: Dialect, top_k: int):
281
+ path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
282
+ if "messages" in body:
283
+ body["top_logprobs"] = top_k
284
+ else:
285
+ body["logprobs"] = top_k
286
+ response = await self.client.post_v1(path, body)
287
+ return parse_response(path, response, dialect), top_k
288
+
289
+ async def _named_token_logprobs(self, prefix, part, dialect, answer_tokens) -> bool:
290
+ if dialect is not Dialect.VLLM or not answer_tokens:
291
+ return False
292
+ ids = sorted(
293
+ {
294
+ answer_tokens[t.text]
295
+ for lbl in part.labels
296
+ for t in part.tokens[lbl]
297
+ if t.text in answer_tokens
298
+ }
299
+ )
300
+ path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
301
+ body.update({"top_logprobs": 1, "logprob_token_ids": ids, "return_as_token_id": True})
302
+ try:
303
+ response = await self.client.post_v1(path, body)
304
+ except BackendError as exc:
305
+ self.notes.append(f"logprob_token_ids rejected: {exc}")
306
+ return False
307
+ parsed = parse_response(path, response, dialect)
308
+ returned = {e.id for e in parsed.entries if e.id is not None}
309
+ return set(ids) <= returned
310
+
311
+ async def _bias_visible(self, prefix, part, dialect, entries, answer_tokens) -> bool | None:
312
+ """Bias the weakest answer token by +50 and see whether its reported logprob moves."""
313
+ candidates = [
314
+ e
315
+ for e in entries
316
+ if any(matches(e, t) for lbl in part.labels for t in part.tokens[lbl])
317
+ ]
318
+ if not candidates:
319
+ return None
320
+ weakest = min(candidates, key=lambda e: e.logprob)
321
+ ident = weakest.id if weakest.id is not None else answer_tokens.get(weakest.text)
322
+ if ident is None:
323
+ return None
324
+ path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
325
+ if "messages" in body:
326
+ body["top_logprobs"] = self.recipe.readout.top_k
327
+ else:
328
+ body["logprobs"] = self.recipe.readout.top_k
329
+ body["logit_bias"] = {str(ident): 50}
330
+ if dialect is Dialect.LLAMACPP:
331
+ neutral_samplers(body)
332
+ try:
333
+ response = await self.client.post_v1(path, body)
334
+ except BackendError as exc:
335
+ self.notes.append(f"logit_bias rejected: {exc}")
336
+ return None
337
+ parsed = parse_response(path, response, dialect)
338
+ for entry in parsed.entries:
339
+ if (entry.id == ident) or (entry.id is None and entry.text == weakest.text):
340
+ return entry.logprob > weakest.logprob + 1.0
341
+ return False
342
+
343
+ async def _grammar_honoured(self, prefix, part, dialect: Dialect) -> bool:
344
+ """Under a grammar over the answer spellings only those (or their prefixes) return."""
345
+ texts = list(dict.fromkeys(t.text for lbl in part.labels for t in part.tokens[lbl]))
346
+ path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
347
+ body["grammar"] = grammar_for(texts)
348
+ neutral_samplers(body)
349
+ body["logprobs"] = len(texts) + 4
350
+ try:
351
+ response = await self.client.post_v1(path, body)
352
+ except BackendError as exc:
353
+ self.notes.append(f"grammar rejected: {exc}")
354
+ return False
355
+ parsed = parse_response(path, response, dialect)
356
+ if not parsed.entries:
357
+ return False
358
+ if all(any(t.startswith(e.text) for t in texts) for e in parsed.entries):
359
+ return True
360
+ self.notes.append(
361
+ "grammar accepted but the reported probabilities are not constrained by it; "
362
+ "the grammar rung is not proven on this build"
363
+ )
364
+ return False
365
+
366
+ async def _prefill_honored(self, dialect: Dialect) -> bool | None:
367
+ if not self.recipe.template.prefill:
368
+ return None
369
+ messages: list[dict[str, Any]] = [{"role": "user", "content": "probe"}]
370
+ messages.append({"role": "assistant", "content": self.recipe.template.prefill})
371
+ try:
372
+ payload = await self.client.post_root(
373
+ "tokenize",
374
+ {
375
+ "model": self.recipe.model.name,
376
+ "messages": messages,
377
+ "add_generation_prompt": False,
378
+ "continue_final_message": True,
379
+ "return_token_strs": True,
380
+ },
381
+ )
382
+ except BackendError:
383
+ return None
384
+ rendered = "".join(payload.get("token_strs") or [])
385
+ return rendered.rstrip().endswith(self.recipe.template.prefill)
386
+
387
+ async def _fit_state(self, text: str, max_context: int | None, dialect: Dialect) -> str:
388
+ """Cut the probe state so the full prompt fits the backend's context (INV-9 says the
389
+ adapter must never truncate a user's state; the probe's own case is ours to size)."""
390
+ if max_context is None:
391
+ return text
392
+ budget = max_context - _CONTEXT_MARGIN
393
+ if budget <= 0:
394
+ raise BackendError(f"context of {max_context} tokens is too small to probe")
395
+ tokens = await self._tokenize(text, dialect)
396
+ count = len(tokens) if tokens is not None else int(len(text) / _CHARS_PER_TOKEN)
397
+ if count <= budget:
398
+ return text
399
+ keep = int(len(text) * budget / count)
400
+ self.notes.append(f"probe state cut to about {budget} tokens to fit the context")
401
+ return text[:keep]
402
+
403
+ # ------------------------------------------------------------------ modalities
404
+ async def _accepts_image(self, dialect: Dialect) -> bool:
405
+ """Send a one-pixel PNG the way this recipe would send any image, and the same
406
+ request without it. The modality is proven only when the image adds prompt tokens:
407
+ a server can accept an image and silently drop it (llama.cpp's native route does
408
+ when the recipe's media marker is not the marker the server generated for this
409
+ process), and an accepted-but-unseen image would answer from text alone."""
410
+ from jevify.domain.questions import ImagePart, State, TextPart
411
+
412
+ recipe = self.recipe
413
+ if dialect is Dialect.LLAMACPP:
414
+ try:
415
+ props = await self.client.get_root("props")
416
+ except BackendError:
417
+ props = {}
418
+ marker = props.get("media_marker")
419
+ if marker and recipe.template is not None:
420
+ self.image_marker = str(marker)
421
+ if str(marker) != recipe.template.image_marker:
422
+ self.notes.append(
423
+ f"server media marker {marker!r} written into template.image_marker; "
424
+ "llama.cpp draws a new one per process unless LLAMA_MEDIA_MARKER is "
425
+ "set at launch, so pin it or re-probe after every restart"
426
+ )
427
+ recipe = recipe.model_copy(
428
+ update={"template": recipe.template.model_copy(update={"image_marker": marker})}
429
+ )
430
+ with_image = State((TextPart("probe image"), ImagePart(_one_pixel_png())))
431
+ without = State((TextPart("probe image"),))
432
+ counts: list[int | None] = []
433
+ for state in (with_image, without):
434
+ try:
435
+ prefix = render_prefix(recipe, state)
436
+ part = render_question(
437
+ recipe, NoulQuestion(id="image", instructions="The image is blank.")
438
+ ).parts[0]
439
+ path, body = base_request(recipe.model.name, prefix, part.text, dialect)
440
+ except ValueError as exc:
441
+ self.notes.append(f"images not sendable with this recipe: {exc}")
442
+ return False
443
+ body["logprobs"] = True if "messages" in body else 1
444
+ if "messages" in body:
445
+ body["top_logprobs"] = 1
446
+ post = self.client.post_root if path == "completion" else self.client.post_v1
447
+ try:
448
+ parsed = parse_response(path, await post(path, body), dialect)
449
+ except BackendError as exc:
450
+ self.notes.append(f"image part rejected: {exc}")
451
+ return False
452
+ if not parsed.entries:
453
+ return False
454
+ counts.append(parsed.prompt_tokens)
455
+ seen, text_only = counts
456
+ if seen is None or text_only is None:
457
+ self.notes.append(
458
+ "image accepted; the server reports no prompt token counts, so whether the "
459
+ "image is seen is unproven"
460
+ )
461
+ return False
462
+ if seen <= text_only:
463
+ self.notes.append(
464
+ f"image accepted but not seen: {seen} prompt tokens with it, {text_only} without"
465
+ )
466
+ return False
467
+ return True
468
+
469
+ # ------------------------------------------------------------------ fan-out
470
+ async def _slots(self, dialect: Dialect) -> int | None:
471
+ if dialect is not Dialect.LLAMACPP:
472
+ return None
473
+ try:
474
+ props = await self.client.get_root("props")
475
+ return int(props.get("total_slots") or 1)
476
+ except BackendError:
477
+ return None
478
+
479
+ async def _fanout(self, prefix, part: RenderedPart, dialect: Dialect, slots: int | None):
480
+ """Four identical-prefix readouts sequentially, then concurrently; the concurrency the
481
+ recipe gets is the concurrent width only when it was measurably faster."""
482
+ width = 4 if slots is None else max(1, min(4, slots))
483
+
484
+ def request(slot: int | None):
485
+ path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
486
+ body["logprobs"] = True if "messages" in body else 1
487
+ if "messages" in body:
488
+ body["top_logprobs"] = 1
489
+ if slot is not None:
490
+ body["id_slot"] = slot
491
+ post = self.client.post_root if path == "completion" else self.client.post_v1
492
+ return post(path, body)
493
+
494
+ slot_of = (lambda i: i % width) if slots else (lambda i: None)
495
+ for i in range(width): # let every slot hold the prefix before timing
496
+ await request(slot_of(i))
497
+ t0 = self.clock()
498
+ for i in range(width):
499
+ await request(slot_of(i))
500
+ sequential_ms = (self.clock() - t0) * 1000.0
501
+ t0 = self.clock()
502
+ await asyncio.gather(*(request(slot_of(i)) for i in range(width)))
503
+ concurrent_ms = (self.clock() - t0) * 1000.0
504
+ concurrency = width if concurrent_ms < 0.7 * sequential_ms else 1
505
+ return FanoutEvidence(
506
+ concurrency=concurrency, concurrent_ms=concurrent_ms, sequential_ms=sequential_ms
507
+ )
508
+
509
+ # ------------------------------------------------------------------ cache
510
+ async def _cache(self, state: State, dialect: Dialect) -> CacheEvidence:
511
+ """Send identical readouts over three prefix lengths; read the cached-token count."""
512
+ text = state.text
513
+ observations: list[tuple[int, int | None]] = []
514
+ first_ms = second_ms = None
515
+ for fraction in (0.35, 0.7, 1.0):
516
+ sub = State.from_jev(text[: int(len(text) * fraction)])
517
+ prefix = render_prefix(self.recipe, sub)
518
+ part = render_question(
519
+ self.recipe, NoulQuestion(id="cache", instructions="cache probe")
520
+ ).parts[0]
521
+ path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
522
+ body["logprobs"] = True if "messages" in body else 1
523
+ if "messages" in body:
524
+ body["top_logprobs"] = 1
525
+ t0 = self.clock()
526
+ first = parse_response(path, await self.client.post_v1(path, body), dialect)
527
+ t1 = self.clock()
528
+ second = parse_response(path, await self.client.post_v1(path, body), dialect)
529
+ t2 = self.clock()
530
+ if fraction == 1.0:
531
+ first_ms, second_ms = (t1 - t0) * 1000.0, (t2 - t1) * 1000.0
532
+ if first.prompt_tokens is not None:
533
+ observations.append((first.prompt_tokens, second.cached_tokens))
534
+ last_cached = observations[-1][1] if observations else None
535
+ if last_cached is None:
536
+ self.notes.append(
537
+ "backend reports no cached-token count; cache evidence is timing only"
538
+ )
539
+ return CacheEvidence(None, first_ms, second_ms, None)
540
+ warm_prefix, warm_reuse = await self._warm_reuse(state, dialect)
541
+ if all(c is not None for _, c in observations):
542
+ tails = {n - c for n, c in observations} # type: ignore[operator]
543
+ if all(0 <= t <= _MAX_REEVALUATED_TAIL for t in tails):
544
+ if (tail := max(tails)) > 1:
545
+ self.notes.append(
546
+ f"token-granular cache; up to {tail} trailing prompt tokens are "
547
+ "re-evaluated on a call"
548
+ )
549
+ return CacheEvidence(1, first_ms, second_ms, last_cached, warm_prefix, warm_reuse)
550
+ consistent = [
551
+ b
552
+ for b in _BLOCK_CANDIDATES
553
+ if b > 1 and all((c is not None) and (c == b * (n // b)) for n, c in observations)
554
+ ]
555
+ block = consistent[0] if consistent else None
556
+ if block is None:
557
+ self.notes.append(f"cached-token counts {observations} fit no block size candidate")
558
+ return CacheEvidence(block, first_ms, second_ms, last_cached, warm_prefix, warm_reuse)
559
+
560
+ async def _warm_reuse(self, state: State, dialect: Dialect) -> tuple[int | None, int | None]:
561
+ """Send the warm form (prefix only), then a question over the same state, and read
562
+ how much of the prefix the question reused. Chat templates close the user turn after
563
+ the state, so a messages-mode warm may end at a prompt no question extends; a
564
+ recurrent model on llama.cpp can then resume nothing (it resumes from checkpoints at
565
+ the end of earlier prompts), while a KV-cache model still reuses the shared tokens."""
566
+ prefix = render_prefix(self.recipe, state)
567
+ path, body = base_request(self.recipe.model.name, prefix, "", dialect)
568
+ body["logprobs"] = True if "messages" in body else 1
569
+ if "messages" in body:
570
+ body["top_logprobs"] = 1
571
+ post = self.client.post_root if path == "completion" else self.client.post_v1
572
+ warm = parse_response(path, await post(path, body), dialect)
573
+ part = render_question(
574
+ self.recipe, NoulQuestion(id="reuse", instructions="warm reuse probe")
575
+ ).parts[0]
576
+ path, body = base_request(self.recipe.model.name, prefix, part.text, dialect)
577
+ body["logprobs"] = True if "messages" in body else 1
578
+ if "messages" in body:
579
+ body["top_logprobs"] = 1
580
+ question = parse_response(path, await post(path, body), dialect)
581
+ if warm.prompt_tokens is None or question.cached_tokens is None:
582
+ return None, None
583
+ if warm.prompt_tokens > 0 and question.cached_tokens < warm.prompt_tokens // 2:
584
+ self.notes.append(
585
+ f"warm prefix is not reused by question requests ({question.cached_tokens} of "
586
+ f"{warm.prompt_tokens} prefix tokens); on a recurrent model use raw mode so the "
587
+ "warm prompt ends at the state boundary"
588
+ )
589
+ return warm.prompt_tokens, question.cached_tokens