crit-eval 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {crit_eval-0.1.2 → crit_eval-0.2.0}/PKG-INFO +3 -3
- {crit_eval-0.1.2 → crit_eval-0.2.0}/README.md +1 -1
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/__init__.py +2 -2
- crit_eval-0.2.0/crit_eval/architectures/__init__.py +3 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/architectures}/gguf_layout.py +5 -5
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/architectures}/gguf_qwen4exp.py +1 -1
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/cli.py +52 -39
- crit_eval-0.2.0/crit_eval/limits/__init__.py +2 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/ceiling.py +5 -5
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/context.py +1 -1
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/headroom.py +2 -2
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/kernels.py +6 -6
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/probe.py +3 -2
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/schemes.py +28 -190
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/tensors.py +1 -1
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/protocol.json +12 -4
- crit_eval-0.2.0/crit_eval/quality/__init__.py +2 -0
- crit_eval-0.2.0/crit_eval/quality/attention.py +79 -0
- crit_eval-0.2.0/crit_eval/quality/checkpoints.py +366 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/fixed.py +2 -2
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/logprob.py +2 -2
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/packages.py +24 -10
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/prepare.py +6 -6
- crit_eval-0.2.0/crit_eval/quality/prompt_state.py +391 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/reference.py +36 -845
- crit_eval-0.2.0/crit_eval/reports/__init__.py +2 -0
- crit_eval-0.1.2/crit_eval/report.py → crit_eval-0.2.0/crit_eval/reports/compare.py +2 -2
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/reports}/metrics.py +6 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/reports}/status.py +11 -9
- crit_eval-0.2.0/crit_eval/run/__init__.py +2 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/ab.py +5 -3
- crit_eval-0.2.0/crit_eval/run/adapters.py +96 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/chat.py +4 -4
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/estimate.py +7 -7
- crit_eval-0.2.0/crit_eval/run/history.py +71 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/host.py +1 -1
- crit_eval-0.2.0/crit_eval/run/long_context.py +215 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/pin.py +4 -3
- crit_eval-0.2.0/crit_eval/run/quality_checks.py +319 -0
- crit_eval-0.2.0/crit_eval/run/runner.py +429 -0
- crit_eval-0.2.0/crit_eval/run/speed.py +322 -0
- crit_eval-0.2.0/crit_eval/run/timing.py +135 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/PKG-INFO +3 -3
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/SOURCES.txt +42 -28
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/requires.txt +1 -1
- {crit_eval-0.1.2 → crit_eval-0.2.0}/pyproject.toml +2 -2
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval.py +137 -94
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_chat_template.py +5 -2
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_deadline.py +5 -4
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_distributions.py +3 -1
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_estimate.py +3 -3
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_followup.py +7 -6
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_formats.py +8 -8
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_gguf.py +8 -7
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_gguf_deepseek4.py +10 -10
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_gguf_qwen4exp.py +3 -3
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_indexers.py +2 -2
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_kernels.py +3 -3
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_packages.py +40 -10
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_quality_modes.py +49 -13
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_server.py +12 -3
- {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_tensors.py +2 -2
- crit_eval-0.1.2/crit_eval/runner.py +0 -1497
- {crit_eval-0.1.2 → crit_eval-0.2.0}/LICENSE +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/__main__.py +0 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/architectures}/gguf_deepseek4.py +0 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/architectures}/indexers.py +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/examples/mock_adapter.py +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/probe/device_probe.cu +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/probe/kernel_trace.cpp +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/probe/logprob_compare.cu +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/probe/logprob_compare.h +0 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/deadline.py +0 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/hostload.py +0 -0
- {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/outcomes.py +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/README.md.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/cli.py.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/headroom.py.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/metrics.py.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/prepare.py.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/reference.py.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/report.py.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/runner.py.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/tools.json +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/followup.json +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/heldout.json +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/long.json +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/long_document.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/quality.json +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/speed.json +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/dependency_links.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/entry_points.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/top_level.txt +0 -0
- {crit_eval-0.1.2 → crit_eval-0.2.0}/setup.cfg +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: crit-eval
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: crit-eval: one speed, quality and exactness suite that every inference engine runs the same way
|
|
5
5
|
License-Expression: Apache-2.0
|
|
6
6
|
Project-URL: Repository, https://github.com/crit-labs/crit-eval
|
|
7
7
|
Requires-Python: >=3.11
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
License-File: LICENSE
|
|
10
|
-
Requires-Dist: crit-core<0.
|
|
10
|
+
Requires-Dist: crit-core<0.13,>=0.12.0
|
|
11
11
|
Provides-Extra: reference
|
|
12
12
|
Requires-Dist: torch; extra == "reference"
|
|
13
13
|
Requires-Dist: transformers>=5.20; extra == "reference"
|
|
@@ -162,7 +162,7 @@ plugs in through a Python [adapter](docs/ADAPTER.md) instead of `--url`.
|
|
|
162
162
|
| --- | --- |
|
|
163
163
|
| Quality reference | Quality is measured against the checkpoint as stored, for example an NVFP4 quantization, not against the full-precision model. `crit-eval` reports the drift the engine adds, never the quantization loss of the checkpoint itself, so quality results of two different quantizations of a model do not compare. |
|
|
164
164
|
| What quality covers | Drift on fixed texts: six conversations (prose, code, math, tool calls and more) and one long text each at 1K, 4K and 32K tokens, as perplexity ratio and KL divergence. It is not a task benchmark: no accuracy on MMLU-style questions, coding tests or agent tasks. |
|
|
165
|
-
| Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages
|
|
165
|
+
| Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages for three checkpoints are published on Hugging Face ([crit-labs/crit-eval-references](https://huggingface.co/datasets/crit-labs/crit-eval-references)) and fetched by default. |
|
|
166
166
|
| Rounding-sensitive models | Where a model amplifies rounding (MoE routing near-ties, sparse-attention ties, recurrent state), two valid FP32 runs of the reference disagree; quality is then judged against twice the reference's own spread instead of the budget, which is looser for those models. |
|
|
167
167
|
| Speed | One request at a time: `crit-eval` measures single-stream decode, prefill and time to first token, not throughput under concurrent requests or batching. A noisy sampled decode is repeated at up to nine seeds per case, so an engine whose speed swings with the drawn text takes longer to measure (`--no-adaptive-sampled` keeps three, an override the result records). |
|
|
168
168
|
| Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded, but no limit uses them yet, so engines that offload weights or compute on the CPU get GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
|
|
@@ -145,7 +145,7 @@ plugs in through a Python [adapter](docs/ADAPTER.md) instead of `--url`.
|
|
|
145
145
|
| --- | --- |
|
|
146
146
|
| Quality reference | Quality is measured against the checkpoint as stored, for example an NVFP4 quantization, not against the full-precision model. `crit-eval` reports the drift the engine adds, never the quantization loss of the checkpoint itself, so quality results of two different quantizations of a model do not compare. |
|
|
147
147
|
| What quality covers | Drift on fixed texts: six conversations (prose, code, math, tool calls and more) and one long text each at 1K, 4K and 32K tokens, as perplexity ratio and KL divergence. It is not a task benchmark: no accuracy on MMLU-style questions, coding tests or agent tasks. |
|
|
148
|
-
| Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages
|
|
148
|
+
| Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages for three checkpoints are published on Hugging Face ([crit-labs/crit-eval-references](https://huggingface.co/datasets/crit-labs/crit-eval-references)) and fetched by default. |
|
|
149
149
|
| Rounding-sensitive models | Where a model amplifies rounding (MoE routing near-ties, sparse-attention ties, recurrent state), two valid FP32 runs of the reference disagree; quality is then judged against twice the reference's own spread instead of the budget, which is looser for those models. |
|
|
150
150
|
| Speed | One request at a time: `crit-eval` measures single-stream decode, prefill and time to first token, not throughput under concurrent requests or batching. A noisy sampled decode is repeated at up to nine seeds per case, so an engine whose speed swings with the drawn text takes longer to measure (`--no-adaptive-sampled` keeps three, an override the result records). |
|
|
151
151
|
| Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded, but no limit uses them yet, so engines that offload weights or compute on the CPU get GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
|
|
@@ -7,7 +7,7 @@ from pathlib import Path
|
|
|
7
7
|
# The package holds its suite, protocol, probes and examples; a checkout of crit-eval is its parent.
|
|
8
8
|
EVAL_DIR = Path(__file__).resolve().parent
|
|
9
9
|
SUITE_DIR = EVAL_DIR / "suite"
|
|
10
|
-
__version__ = "0.
|
|
10
|
+
__version__ = "0.2.0"
|
|
11
11
|
CONTRACT = "eval-v1"
|
|
12
12
|
PROTOCOL_ID = re.compile(r"crit-eval-v(\d+)\.(\d+)\.(\d+)")
|
|
13
13
|
# The ids before crit-eval had its own repository (crit-eval-v4, crit-eval-v4.1, ... crit-eval-v8): major and minor.
|
|
@@ -49,7 +49,7 @@ def checkouts():
|
|
|
49
49
|
|
|
50
50
|
def machine_settings():
|
|
51
51
|
"""Memory thresholds from settings.toml (vram_headroom_gib, context_reserve_gib), the wanted context
|
|
52
|
-
(context_tokens) and the reference package source (eval_packages, crit_eval.packages): crit-eval's checkout's,
|
|
52
|
+
(context_tokens) and the reference package source (eval_packages, crit_eval.quality.packages): crit-eval's checkout's,
|
|
53
53
|
else the crit checkout's (checkouts); without one crit-eval uses the protocol's defaults and
|
|
54
54
|
DEFAULT_CONTEXT_TOKENS."""
|
|
55
55
|
path = next((root / "settings.toml" for root in checkouts() if (root / "settings.toml").is_file()), None)
|
|
@@ -16,8 +16,8 @@ whole blocks of its format, so the rows a lookup needs are read and decoded on t
|
|
|
16
16
|
|
|
17
17
|
An architecture is read only when LAYOUTS has its rules; each rule set is the inverse of the converter class for
|
|
18
18
|
that architecture and is proven by a round trip (tests/test_eval_gguf.py: a model converted by the reference
|
|
19
|
-
converter comes back tensor for tensor; DeepSeek V4, whose rules are crit_eval.gguf_deepseek4, in
|
|
20
|
-
tests/test_eval_gguf_deepseek4.py; Qwen3.8-Flash-Next, whose rules are crit_eval.gguf_qwen4exp, in
|
|
19
|
+
converter comes back tensor for tensor; DeepSeek V4, whose rules are crit_eval.architectures.gguf_deepseek4, in
|
|
20
|
+
tests/test_eval_gguf_deepseek4.py; Qwen3.8-Flash-Next, whose rules are crit_eval.architectures.gguf_qwen4exp, in
|
|
21
21
|
tests/test_eval_gguf_qwen4exp.py). A Llama-architecture GGUF is assumed converted from the Hugging Face
|
|
22
22
|
layout (Q and K permuted); one converted from Mistral's native format is not. Requires numpy, torch and gguf.
|
|
23
23
|
"""
|
|
@@ -190,7 +190,7 @@ class GgufCheckpoint:
|
|
|
190
190
|
architecture = gguf.architecture
|
|
191
191
|
rules = LAYOUTS.get(architecture)
|
|
192
192
|
if rules is None:
|
|
193
|
-
raise GgufLayoutError(f"GGUF architecture {architecture!r} has no layout rules in crit_eval.gguf_layout "
|
|
193
|
+
raise GgufLayoutError(f"GGUF architecture {architecture!r} has no layout rules in crit_eval.architectures.gguf_layout "
|
|
194
194
|
f"(supported: {', '.join(sorted(LAYOUTS))})")
|
|
195
195
|
enums = {name: arch for arch, name in gguf_py.MODEL_ARCH_NAMES.items()}
|
|
196
196
|
if architecture not in enums:
|
|
@@ -257,10 +257,10 @@ class GgufCheckpoint:
|
|
|
257
257
|
|
|
258
258
|
def rows(self, gguf_name, index):
|
|
259
259
|
"""Rows `index` of 2-D GGUF tensor `gguf_name` in FP32: each distinct row read once, with a read of exactly
|
|
260
|
-
its bytes (consecutive rows in one read) by
|
|
260
|
+
its bytes (consecutive rows in one read) by checkpoints._read_spans' parallel readers, and decoded alone."""
|
|
261
261
|
import numpy as np
|
|
262
262
|
import torch
|
|
263
|
-
from .
|
|
263
|
+
from ..quality.checkpoints import _read_spans
|
|
264
264
|
tensor = self.gguf.tensors[gguf_name]
|
|
265
265
|
count, columns = tensor.shape
|
|
266
266
|
index = index.reshape(-1)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"""The reference converter's qwen4exp layout undone (Qwen3.8-Flash-Next; transformers' qwen4_exp_text), for
|
|
2
|
-
crit_eval.gguf_layout.
|
|
2
|
+
crit_eval.architectures.gguf_layout.
|
|
3
3
|
|
|
4
4
|
The converter (conversion/qwen4exp.py, Qwen4ExpTextModel) runs Qwen3.5's gated-delta-net steps (GatedDeltaNetLayout:
|
|
5
5
|
value heads tiled, A_log as -exp(A_log), dt_bias renamed, convolution kernels squeezed, zero-centred norms stored as
|
|
@@ -7,7 +7,21 @@ import sys
|
|
|
7
7
|
# runner.PARTS, kept here so parsing imports nothing else.
|
|
8
8
|
PARTS = ("speed", "quality", "exactness", "context", "heldout", "history", "followup")
|
|
9
9
|
CHAT_TEMPLATE_HELP = ("Chat template (jinja) to render the prompts with, for a model that carries none (default: the "
|
|
10
|
-
"one crit ships for
|
|
10
|
+
"one crit ships for the model's family); it becomes part of the model's identity")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def json_option(text, option):
|
|
14
|
+
"""An option's JSON text, else the file holding it."""
|
|
15
|
+
try:
|
|
16
|
+
return json.loads(text)
|
|
17
|
+
except json.JSONDecodeError as error:
|
|
18
|
+
path = Path(text)
|
|
19
|
+
if not path.is_file():
|
|
20
|
+
raise SystemExit(f"{option} is neither JSON ({error}) nor a file: {text}")
|
|
21
|
+
try:
|
|
22
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
23
|
+
except json.JSONDecodeError as error:
|
|
24
|
+
raise SystemExit(f"{option} file {path} is not JSON: {error}")
|
|
11
25
|
|
|
12
26
|
|
|
13
27
|
def main(argv=None):
|
|
@@ -25,7 +39,7 @@ def main(argv=None):
|
|
|
25
39
|
run.add_argument("--model", required=True, help="Model directory passed to the adapter")
|
|
26
40
|
run.add_argument("--data", required=True, help="Prepared suite directory for this model")
|
|
27
41
|
run.add_argument("--out", required=True, help="Result JSON path")
|
|
28
|
-
run.add_argument("--config", default="{}", help="JSON configuration overrides
|
|
42
|
+
run.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
|
|
29
43
|
run.add_argument("--only", choices=list(PARTS), action="append",
|
|
30
44
|
help="Run only these parts (repeatable); default is all. context: the long-context curve and "
|
|
31
45
|
"retrieval up to the required context and the speed cases again at a lower context; "
|
|
@@ -143,14 +157,14 @@ def main(argv=None):
|
|
|
143
157
|
pin.add_argument("--adapter", required=True, help="Python file exporting create_eval_engine(context)")
|
|
144
158
|
pin.add_argument("--model", required=True, help="Model directory passed to the adapter")
|
|
145
159
|
pin.add_argument("--data", required=True, help="Prepared data directory to pin the continuation in")
|
|
146
|
-
pin.add_argument("--config", default="{}", help="JSON configuration overrides
|
|
160
|
+
pin.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
|
|
147
161
|
pin.add_argument("--context", type=int, metavar="TOKENS", help="Context the engine is built for (the run's kv_tokens)")
|
|
148
162
|
|
|
149
163
|
ab = subs.add_parser("ab", help="Alternate two engines' speed runs in one session and report paired ratios")
|
|
150
164
|
ab.add_argument("--adapter", required=True, help="Adapter of engine A")
|
|
151
|
-
ab.add_argument("--config", default="{}", help="JSON configuration of engine A")
|
|
165
|
+
ab.add_argument("--config", default="{}", help="JSON configuration of engine A (text or a file)")
|
|
152
166
|
ab.add_argument("--adapter-b", help="Adapter of engine B (default: engine A's adapter)")
|
|
153
|
-
ab.add_argument("--config-b", default="{}", help="JSON configuration of engine B")
|
|
167
|
+
ab.add_argument("--config-b", default="{}", help="JSON configuration of engine B (text or a file)")
|
|
154
168
|
ab.add_argument("--model", required=True)
|
|
155
169
|
ab.add_argument("--data", required=True)
|
|
156
170
|
ab.add_argument("--out", required=True, help="Result JSON path")
|
|
@@ -165,7 +179,7 @@ def main(argv=None):
|
|
|
165
179
|
kernels.add_argument("--model", required=True, help="Model directory passed to the adapter")
|
|
166
180
|
kernels.add_argument("--data", required=True, help="Prepared suite directory for this model")
|
|
167
181
|
kernels.add_argument("--out", required=True, help="Kernel report JSON path")
|
|
168
|
-
kernels.add_argument("--config", default="{}", help="JSON configuration overrides
|
|
182
|
+
kernels.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
|
|
169
183
|
kernels.add_argument("--case", action="append", help="Trace only these speed cases (repeatable); default all")
|
|
170
184
|
kernels.add_argument("--device", metavar="JSON", help="Device ceilings from `crit eval probe` for share of peak")
|
|
171
185
|
kernels.add_argument("--nvcc", help="nvcc of a CUDA toolkit that ships CUPTI")
|
|
@@ -192,24 +206,20 @@ ENGINE_COMMANDS = {"run", "pin", "ab", "kernels", "reference"}
|
|
|
192
206
|
def _command(args):
|
|
193
207
|
if args.command == "kernels":
|
|
194
208
|
from . import load_json as load
|
|
195
|
-
from .kernels import render as render_kernels, trace
|
|
196
|
-
from .prepare import PREPARED
|
|
209
|
+
from .limits.kernels import render as render_kernels, trace
|
|
210
|
+
from .quality.prepare import PREPARED
|
|
197
211
|
if not (Path(args.data) / PREPARED).is_file():
|
|
198
|
-
from .prepare import prepare as do_prepare
|
|
212
|
+
from .quality.prepare import prepare as do_prepare
|
|
199
213
|
do_prepare(args.model, args.data)
|
|
200
|
-
work = None
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
work = json.loads(args.work)
|
|
204
|
-
except json.JSONDecodeError:
|
|
205
|
-
work = json.loads(Path(args.work).read_text(encoding="utf-8"))
|
|
206
|
-
result = trace(args.adapter, args.model, args.data, args.out, json.loads(args.config), args.case, args.nvcc,
|
|
214
|
+
work = json_option(args.work, "--work") if args.work else None
|
|
215
|
+
result = trace(args.adapter, args.model, args.data, args.out, json_option(args.config, "--config"), args.case,
|
|
216
|
+
args.nvcc,
|
|
207
217
|
load(args.device) if args.device else None, context_tokens=args.context, work=work)
|
|
208
218
|
print(f"wrote {args.out}")
|
|
209
219
|
print(render_kernels(result, args.top))
|
|
210
220
|
return 0
|
|
211
221
|
if args.command == "prepare":
|
|
212
|
-
from .prepare import prepare as do_prepare
|
|
222
|
+
from .quality.prepare import prepare as do_prepare
|
|
213
223
|
try:
|
|
214
224
|
prepared = do_prepare(args.model, args.data, sampling=args.sampling, chat_template=args.chat_template)
|
|
215
225
|
except ValueError as error:
|
|
@@ -223,8 +233,8 @@ def _command(args):
|
|
|
223
233
|
print(f"quality: score them once on the checkpoint: crit eval reference --model {args.model} --data {args.data}")
|
|
224
234
|
return 0
|
|
225
235
|
if args.command == "reference":
|
|
226
|
-
from .reference import score_data
|
|
227
|
-
from .fixed import STORES
|
|
236
|
+
from .quality.reference import score_data
|
|
237
|
+
from .quality.fixed import STORES
|
|
228
238
|
how = score_data(args.model, args.data, log=lambda line: print(line, flush=True), quality_mode=args.quality_mode)
|
|
229
239
|
print(f"reference scores of {how['positions']:,} positions {how['method']} in "
|
|
230
240
|
f"{Path(args.data) / STORES[args.quality_mode]}"
|
|
@@ -232,8 +242,8 @@ def _command(args):
|
|
|
232
242
|
+ (f"; the reference's own spread on the long texts {how['spread']}" if how.get("spread") else ""))
|
|
233
243
|
return 0
|
|
234
244
|
if args.command == "fetch":
|
|
235
|
-
from .packages import fetch as do_fetch, source
|
|
236
|
-
from .prepare import read_template
|
|
245
|
+
from .quality.packages import fetch as do_fetch, source
|
|
246
|
+
from .quality.prepare import read_template
|
|
237
247
|
where = source(args.packages)
|
|
238
248
|
if where is None:
|
|
239
249
|
raise SystemExit("crit eval fetch: name a package source (--packages, or eval_packages in settings.toml)")
|
|
@@ -247,41 +257,43 @@ def _command(args):
|
|
|
247
257
|
return 1
|
|
248
258
|
return 0
|
|
249
259
|
if args.command == "pack":
|
|
250
|
-
from .packages import pack as do_pack
|
|
260
|
+
from .quality.packages import pack as do_pack
|
|
251
261
|
entry = do_pack(args.data, args.packages)
|
|
252
262
|
print(f"packed {entry['name']} ({entry['bytes'] / 2**20:.1f} MiB) into {args.packages}")
|
|
253
263
|
return 0
|
|
254
264
|
if args.command == "context":
|
|
255
265
|
from . import load_json as load
|
|
256
|
-
from .context import GIB, max_context, render as render_context
|
|
266
|
+
from .limits.context import GIB, max_context, render as render_context
|
|
257
267
|
from . import EVAL_DIR
|
|
258
268
|
settings = context_settings(load(EVAL_DIR / "protocol.json")["context"], args)
|
|
259
269
|
value = max_context(args.model, load(args.device), headroom_gib(args) * GIB, settings, requested(args))
|
|
260
270
|
print(json.dumps(value, indent=1) if args.json else render_context(value))
|
|
261
271
|
return 0 if value["tokens"] else 1
|
|
262
272
|
if args.command == "compare":
|
|
263
|
-
from .
|
|
273
|
+
from .reports.compare import compare as do_compare
|
|
264
274
|
print(do_compare(args.results))
|
|
265
275
|
return 0
|
|
266
276
|
if args.command == "pin":
|
|
267
|
-
from .pin import CONTINUATIONS, pin as do_pin
|
|
268
|
-
from .prepare import PREPARED
|
|
277
|
+
from .run.pin import CONTINUATIONS, pin as do_pin
|
|
278
|
+
from .quality.prepare import PREPARED
|
|
269
279
|
if not (Path(args.data) / PREPARED).is_file():
|
|
270
|
-
from .prepare import prepare as do_prepare
|
|
280
|
+
from .quality.prepare import prepare as do_prepare
|
|
271
281
|
do_prepare(args.model, args.data)
|
|
272
|
-
pinned = do_pin(args.adapter, args.model, args.data,
|
|
282
|
+
pinned = do_pin(args.adapter, args.model, args.data, json_option(args.config, "--config"),
|
|
283
|
+
context_tokens=args.context)
|
|
273
284
|
print(f"pinned {len(pinned['cases'])} speed cases in {Path(args.data) / CONTINUATIONS} (sha256 {pinned['sha256'][:12]})")
|
|
274
285
|
return 0
|
|
275
286
|
if args.command == "ab":
|
|
276
|
-
from .ab import render as render_ab, run_ab
|
|
277
|
-
result = run_ab(args.adapter,
|
|
287
|
+
from .run.ab import render as render_ab, run_ab
|
|
288
|
+
result = run_ab(args.adapter, json_option(args.config, "--config"), args.adapter_b or args.adapter,
|
|
289
|
+
json_option(args.config_b, "--config-b"),
|
|
278
290
|
args.model, args.data, args.out, args.rounds, args.warmups, args.repetitions,
|
|
279
291
|
context_tokens=args.context)
|
|
280
292
|
print(f"wrote {args.out}")
|
|
281
293
|
print(render_ab(result))
|
|
282
294
|
return 0
|
|
283
295
|
if args.command == "probe":
|
|
284
|
-
from .probe import probe as do_probe
|
|
296
|
+
from .limits.probe import probe as do_probe
|
|
285
297
|
device = do_probe(args.out, args.nvcc, args.repeats)
|
|
286
298
|
print(f"wrote {args.out}: {device['bandwidth_gbs']:.0f} GB/s, "
|
|
287
299
|
+ ", ".join(f"{k} {v:.0f} TFLOPS" for k, v in device["tflops"].items())
|
|
@@ -291,20 +303,20 @@ def _command(args):
|
|
|
291
303
|
return 0
|
|
292
304
|
if args.command == "status":
|
|
293
305
|
from . import load_json as load
|
|
294
|
-
from .status import render, status_of
|
|
306
|
+
from .reports.status import render, status_of
|
|
295
307
|
from . import EVAL_DIR
|
|
296
308
|
protocol = load(EVAL_DIR / "protocol.json")
|
|
297
309
|
current = status_of(args.result, protocol, args.baseline, args.device, args.data, args.kernels)
|
|
298
310
|
print(json.dumps(current, indent=1) if args.json else render(current))
|
|
299
311
|
return 0
|
|
300
|
-
from .prepare import PREPARED, read_template
|
|
301
|
-
from .runner import run as do_run
|
|
312
|
+
from .quality.prepare import PREPARED, read_template
|
|
313
|
+
from .run.runner import run as do_run
|
|
302
314
|
parts = tuple(args.only or PARTS)
|
|
303
315
|
data = Path(args.data)
|
|
304
316
|
template = read_template(args.chat_template)
|
|
305
317
|
if not (data / PREPARED).is_file():
|
|
306
|
-
from .packages import fetch as do_fetch, source
|
|
307
|
-
from .prepare import prepare as do_prepare
|
|
318
|
+
from .quality.packages import fetch as do_fetch, source
|
|
319
|
+
from .quality.prepare import prepare as do_prepare
|
|
308
320
|
try:
|
|
309
321
|
# A published package brings the reference scores too; otherwise prepare here.
|
|
310
322
|
if source() is None or do_fetch(args.model, data, source(), chat_template=template) is None:
|
|
@@ -312,12 +324,13 @@ def _command(args):
|
|
|
312
324
|
except ValueError as error: # no chat template to render with, or one for a model that carries its own
|
|
313
325
|
raise SystemExit(f"crit eval run: {error}")
|
|
314
326
|
from . import load_json as load
|
|
315
|
-
from .status import render
|
|
327
|
+
from .reports.status import render
|
|
316
328
|
device = load(args.device) if args.device else None
|
|
317
329
|
baseline = load(args.baseline) if args.baseline else None
|
|
318
|
-
result = do_run(args.adapter, args.model, data, args.out,
|
|
330
|
+
result = do_run(args.adapter, args.model, data, args.out, json_option(args.config, "--config"), parts,
|
|
319
331
|
args.warmups, args.repetitions, args.label, bandwidth_gbs=args.bandwidth, device=device,
|
|
320
|
-
baseline=baseline, quality_budget=
|
|
332
|
+
baseline=baseline, quality_budget=(json_option(args.quality_budget, "--quality-budget")
|
|
333
|
+
if args.quality_budget else None),
|
|
321
334
|
exactness=args.exactness, context_tokens=args.context, requested_context=requested(args),
|
|
322
335
|
vram_headroom_bytes=round(headroom_gib(args) * (1 << 30)),
|
|
323
336
|
context_reserve_bytes=round(reserve_gib(args) * (1 << 30)) if reserve_gib(args) is not None else None,
|
|
@@ -9,7 +9,7 @@ Prefill is compute bound. Its floor for a prompt of N tokens is
|
|
|
9
9
|
recurrent N x linear-attention FLOPs per token / the BF16 rate (chunked form, approximate)
|
|
10
10
|
|
|
11
11
|
A projection weight of R x K elements costs 2RK FLOPs per token at the rate of the format it is stored in
|
|
12
|
-
(crit_eval.schemes): BF16/F16/F32 (bf16), FP8 (fp8), NVFP4 (nvfp4), weight-only integer codes (AWQ, GPTQ,
|
|
12
|
+
(crit_eval.limits.schemes): BF16/F16/F32 (bf16), FP8 (fp8), NVFP4 (nvfp4), weight-only integer codes (AWQ, GPTQ,
|
|
13
13
|
MLX, bitsandbytes NF4, compressed-tensors pack-quantized: bf16, as engines widen them for the GEMM) and 8-bit
|
|
14
14
|
integer weights with per-row scales (bitsandbytes LLM.int8, compressed-tensors int-quantized: fp8, the int8 rate).
|
|
15
15
|
A GGUF's ggml block types
|
|
@@ -18,7 +18,7 @@ the same rate), its MXFP4 and NVFP4 at the nvfp4 rate and its F32/F16/BF16 at bf
|
|
|
18
18
|
(gathered), vision towers, norms, convolutions and the output head (one row of logits per
|
|
19
19
|
prompt) are left out. Engines may compute in other formats; the floor uses the stored ones, and
|
|
20
20
|
prefill_floor_ms prices the quantized matrices with other prompt activations on request (ACTIVATIONS:
|
|
21
|
-
BF16, FP16 accumulation, FP8), as crit_eval.headroom.activation_ceilings reports them.
|
|
21
|
+
BF16, FP16 accumulation, FP8), as crit_eval.limits.headroom.activation_ceilings reports them.
|
|
22
22
|
|
|
23
23
|
Time to first token cannot beat either floor: the prompt's arithmetic, or reading the weights
|
|
24
24
|
once (language-model tensors and the output head, not embeddings, vision or draft layers)
|
|
@@ -68,7 +68,7 @@ def _read(role):
|
|
|
68
68
|
|
|
69
69
|
|
|
70
70
|
def weight_bytes(tensors, file_name="", draft_blocks=()):
|
|
71
|
-
"""Bytes of the tensors one forward pass reads (crit_eval.tensors: layers, experts and the output head, not
|
|
71
|
+
"""Bytes of the tensors one forward pass reads (crit_eval.limits.tensors: layers, experts and the output head, not
|
|
72
72
|
embeddings, other modalities or draft layers). tensors: {name: crit_core.model.TensorInfo} of one file."""
|
|
73
73
|
return sum(t.nbytes for name, t in tensors.items() if _read(tensor_role(name, file_name, draft_blocks)))
|
|
74
74
|
|
|
@@ -87,7 +87,7 @@ def matmul_flops(tensors, file_name="", draft_blocks=(), quantization=None):
|
|
|
87
87
|
{name: TensorInfo}. The output head is left out (one row of logits per prompt). A 3-D expert tensor counts as
|
|
88
88
|
one matrix per expert, as the per-expert matrices of a checkpoint that stores them apart do. quantization: the
|
|
89
89
|
config's quantization_config (or MLX's quantization), which names the scheme of packed integer weights
|
|
90
|
-
(crit_eval.schemes)."""
|
|
90
|
+
(crit_eval.limits.schemes)."""
|
|
91
91
|
flops, matrices, seen = {}, {}, set()
|
|
92
92
|
for name, tensor in tensors.items():
|
|
93
93
|
role = tensor_role(name, file_name, draft_blocks)
|
|
@@ -166,7 +166,7 @@ def best_rates(device):
|
|
|
166
166
|
return rates, sources
|
|
167
167
|
|
|
168
168
|
|
|
169
|
-
# The activation formats a quantized matrix's prompt GEMM can take, and the rates (crit_eval.probe keys) each runs at
|
|
169
|
+
# The activation formats a quantized matrix's prompt GEMM can take, and the rates (crit_eval.limits.probe keys) each runs at
|
|
170
170
|
# for each stored weight format, the best available one counting: BF16 activations on BF16 tensor cores with FP32
|
|
171
171
|
# accumulation, FP16 activations accumulating in FP16, FP8 activations (FP4 weights times FP8 through kind::f8f6f4, or
|
|
172
172
|
# widened to FP8), and the stored format itself (W8A8, W4A4). Weights stored in BF16 stay at the BF16 rate.
|
|
@@ -32,7 +32,7 @@ from .ceiling import model_config, weight_files
|
|
|
32
32
|
from .tensors import tensor_role
|
|
33
33
|
|
|
34
34
|
GIB = 1 << 30
|
|
35
|
-
DRAFT_HEAD = re.compile(r"(^|\.)head\.weight$") # a draft layer's own output head (
|
|
35
|
+
DRAFT_HEAD = re.compile(r"(^|\.)head\.weight$") # a draft layer's own output head (shared_head.head)
|
|
36
36
|
|
|
37
37
|
|
|
38
38
|
def weight_split(model, config):
|
|
@@ -64,7 +64,7 @@ crit-eval does not judge or classify speed. Noise is the spread of each score ac
|
|
|
64
64
|
repetitions.
|
|
65
65
|
"""
|
|
66
66
|
from . import ceiling as model_ceiling
|
|
67
|
-
from
|
|
67
|
+
from ..reports import metrics
|
|
68
68
|
|
|
69
69
|
|
|
70
70
|
def decode_suite(result):
|
|
@@ -586,7 +586,7 @@ def trace_decode_parts(result, trace):
|
|
|
586
586
|
|
|
587
587
|
|
|
588
588
|
def price_prompts(result, rates, sources, prefill_cases, evidence=()):
|
|
589
|
-
"""A result's headroom with prefill and time to first token priced at `rates` (crit_eval.ceiling.best_rates of a
|
|
589
|
+
"""A result's headroom with prefill and time to first token priced at `rates` (crit_eval.limits.ceiling.best_rates of a
|
|
590
590
|
device) on the model arithmetic it records (result ceiling): the stored-format prefill and TTFT summaries, the
|
|
591
591
|
ceilings by prompt activation format (activation_ceilings, judged with this result's and `evidence`'s quality),
|
|
592
592
|
and their gaps in place of the old ones. The run prices them so; `crit eval status --device/--data` prices a
|
|
@@ -27,9 +27,9 @@ import sys
|
|
|
27
27
|
import tempfile
|
|
28
28
|
import time
|
|
29
29
|
|
|
30
|
-
from
|
|
30
|
+
from .. import CONTRACT, EVAL_DIR, load_json, portable_path, write_json
|
|
31
31
|
from .ceiling import best_rates
|
|
32
|
-
from .prepare import PREPARED
|
|
32
|
+
from ..quality.prepare import PREPARED
|
|
33
33
|
|
|
34
34
|
SOURCE = EVAL_DIR / "probe" / "kernel_trace.cpp"
|
|
35
35
|
TOP = 12
|
|
@@ -115,7 +115,7 @@ def _timed(engine, ids, n, now=time.time_ns):
|
|
|
115
115
|
|
|
116
116
|
|
|
117
117
|
def _child(spec):
|
|
118
|
-
from .
|
|
118
|
+
from ..run.adapters import load_adapter, open_engine
|
|
119
119
|
prepared = load_json(Path(spec["data"]) / PREPARED)
|
|
120
120
|
module = load_adapter(spec["adapter"])
|
|
121
121
|
engine, info = open_engine(module, spec["model"], spec["config"], speculative=True,
|
|
@@ -223,7 +223,7 @@ def _work_rule(rules, name):
|
|
|
223
223
|
return None
|
|
224
224
|
|
|
225
225
|
|
|
226
|
-
# The terms of the prefill floor a kernel's time joins (crit_eval.ceiling), and draft work kept apart from them.
|
|
226
|
+
# The terms of the prefill floor a kernel's time joins (crit_eval.limits.ceiling), and draft work kept apart from them.
|
|
227
227
|
TERMS = ("matmul", "attention", "recurrent", "other", "draft")
|
|
228
228
|
# Name heuristics for kernels no kernel_work rule classes, checked in order on the lower-cased demangled name.
|
|
229
229
|
# Weight-only GEMMs go by their scheme (w4a16, marlin) and crit's stored-tile projections by their namespace.
|
|
@@ -305,7 +305,7 @@ def analyze(kernels, device, marks, device_ceilings=None):
|
|
|
305
305
|
sorted(unclassed.items(), key=lambda x: -x[1])[:UNCLASSED]]}
|
|
306
306
|
if phase == "prefill" and "prefill_first" in case:
|
|
307
307
|
# The fresh request's time to its first token, and the GPU's busy time inside it: the rest is the
|
|
308
|
-
# engine's host work, launches and synchronization (crit_eval.status: the TTFT breakdown).
|
|
308
|
+
# engine's host work, launches and synchronization (crit_eval.reports.status: the TTFT breakdown).
|
|
309
309
|
first_ms = (case["prefill_first"] - lo) / 1e6
|
|
310
310
|
gpu = sum(ns for k, ns in inside if k[0] < case["prefill_first"]) / 1e6
|
|
311
311
|
cases[case["id"]][phase].update(first_token_ms=first_ms, gpu_before_first_ms=min(gpu, first_ms))
|
|
@@ -422,7 +422,7 @@ def trace(adapter, model, data, out, config=None, cases=None, nvcc=None, device=
|
|
|
422
422
|
env = dict(os.environ, CUDA_INJECTION64_PATH=str(library), CRIT_KTRACE_DIR=directory,
|
|
423
423
|
PYTHONPATH=os.pathsep.join(filter(None, paths)))
|
|
424
424
|
log(f"tracing kernels with {library.name} ({toolkit['nvcc']})")
|
|
425
|
-
done = subprocess.run([sys.executable, "-B", "-m", "crit_eval.kernels", "--child", json.dumps(spec)], env=env)
|
|
425
|
+
done = subprocess.run([sys.executable, "-B", "-m", "crit_eval.limits.kernels", "--child", json.dumps(spec)], env=env)
|
|
426
426
|
if done.returncode != 0:
|
|
427
427
|
raise SystemExit(f"traced engine process failed ({done.returncode})")
|
|
428
428
|
marks = load_json(Path(directory) / "marks.json")
|
|
@@ -5,7 +5,7 @@ keeps the median of each value: achievable copy bandwidth (GB/s, bytes read plus
|
|
|
5
5
|
the decode pass floor, dense cuBLASLt GEMM TFLOPS per weight format (bf16, fp8, nvfp4) and raw
|
|
6
6
|
tensor-core (mma.sync) TFLOPS per input and accumulator format (bf16, fp16, fp16_acc, fp8, nvfp4,
|
|
7
7
|
fp4_fp8) for the prefill floor, which takes the better of the two per format
|
|
8
|
-
(crit_eval.ceiling.best_rates). It also keeps the least free device memory any run saw, for the max context,
|
|
8
|
+
(crit_eval.limits.ceiling.best_rates). It also keeps the least free device memory any run saw, for the max context,
|
|
9
9
|
the host's physical and available memory, and host-to-device rates per transfer size (pinned
|
|
10
10
|
copy, pageable copy, mapped read) for engines that keep weights in host memory. Pass the file to `crit eval run --device
|
|
11
11
|
device.json`. Probe on the machine and in the environment the engine runs in, with the same
|
|
@@ -21,7 +21,8 @@ import subprocess
|
|
|
21
21
|
import sys
|
|
22
22
|
import tempfile
|
|
23
23
|
|
|
24
|
-
from
|
|
24
|
+
from .. import EVAL_DIR, portable_path, write_json
|
|
25
|
+
from ..reports import metrics
|
|
25
26
|
|
|
26
27
|
SOURCE = EVAL_DIR / "probe" / "device_probe.cu"
|
|
27
28
|
|