crit-eval 0.1.2__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. {crit_eval-0.1.2 → crit_eval-0.2.0}/PKG-INFO +3 -3
  2. {crit_eval-0.1.2 → crit_eval-0.2.0}/README.md +1 -1
  3. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/__init__.py +2 -2
  4. crit_eval-0.2.0/crit_eval/architectures/__init__.py +3 -0
  5. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/architectures}/gguf_layout.py +5 -5
  6. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/architectures}/gguf_qwen4exp.py +1 -1
  7. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/cli.py +52 -39
  8. crit_eval-0.2.0/crit_eval/limits/__init__.py +2 -0
  9. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/ceiling.py +5 -5
  10. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/context.py +1 -1
  11. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/headroom.py +2 -2
  12. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/kernels.py +6 -6
  13. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/probe.py +3 -2
  14. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/schemes.py +28 -190
  15. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/limits}/tensors.py +1 -1
  16. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/protocol.json +12 -4
  17. crit_eval-0.2.0/crit_eval/quality/__init__.py +2 -0
  18. crit_eval-0.2.0/crit_eval/quality/attention.py +79 -0
  19. crit_eval-0.2.0/crit_eval/quality/checkpoints.py +366 -0
  20. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/fixed.py +2 -2
  21. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/logprob.py +2 -2
  22. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/packages.py +24 -10
  23. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/prepare.py +6 -6
  24. crit_eval-0.2.0/crit_eval/quality/prompt_state.py +391 -0
  25. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/quality}/reference.py +36 -845
  26. crit_eval-0.2.0/crit_eval/reports/__init__.py +2 -0
  27. crit_eval-0.1.2/crit_eval/report.py → crit_eval-0.2.0/crit_eval/reports/compare.py +2 -2
  28. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/reports}/metrics.py +6 -0
  29. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/reports}/status.py +11 -9
  30. crit_eval-0.2.0/crit_eval/run/__init__.py +2 -0
  31. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/ab.py +5 -3
  32. crit_eval-0.2.0/crit_eval/run/adapters.py +96 -0
  33. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/chat.py +4 -4
  34. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/estimate.py +7 -7
  35. crit_eval-0.2.0/crit_eval/run/history.py +71 -0
  36. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/host.py +1 -1
  37. crit_eval-0.2.0/crit_eval/run/long_context.py +215 -0
  38. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/pin.py +4 -3
  39. crit_eval-0.2.0/crit_eval/run/quality_checks.py +319 -0
  40. crit_eval-0.2.0/crit_eval/run/runner.py +429 -0
  41. crit_eval-0.2.0/crit_eval/run/speed.py +322 -0
  42. crit_eval-0.2.0/crit_eval/run/timing.py +135 -0
  43. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/PKG-INFO +3 -3
  44. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/SOURCES.txt +42 -28
  45. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/requires.txt +1 -1
  46. {crit_eval-0.1.2 → crit_eval-0.2.0}/pyproject.toml +2 -2
  47. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval.py +137 -94
  48. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_chat_template.py +5 -2
  49. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_deadline.py +5 -4
  50. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_distributions.py +3 -1
  51. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_estimate.py +3 -3
  52. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_followup.py +7 -6
  53. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_formats.py +8 -8
  54. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_gguf.py +8 -7
  55. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_gguf_deepseek4.py +10 -10
  56. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_gguf_qwen4exp.py +3 -3
  57. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_indexers.py +2 -2
  58. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_kernels.py +3 -3
  59. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_packages.py +40 -10
  60. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_quality_modes.py +49 -13
  61. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_server.py +12 -3
  62. {crit_eval-0.1.2 → crit_eval-0.2.0}/tests/test_eval_tensors.py +2 -2
  63. crit_eval-0.1.2/crit_eval/runner.py +0 -1497
  64. {crit_eval-0.1.2 → crit_eval-0.2.0}/LICENSE +0 -0
  65. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/__main__.py +0 -0
  66. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/architectures}/gguf_deepseek4.py +0 -0
  67. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/architectures}/indexers.py +0 -0
  68. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/examples/mock_adapter.py +0 -0
  69. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/probe/device_probe.cu +0 -0
  70. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/probe/kernel_trace.cpp +0 -0
  71. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/probe/logprob_compare.cu +0 -0
  72. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/probe/logprob_compare.h +0 -0
  73. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/deadline.py +0 -0
  74. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/hostload.py +0 -0
  75. {crit_eval-0.1.2/crit_eval → crit_eval-0.2.0/crit_eval/run}/outcomes.py +0 -0
  76. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/README.md.txt +0 -0
  77. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/cli.py.txt +0 -0
  78. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/headroom.py.txt +0 -0
  79. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/metrics.py.txt +0 -0
  80. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/prepare.py.txt +0 -0
  81. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/reference.py.txt +0 -0
  82. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/report.py.txt +0 -0
  83. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/runner.py.txt +0 -0
  84. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/agent/tools.json +0 -0
  85. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/followup.json +0 -0
  86. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/heldout.json +0 -0
  87. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/long.json +0 -0
  88. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/long_document.txt +0 -0
  89. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/quality.json +0 -0
  90. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval/suite/speed.json +0 -0
  91. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/dependency_links.txt +0 -0
  92. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/entry_points.txt +0 -0
  93. {crit_eval-0.1.2 → crit_eval-0.2.0}/crit_eval.egg-info/top_level.txt +0 -0
  94. {crit_eval-0.1.2 → crit_eval-0.2.0}/setup.cfg +0 -0
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: crit-eval
3
- Version: 0.1.2
3
+ Version: 0.2.0
4
4
  Summary: crit-eval: one speed, quality and exactness suite that every inference engine runs the same way
5
5
  License-Expression: Apache-2.0
6
6
  Project-URL: Repository, https://github.com/crit-labs/crit-eval
7
7
  Requires-Python: >=3.11
8
8
  Description-Content-Type: text/markdown
9
9
  License-File: LICENSE
10
- Requires-Dist: crit-core<0.12,>=0.10.0
10
+ Requires-Dist: crit-core<0.13,>=0.12.0
11
11
  Provides-Extra: reference
12
12
  Requires-Dist: torch; extra == "reference"
13
13
  Requires-Dist: transformers>=5.20; extra == "reference"
@@ -162,7 +162,7 @@ plugs in through a Python [adapter](docs/ADAPTER.md) instead of `--url`.
162
162
  | --- | --- |
163
163
  | Quality reference | Quality is measured against the checkpoint as stored, for example an NVFP4 quantization, not against the full-precision model. `crit-eval` reports the drift the engine adds, never the quantization loss of the checkpoint itself, so quality results of two different quantizations of a model do not compare. |
164
164
  | What quality covers | Drift on fixed texts: six conversations (prose, code, math, tool calls and more) and one long text each at 1K, 4K and 32K tokens, as perplexity ratio and KL divergence. It is not a task benchmark: no accuracy on MMLU-style questions, coding tests or agent tasks. |
165
- | Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages exist for three checkpoints and are not published yet. |
165
+ | Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages for three checkpoints are published on Hugging Face ([crit-labs/crit-eval-references](https://huggingface.co/datasets/crit-labs/crit-eval-references)) and fetched by default. |
166
166
  | Rounding-sensitive models | Where a model amplifies rounding (MoE routing near-ties, sparse-attention ties, recurrent state), two valid FP32 runs of the reference disagree; quality is then judged against twice the reference's own spread instead of the budget, which is looser for those models. |
167
167
  | Speed | One request at a time: `crit-eval` measures single-stream decode, prefill and time to first token, not throughput under concurrent requests or batching. A noisy sampled decode is repeated at up to nine seeds per case, so an engine whose speed swings with the drawn text takes longer to measure (`--no-adaptive-sampled` keeps three, an override the result records). |
168
168
  | Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded, but no limit uses them yet, so engines that offload weights or compute on the CPU get GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
@@ -145,7 +145,7 @@ plugs in through a Python [adapter](docs/ADAPTER.md) instead of `--url`.
145
145
  | --- | --- |
146
146
  | Quality reference | Quality is measured against the checkpoint as stored, for example an NVFP4 quantization, not against the full-precision model. `crit-eval` reports the drift the engine adds, never the quantization loss of the checkpoint itself, so quality results of two different quantizations of a model do not compare. |
147
147
  | What quality covers | Drift on fixed texts: six conversations (prose, code, math, tool calls and more) and one long text each at 1K, 4K and 32K tokens, as perplexity ratio and KL divergence. It is not a task benchmark: no accuracy on MMLU-style questions, coding tests or agent tasks. |
148
- | Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages exist for three checkpoints and are not published yet. |
148
+ | Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages for three checkpoints are published on Hugging Face ([crit-labs/crit-eval-references](https://huggingface.co/datasets/crit-labs/crit-eval-references)) and fetched by default. |
149
149
  | Rounding-sensitive models | Where a model amplifies rounding (MoE routing near-ties, sparse-attention ties, recurrent state), two valid FP32 runs of the reference disagree; quality is then judged against twice the reference's own spread instead of the budget, which is looser for those models. |
150
150
  | Speed | One request at a time: `crit-eval` measures single-stream decode, prefill and time to first token, not throughput under concurrent requests or batching. A noisy sampled decode is repeated at up to nine seeds per case, so an engine whose speed swings with the drawn text takes longer to measure (`--no-adaptive-sampled` keeps three, an override the result records). |
151
151
  | Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded, but no limit uses them yet, so engines that offload weights or compute on the CPU get GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
@@ -7,7 +7,7 @@ from pathlib import Path
7
7
  # The package holds its suite, protocol, probes and examples; a checkout of crit-eval is its parent.
8
8
  EVAL_DIR = Path(__file__).resolve().parent
9
9
  SUITE_DIR = EVAL_DIR / "suite"
10
- __version__ = "0.1.2"
10
+ __version__ = "0.2.0"
11
11
  CONTRACT = "eval-v1"
12
12
  PROTOCOL_ID = re.compile(r"crit-eval-v(\d+)\.(\d+)\.(\d+)")
13
13
  # The ids before crit-eval had its own repository (crit-eval-v4, crit-eval-v4.1, ... crit-eval-v8): major and minor.
@@ -49,7 +49,7 @@ def checkouts():
49
49
 
50
50
  def machine_settings():
51
51
  """Memory thresholds from settings.toml (vram_headroom_gib, context_reserve_gib), the wanted context
52
- (context_tokens) and the reference package source (eval_packages, crit_eval.packages): crit-eval's checkout's,
52
+ (context_tokens) and the reference package source (eval_packages, crit_eval.quality.packages): crit-eval's checkout's,
53
53
  else the crit checkout's (checkouts); without one crit-eval uses the protocol's defaults and
54
54
  DEFAULT_CONTEXT_TOKENS."""
55
55
  path = next((root / "settings.toml" for root in checkouts() if (root / "settings.toml").is_file()), None)
@@ -0,0 +1,3 @@
1
+ """Architecture-specific code the reference needs, one place for all of it: GGUF checkpoints read in the layout
2
+ transformers' modeling code expects, per architecture, and faster forms of modeling code that would otherwise run per
3
+ token in Python."""
@@ -16,8 +16,8 @@ whole blocks of its format, so the rows a lookup needs are read and decoded on t
16
16
 
17
17
  An architecture is read only when LAYOUTS has its rules; each rule set is the inverse of the converter class for
18
18
  that architecture and is proven by a round trip (tests/test_eval_gguf.py: a model converted by the reference
19
- converter comes back tensor for tensor; DeepSeek V4, whose rules are crit_eval.gguf_deepseek4, in
20
- tests/test_eval_gguf_deepseek4.py; Qwen3.8-Flash-Next, whose rules are crit_eval.gguf_qwen4exp, in
19
+ converter comes back tensor for tensor; DeepSeek V4, whose rules are crit_eval.architectures.gguf_deepseek4, in
20
+ tests/test_eval_gguf_deepseek4.py; Qwen3.8-Flash-Next, whose rules are crit_eval.architectures.gguf_qwen4exp, in
21
21
  tests/test_eval_gguf_qwen4exp.py). A Llama-architecture GGUF is assumed converted from the Hugging Face
22
22
  layout (Q and K permuted); one converted from Mistral's native format is not. Requires numpy, torch and gguf.
23
23
  """
@@ -190,7 +190,7 @@ class GgufCheckpoint:
190
190
  architecture = gguf.architecture
191
191
  rules = LAYOUTS.get(architecture)
192
192
  if rules is None:
193
- raise GgufLayoutError(f"GGUF architecture {architecture!r} has no layout rules in crit_eval.gguf_layout "
193
+ raise GgufLayoutError(f"GGUF architecture {architecture!r} has no layout rules in crit_eval.architectures.gguf_layout "
194
194
  f"(supported: {', '.join(sorted(LAYOUTS))})")
195
195
  enums = {name: arch for arch, name in gguf_py.MODEL_ARCH_NAMES.items()}
196
196
  if architecture not in enums:
@@ -257,10 +257,10 @@ class GgufCheckpoint:
257
257
 
258
258
  def rows(self, gguf_name, index):
259
259
  """Rows `index` of 2-D GGUF tensor `gguf_name` in FP32: each distinct row read once, with a read of exactly
260
- its bytes (consecutive rows in one read) by reference._read_spans' parallel readers, and decoded alone."""
260
+ its bytes (consecutive rows in one read) by checkpoints._read_spans' parallel readers, and decoded alone."""
261
261
  import numpy as np
262
262
  import torch
263
- from .reference import _read_spans
263
+ from ..quality.checkpoints import _read_spans
264
264
  tensor = self.gguf.tensors[gguf_name]
265
265
  count, columns = tensor.shape
266
266
  index = index.reshape(-1)
@@ -1,5 +1,5 @@
1
1
  """The reference converter's qwen4exp layout undone (Qwen3.8-Flash-Next; transformers' qwen4_exp_text), for
2
- crit_eval.gguf_layout.
2
+ crit_eval.architectures.gguf_layout.
3
3
 
4
4
  The converter (conversion/qwen4exp.py, Qwen4ExpTextModel) runs Qwen3.5's gated-delta-net steps (GatedDeltaNetLayout:
5
5
  value heads tiled, A_log as -exp(A_log), dt_bias renamed, convolution kernels squeezed, zero-centred norms stored as
@@ -7,7 +7,21 @@ import sys
7
7
  # runner.PARTS, kept here so parsing imports nothing else.
8
8
  PARTS = ("speed", "quality", "exactness", "context", "heldout", "history", "followup")
9
9
  CHAT_TEMPLATE_HELP = ("Chat template (jinja) to render the prompts with, for a model that carries none (default: the "
10
- "one crit ships for its family, as for DeepSeek V4); it becomes part of the model's identity")
10
+ "one crit ships for the model's family); it becomes part of the model's identity")
11
+
12
+
13
+ def json_option(text, option):
14
+ """An option's JSON text, else the file holding it."""
15
+ try:
16
+ return json.loads(text)
17
+ except json.JSONDecodeError as error:
18
+ path = Path(text)
19
+ if not path.is_file():
20
+ raise SystemExit(f"{option} is neither JSON ({error}) nor a file: {text}")
21
+ try:
22
+ return json.loads(path.read_text(encoding="utf-8"))
23
+ except json.JSONDecodeError as error:
24
+ raise SystemExit(f"{option} file {path} is not JSON: {error}")
11
25
 
12
26
 
13
27
  def main(argv=None):
@@ -25,7 +39,7 @@ def main(argv=None):
25
39
  run.add_argument("--model", required=True, help="Model directory passed to the adapter")
26
40
  run.add_argument("--data", required=True, help="Prepared suite directory for this model")
27
41
  run.add_argument("--out", required=True, help="Result JSON path")
28
- run.add_argument("--config", default="{}", help="JSON configuration overrides passed to the adapter")
42
+ run.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
29
43
  run.add_argument("--only", choices=list(PARTS), action="append",
30
44
  help="Run only these parts (repeatable); default is all. context: the long-context curve and "
31
45
  "retrieval up to the required context and the speed cases again at a lower context; "
@@ -143,14 +157,14 @@ def main(argv=None):
143
157
  pin.add_argument("--adapter", required=True, help="Python file exporting create_eval_engine(context)")
144
158
  pin.add_argument("--model", required=True, help="Model directory passed to the adapter")
145
159
  pin.add_argument("--data", required=True, help="Prepared data directory to pin the continuation in")
146
- pin.add_argument("--config", default="{}", help="JSON configuration overrides passed to the adapter")
160
+ pin.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
147
161
  pin.add_argument("--context", type=int, metavar="TOKENS", help="Context the engine is built for (the run's kv_tokens)")
148
162
 
149
163
  ab = subs.add_parser("ab", help="Alternate two engines' speed runs in one session and report paired ratios")
150
164
  ab.add_argument("--adapter", required=True, help="Adapter of engine A")
151
- ab.add_argument("--config", default="{}", help="JSON configuration of engine A")
165
+ ab.add_argument("--config", default="{}", help="JSON configuration of engine A (text or a file)")
152
166
  ab.add_argument("--adapter-b", help="Adapter of engine B (default: engine A's adapter)")
153
- ab.add_argument("--config-b", default="{}", help="JSON configuration of engine B")
167
+ ab.add_argument("--config-b", default="{}", help="JSON configuration of engine B (text or a file)")
154
168
  ab.add_argument("--model", required=True)
155
169
  ab.add_argument("--data", required=True)
156
170
  ab.add_argument("--out", required=True, help="Result JSON path")
@@ -165,7 +179,7 @@ def main(argv=None):
165
179
  kernels.add_argument("--model", required=True, help="Model directory passed to the adapter")
166
180
  kernels.add_argument("--data", required=True, help="Prepared suite directory for this model")
167
181
  kernels.add_argument("--out", required=True, help="Kernel report JSON path")
168
- kernels.add_argument("--config", default="{}", help="JSON configuration overrides passed to the adapter")
182
+ kernels.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
169
183
  kernels.add_argument("--case", action="append", help="Trace only these speed cases (repeatable); default all")
170
184
  kernels.add_argument("--device", metavar="JSON", help="Device ceilings from `crit eval probe` for share of peak")
171
185
  kernels.add_argument("--nvcc", help="nvcc of a CUDA toolkit that ships CUPTI")
@@ -192,24 +206,20 @@ ENGINE_COMMANDS = {"run", "pin", "ab", "kernels", "reference"}
192
206
  def _command(args):
193
207
  if args.command == "kernels":
194
208
  from . import load_json as load
195
- from .kernels import render as render_kernels, trace
196
- from .prepare import PREPARED
209
+ from .limits.kernels import render as render_kernels, trace
210
+ from .quality.prepare import PREPARED
197
211
  if not (Path(args.data) / PREPARED).is_file():
198
- from .prepare import prepare as do_prepare
212
+ from .quality.prepare import prepare as do_prepare
199
213
  do_prepare(args.model, args.data)
200
- work = None
201
- if args.work: # JSON text, else a file holding it
202
- try:
203
- work = json.loads(args.work)
204
- except json.JSONDecodeError:
205
- work = json.loads(Path(args.work).read_text(encoding="utf-8"))
206
- result = trace(args.adapter, args.model, args.data, args.out, json.loads(args.config), args.case, args.nvcc,
214
+ work = json_option(args.work, "--work") if args.work else None
215
+ result = trace(args.adapter, args.model, args.data, args.out, json_option(args.config, "--config"), args.case,
216
+ args.nvcc,
207
217
  load(args.device) if args.device else None, context_tokens=args.context, work=work)
208
218
  print(f"wrote {args.out}")
209
219
  print(render_kernels(result, args.top))
210
220
  return 0
211
221
  if args.command == "prepare":
212
- from .prepare import prepare as do_prepare
222
+ from .quality.prepare import prepare as do_prepare
213
223
  try:
214
224
  prepared = do_prepare(args.model, args.data, sampling=args.sampling, chat_template=args.chat_template)
215
225
  except ValueError as error:
@@ -223,8 +233,8 @@ def _command(args):
223
233
  print(f"quality: score them once on the checkpoint: crit eval reference --model {args.model} --data {args.data}")
224
234
  return 0
225
235
  if args.command == "reference":
226
- from .reference import score_data
227
- from .fixed import STORES
236
+ from .quality.reference import score_data
237
+ from .quality.fixed import STORES
228
238
  how = score_data(args.model, args.data, log=lambda line: print(line, flush=True), quality_mode=args.quality_mode)
229
239
  print(f"reference scores of {how['positions']:,} positions {how['method']} in "
230
240
  f"{Path(args.data) / STORES[args.quality_mode]}"
@@ -232,8 +242,8 @@ def _command(args):
232
242
  + (f"; the reference's own spread on the long texts {how['spread']}" if how.get("spread") else ""))
233
243
  return 0
234
244
  if args.command == "fetch":
235
- from .packages import fetch as do_fetch, source
236
- from .prepare import read_template
245
+ from .quality.packages import fetch as do_fetch, source
246
+ from .quality.prepare import read_template
237
247
  where = source(args.packages)
238
248
  if where is None:
239
249
  raise SystemExit("crit eval fetch: name a package source (--packages, or eval_packages in settings.toml)")
@@ -247,41 +257,43 @@ def _command(args):
247
257
  return 1
248
258
  return 0
249
259
  if args.command == "pack":
250
- from .packages import pack as do_pack
260
+ from .quality.packages import pack as do_pack
251
261
  entry = do_pack(args.data, args.packages)
252
262
  print(f"packed {entry['name']} ({entry['bytes'] / 2**20:.1f} MiB) into {args.packages}")
253
263
  return 0
254
264
  if args.command == "context":
255
265
  from . import load_json as load
256
- from .context import GIB, max_context, render as render_context
266
+ from .limits.context import GIB, max_context, render as render_context
257
267
  from . import EVAL_DIR
258
268
  settings = context_settings(load(EVAL_DIR / "protocol.json")["context"], args)
259
269
  value = max_context(args.model, load(args.device), headroom_gib(args) * GIB, settings, requested(args))
260
270
  print(json.dumps(value, indent=1) if args.json else render_context(value))
261
271
  return 0 if value["tokens"] else 1
262
272
  if args.command == "compare":
263
- from .report import compare as do_compare
273
+ from .reports.compare import compare as do_compare
264
274
  print(do_compare(args.results))
265
275
  return 0
266
276
  if args.command == "pin":
267
- from .pin import CONTINUATIONS, pin as do_pin
268
- from .prepare import PREPARED
277
+ from .run.pin import CONTINUATIONS, pin as do_pin
278
+ from .quality.prepare import PREPARED
269
279
  if not (Path(args.data) / PREPARED).is_file():
270
- from .prepare import prepare as do_prepare
280
+ from .quality.prepare import prepare as do_prepare
271
281
  do_prepare(args.model, args.data)
272
- pinned = do_pin(args.adapter, args.model, args.data, json.loads(args.config), context_tokens=args.context)
282
+ pinned = do_pin(args.adapter, args.model, args.data, json_option(args.config, "--config"),
283
+ context_tokens=args.context)
273
284
  print(f"pinned {len(pinned['cases'])} speed cases in {Path(args.data) / CONTINUATIONS} (sha256 {pinned['sha256'][:12]})")
274
285
  return 0
275
286
  if args.command == "ab":
276
- from .ab import render as render_ab, run_ab
277
- result = run_ab(args.adapter, json.loads(args.config), args.adapter_b or args.adapter, json.loads(args.config_b),
287
+ from .run.ab import render as render_ab, run_ab
288
+ result = run_ab(args.adapter, json_option(args.config, "--config"), args.adapter_b or args.adapter,
289
+ json_option(args.config_b, "--config-b"),
278
290
  args.model, args.data, args.out, args.rounds, args.warmups, args.repetitions,
279
291
  context_tokens=args.context)
280
292
  print(f"wrote {args.out}")
281
293
  print(render_ab(result))
282
294
  return 0
283
295
  if args.command == "probe":
284
- from .probe import probe as do_probe
296
+ from .limits.probe import probe as do_probe
285
297
  device = do_probe(args.out, args.nvcc, args.repeats)
286
298
  print(f"wrote {args.out}: {device['bandwidth_gbs']:.0f} GB/s, "
287
299
  + ", ".join(f"{k} {v:.0f} TFLOPS" for k, v in device["tflops"].items())
@@ -291,20 +303,20 @@ def _command(args):
291
303
  return 0
292
304
  if args.command == "status":
293
305
  from . import load_json as load
294
- from .status import render, status_of
306
+ from .reports.status import render, status_of
295
307
  from . import EVAL_DIR
296
308
  protocol = load(EVAL_DIR / "protocol.json")
297
309
  current = status_of(args.result, protocol, args.baseline, args.device, args.data, args.kernels)
298
310
  print(json.dumps(current, indent=1) if args.json else render(current))
299
311
  return 0
300
- from .prepare import PREPARED, read_template
301
- from .runner import run as do_run
312
+ from .quality.prepare import PREPARED, read_template
313
+ from .run.runner import run as do_run
302
314
  parts = tuple(args.only or PARTS)
303
315
  data = Path(args.data)
304
316
  template = read_template(args.chat_template)
305
317
  if not (data / PREPARED).is_file():
306
- from .packages import fetch as do_fetch, source
307
- from .prepare import prepare as do_prepare
318
+ from .quality.packages import fetch as do_fetch, source
319
+ from .quality.prepare import prepare as do_prepare
308
320
  try:
309
321
  # A published package brings the reference scores too; otherwise prepare here.
310
322
  if source() is None or do_fetch(args.model, data, source(), chat_template=template) is None:
@@ -312,12 +324,13 @@ def _command(args):
312
324
  except ValueError as error: # no chat template to render with, or one for a model that carries its own
313
325
  raise SystemExit(f"crit eval run: {error}")
314
326
  from . import load_json as load
315
- from .status import render
327
+ from .reports.status import render
316
328
  device = load(args.device) if args.device else None
317
329
  baseline = load(args.baseline) if args.baseline else None
318
- result = do_run(args.adapter, args.model, data, args.out, json.loads(args.config), parts,
330
+ result = do_run(args.adapter, args.model, data, args.out, json_option(args.config, "--config"), parts,
319
331
  args.warmups, args.repetitions, args.label, bandwidth_gbs=args.bandwidth, device=device,
320
- baseline=baseline, quality_budget=json.loads(args.quality_budget) if args.quality_budget else None,
332
+ baseline=baseline, quality_budget=(json_option(args.quality_budget, "--quality-budget")
333
+ if args.quality_budget else None),
321
334
  exactness=args.exactness, context_tokens=args.context, requested_context=requested(args),
322
335
  vram_headroom_bytes=round(headroom_gib(args) * (1 << 30)),
323
336
  context_reserve_bytes=round(reserve_gib(args) * (1 << 30)) if reserve_gib(args) is not None else None,
@@ -0,0 +1,2 @@
1
+ """The physical limits an engine is measured against: the device probe, the model's arithmetic and weight bytes per
2
+ token, the longest request it must hold, where GPU time goes, and how a checkpoint stores its tensors."""
@@ -9,7 +9,7 @@ Prefill is compute bound. Its floor for a prompt of N tokens is
9
9
  recurrent N x linear-attention FLOPs per token / the BF16 rate (chunked form, approximate)
10
10
 
11
11
  A projection weight of R x K elements costs 2RK FLOPs per token at the rate of the format it is stored in
12
- (crit_eval.schemes): BF16/F16/F32 (bf16), FP8 (fp8), NVFP4 (nvfp4), weight-only integer codes (AWQ, GPTQ,
12
+ (crit_eval.limits.schemes): BF16/F16/F32 (bf16), FP8 (fp8), NVFP4 (nvfp4), weight-only integer codes (AWQ, GPTQ,
13
13
  MLX, bitsandbytes NF4, compressed-tensors pack-quantized: bf16, as engines widen them for the GEMM) and 8-bit
14
14
  integer weights with per-row scales (bitsandbytes LLM.int8, compressed-tensors int-quantized: fp8, the int8 rate).
15
15
  A GGUF's ggml block types
@@ -18,7 +18,7 @@ the same rate), its MXFP4 and NVFP4 at the nvfp4 rate and its F32/F16/BF16 at bf
18
18
  (gathered), vision towers, norms, convolutions and the output head (one row of logits per
19
19
  prompt) are left out. Engines may compute in other formats; the floor uses the stored ones, and
20
20
  prefill_floor_ms prices the quantized matrices with other prompt activations on request (ACTIVATIONS:
21
- BF16, FP16 accumulation, FP8), as crit_eval.headroom.activation_ceilings reports them.
21
+ BF16, FP16 accumulation, FP8), as crit_eval.limits.headroom.activation_ceilings reports them.
22
22
 
23
23
  Time to first token cannot beat either floor: the prompt's arithmetic, or reading the weights
24
24
  once (language-model tensors and the output head, not embeddings, vision or draft layers)
@@ -68,7 +68,7 @@ def _read(role):
68
68
 
69
69
 
70
70
  def weight_bytes(tensors, file_name="", draft_blocks=()):
71
- """Bytes of the tensors one forward pass reads (crit_eval.tensors: layers, experts and the output head, not
71
+ """Bytes of the tensors one forward pass reads (crit_eval.limits.tensors: layers, experts and the output head, not
72
72
  embeddings, other modalities or draft layers). tensors: {name: crit_core.model.TensorInfo} of one file."""
73
73
  return sum(t.nbytes for name, t in tensors.items() if _read(tensor_role(name, file_name, draft_blocks)))
74
74
 
@@ -87,7 +87,7 @@ def matmul_flops(tensors, file_name="", draft_blocks=(), quantization=None):
87
87
  {name: TensorInfo}. The output head is left out (one row of logits per prompt). A 3-D expert tensor counts as
88
88
  one matrix per expert, as the per-expert matrices of a checkpoint that stores them apart do. quantization: the
89
89
  config's quantization_config (or MLX's quantization), which names the scheme of packed integer weights
90
- (crit_eval.schemes)."""
90
+ (crit_eval.limits.schemes)."""
91
91
  flops, matrices, seen = {}, {}, set()
92
92
  for name, tensor in tensors.items():
93
93
  role = tensor_role(name, file_name, draft_blocks)
@@ -166,7 +166,7 @@ def best_rates(device):
166
166
  return rates, sources
167
167
 
168
168
 
169
- # The activation formats a quantized matrix's prompt GEMM can take, and the rates (crit_eval.probe keys) each runs at
169
+ # The activation formats a quantized matrix's prompt GEMM can take, and the rates (crit_eval.limits.probe keys) each runs at
170
170
  # for each stored weight format, the best available one counting: BF16 activations on BF16 tensor cores with FP32
171
171
  # accumulation, FP16 activations accumulating in FP16, FP8 activations (FP4 weights times FP8 through kind::f8f6f4, or
172
172
  # widened to FP8), and the stored format itself (W8A8, W4A4). Weights stored in BF16 stay at the BF16 rate.
@@ -32,7 +32,7 @@ from .ceiling import model_config, weight_files
32
32
  from .tensors import tensor_role
33
33
 
34
34
  GIB = 1 << 30
35
- DRAFT_HEAD = re.compile(r"(^|\.)head\.weight$") # a draft layer's own output head (DeepSeek-style shared_head.head)
35
+ DRAFT_HEAD = re.compile(r"(^|\.)head\.weight$") # a draft layer's own output head (shared_head.head)
36
36
 
37
37
 
38
38
  def weight_split(model, config):
@@ -64,7 +64,7 @@ crit-eval does not judge or classify speed. Noise is the spread of each score ac
64
64
  repetitions.
65
65
  """
66
66
  from . import ceiling as model_ceiling
67
- from . import metrics
67
+ from ..reports import metrics
68
68
 
69
69
 
70
70
  def decode_suite(result):
@@ -586,7 +586,7 @@ def trace_decode_parts(result, trace):
586
586
 
587
587
 
588
588
  def price_prompts(result, rates, sources, prefill_cases, evidence=()):
589
- """A result's headroom with prefill and time to first token priced at `rates` (crit_eval.ceiling.best_rates of a
589
+ """A result's headroom with prefill and time to first token priced at `rates` (crit_eval.limits.ceiling.best_rates of a
590
590
  device) on the model arithmetic it records (result ceiling): the stored-format prefill and TTFT summaries, the
591
591
  ceilings by prompt activation format (activation_ceilings, judged with this result's and `evidence`'s quality),
592
592
  and their gaps in place of the old ones. The run prices them so; `crit eval status --device/--data` prices a
@@ -27,9 +27,9 @@ import sys
27
27
  import tempfile
28
28
  import time
29
29
 
30
- from . import CONTRACT, EVAL_DIR, load_json, portable_path, write_json
30
+ from .. import CONTRACT, EVAL_DIR, load_json, portable_path, write_json
31
31
  from .ceiling import best_rates
32
- from .prepare import PREPARED
32
+ from ..quality.prepare import PREPARED
33
33
 
34
34
  SOURCE = EVAL_DIR / "probe" / "kernel_trace.cpp"
35
35
  TOP = 12
@@ -115,7 +115,7 @@ def _timed(engine, ids, n, now=time.time_ns):
115
115
 
116
116
 
117
117
  def _child(spec):
118
- from .runner import load_adapter, open_engine
118
+ from ..run.adapters import load_adapter, open_engine
119
119
  prepared = load_json(Path(spec["data"]) / PREPARED)
120
120
  module = load_adapter(spec["adapter"])
121
121
  engine, info = open_engine(module, spec["model"], spec["config"], speculative=True,
@@ -223,7 +223,7 @@ def _work_rule(rules, name):
223
223
  return None
224
224
 
225
225
 
226
- # The terms of the prefill floor a kernel's time joins (crit_eval.ceiling), and draft work kept apart from them.
226
+ # The terms of the prefill floor a kernel's time joins (crit_eval.limits.ceiling), and draft work kept apart from them.
227
227
  TERMS = ("matmul", "attention", "recurrent", "other", "draft")
228
228
  # Name heuristics for kernels no kernel_work rule classes, checked in order on the lower-cased demangled name.
229
229
  # Weight-only GEMMs go by their scheme (w4a16, marlin) and crit's stored-tile projections by their namespace.
@@ -305,7 +305,7 @@ def analyze(kernels, device, marks, device_ceilings=None):
305
305
  sorted(unclassed.items(), key=lambda x: -x[1])[:UNCLASSED]]}
306
306
  if phase == "prefill" and "prefill_first" in case:
307
307
  # The fresh request's time to its first token, and the GPU's busy time inside it: the rest is the
308
- # engine's host work, launches and synchronization (crit_eval.status: the TTFT breakdown).
308
+ # engine's host work, launches and synchronization (crit_eval.reports.status: the TTFT breakdown).
309
309
  first_ms = (case["prefill_first"] - lo) / 1e6
310
310
  gpu = sum(ns for k, ns in inside if k[0] < case["prefill_first"]) / 1e6
311
311
  cases[case["id"]][phase].update(first_token_ms=first_ms, gpu_before_first_ms=min(gpu, first_ms))
@@ -422,7 +422,7 @@ def trace(adapter, model, data, out, config=None, cases=None, nvcc=None, device=
422
422
  env = dict(os.environ, CUDA_INJECTION64_PATH=str(library), CRIT_KTRACE_DIR=directory,
423
423
  PYTHONPATH=os.pathsep.join(filter(None, paths)))
424
424
  log(f"tracing kernels with {library.name} ({toolkit['nvcc']})")
425
- done = subprocess.run([sys.executable, "-B", "-m", "crit_eval.kernels", "--child", json.dumps(spec)], env=env)
425
+ done = subprocess.run([sys.executable, "-B", "-m", "crit_eval.limits.kernels", "--child", json.dumps(spec)], env=env)
426
426
  if done.returncode != 0:
427
427
  raise SystemExit(f"traced engine process failed ({done.returncode})")
428
428
  marks = load_json(Path(directory) / "marks.json")
@@ -5,7 +5,7 @@ keeps the median of each value: achievable copy bandwidth (GB/s, bytes read plus
5
5
  the decode pass floor, dense cuBLASLt GEMM TFLOPS per weight format (bf16, fp8, nvfp4) and raw
6
6
  tensor-core (mma.sync) TFLOPS per input and accumulator format (bf16, fp16, fp16_acc, fp8, nvfp4,
7
7
  fp4_fp8) for the prefill floor, which takes the better of the two per format
8
- (crit_eval.ceiling.best_rates). It also keeps the least free device memory any run saw, for the max context,
8
+ (crit_eval.limits.ceiling.best_rates). It also keeps the least free device memory any run saw, for the max context,
9
9
  the host's physical and available memory, and host-to-device rates per transfer size (pinned
10
10
  copy, pageable copy, mapped read) for engines that keep weights in host memory. Pass the file to `crit eval run --device
11
11
  device.json`. Probe on the machine and in the environment the engine runs in, with the same
@@ -21,7 +21,8 @@ import subprocess
21
21
  import sys
22
22
  import tempfile
23
23
 
24
- from . import EVAL_DIR, metrics, portable_path, write_json
24
+ from .. import EVAL_DIR, portable_path, write_json
25
+ from ..reports import metrics
25
26
 
26
27
  SOURCE = EVAL_DIR / "probe" / "device_probe.cu"
27
28