crit-eval 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. {crit_eval-0.2.0 → crit_eval-0.3.0}/PKG-INFO +3 -3
  2. {crit_eval-0.2.0 → crit_eval-0.3.0}/README.md +2 -2
  3. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/__init__.py +3 -3
  4. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/cli.py +70 -22
  5. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/context.py +137 -6
  6. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/headroom.py +75 -21
  7. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/probe.py +18 -0
  8. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/protocol.json +21 -4
  9. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/fixed.py +33 -5
  10. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/packages.py +12 -21
  11. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/prepare.py +22 -5
  12. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/reference.py +282 -43
  13. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/reports/status.py +65 -6
  14. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/chat.py +5 -2
  15. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/long_context.py +9 -0
  16. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/quality_checks.py +35 -13
  17. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/runner.py +28 -13
  18. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/timing.py +61 -4
  19. crit_eval-0.3.0/crit_eval/workbench/__init__.py +71 -0
  20. crit_eval-0.3.0/crit_eval/workbench/harness.py +166 -0
  21. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/PKG-INFO +3 -3
  22. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/SOURCES.txt +4 -1
  23. {crit_eval-0.2.0 → crit_eval-0.3.0}/pyproject.toml +2 -2
  24. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval.py +325 -8
  25. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_packages.py +17 -17
  26. crit_eval-0.3.0/tests/test_eval_quality_modes.py +408 -0
  27. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_server.py +5 -2
  28. crit_eval-0.3.0/tests/test_eval_workbench.py +54 -0
  29. crit_eval-0.2.0/tests/test_eval_quality_modes.py +0 -240
  30. {crit_eval-0.2.0 → crit_eval-0.3.0}/LICENSE +0 -0
  31. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/__main__.py +0 -0
  32. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/__init__.py +0 -0
  33. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/gguf_deepseek4.py +0 -0
  34. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/gguf_layout.py +0 -0
  35. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/gguf_qwen4exp.py +0 -0
  36. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/indexers.py +0 -0
  37. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/examples/mock_adapter.py +0 -0
  38. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/__init__.py +0 -0
  39. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/ceiling.py +0 -0
  40. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/kernels.py +0 -0
  41. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/schemes.py +0 -0
  42. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/tensors.py +0 -0
  43. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/probe/device_probe.cu +0 -0
  44. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/probe/kernel_trace.cpp +0 -0
  45. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/probe/logprob_compare.cu +0 -0
  46. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/probe/logprob_compare.h +0 -0
  47. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/__init__.py +0 -0
  48. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/attention.py +0 -0
  49. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/checkpoints.py +0 -0
  50. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/logprob.py +0 -0
  51. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/prompt_state.py +0 -0
  52. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/reports/__init__.py +0 -0
  53. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/reports/compare.py +0 -0
  54. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/reports/metrics.py +0 -0
  55. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/__init__.py +0 -0
  56. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/ab.py +0 -0
  57. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/adapters.py +0 -0
  58. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/deadline.py +0 -0
  59. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/estimate.py +0 -0
  60. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/history.py +0 -0
  61. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/host.py +0 -0
  62. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/hostload.py +0 -0
  63. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/outcomes.py +0 -0
  64. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/pin.py +0 -0
  65. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/speed.py +0 -0
  66. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/README.md.txt +0 -0
  67. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/cli.py.txt +0 -0
  68. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/headroom.py.txt +0 -0
  69. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/metrics.py.txt +0 -0
  70. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/prepare.py.txt +0 -0
  71. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/reference.py.txt +0 -0
  72. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/report.py.txt +0 -0
  73. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/runner.py.txt +0 -0
  74. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/tools.json +0 -0
  75. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/followup.json +0 -0
  76. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/heldout.json +0 -0
  77. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/long.json +0 -0
  78. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/long_document.txt +0 -0
  79. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/quality.json +0 -0
  80. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/speed.json +0 -0
  81. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/dependency_links.txt +0 -0
  82. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/entry_points.txt +0 -0
  83. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/requires.txt +0 -0
  84. {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/top_level.txt +0 -0
  85. {crit_eval-0.2.0 → crit_eval-0.3.0}/setup.cfg +0 -0
  86. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_chat_template.py +0 -0
  87. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_deadline.py +0 -0
  88. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_distributions.py +0 -0
  89. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_estimate.py +0 -0
  90. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_followup.py +0 -0
  91. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_formats.py +0 -0
  92. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_gguf.py +0 -0
  93. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_gguf_deepseek4.py +0 -0
  94. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_gguf_qwen4exp.py +0 -0
  95. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_indexers.py +0 -0
  96. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_kernels.py +0 -0
  97. {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_tensors.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: crit-eval
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: crit-eval: one speed, quality and exactness suite that every inference engine runs the same way
5
5
  License-Expression: Apache-2.0
6
6
  Project-URL: Repository, https://github.com/crit-labs/crit-eval
@@ -25,7 +25,7 @@ Dynamic: license-file
25
25
 
26
26
  ### Speed and quality measurement tool to compare inference engines honestly.
27
27
 
28
- [![Version 0.1.2](docs/assets/badge-version.svg)](CHANGELOG.md)
28
+ [![Version 0.3.0](docs/assets/badge-version.svg)](CHANGELOG.md)
29
29
  [![License Apache-2.0](docs/assets/badge-license.svg)](LICENSE)
30
30
  [![Python 3.11+](docs/assets/badge-python.svg)](pyproject.toml)
31
31
 
@@ -165,7 +165,7 @@ plugs in through a Python [adapter](docs/ADAPTER.md) instead of `--url`.
165
165
  | Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages for three checkpoints are published on Hugging Face ([crit-labs/crit-eval-references](https://huggingface.co/datasets/crit-labs/crit-eval-references)) and fetched by default. |
166
166
  | Rounding-sensitive models | Where a model amplifies rounding (MoE routing near-ties, sparse-attention ties, recurrent state), two valid FP32 runs of the reference disagree; quality is then judged against twice the reference's own spread instead of the budget, which is looser for those models. |
167
167
  | Speed | One request at a time: `crit-eval` measures single-stream decode, prefill and time to first token, not throughput under concurrent requests or batching. A noisy sampled decode is repeated at up to nine seeds per case, so an engine whose speed swings with the drawn text takes longer to measure (`--no-adaptive-sampled` keeps three, an override the result records). |
168
- | Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded, but no limit uses them yet, so engines that offload weights or compute on the CPU get GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
168
+ | Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded; an engine that declares the bytes it reads from host memory per pass (`host_bytes_per_pass`) gets them priced at those rates, every other engine gets GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
169
169
  | Long context | The hub profile measures the long-context curve at its two ends and interpolates the points between; only the full profile measures every point. |
170
170
  | Comparability | Results compare only under the same protocol series and profile, on prepared data of the same checkpoint. |
171
171
 
@@ -8,7 +8,7 @@
8
8
 
9
9
  ### Speed and quality measurement tool to compare inference engines honestly.
10
10
 
11
- [![Version 0.1.2](docs/assets/badge-version.svg)](CHANGELOG.md)
11
+ [![Version 0.3.0](docs/assets/badge-version.svg)](CHANGELOG.md)
12
12
  [![License Apache-2.0](docs/assets/badge-license.svg)](LICENSE)
13
13
  [![Python 3.11+](docs/assets/badge-python.svg)](pyproject.toml)
14
14
 
@@ -148,7 +148,7 @@ plugs in through a Python [adapter](docs/ADAPTER.md) instead of `--url`.
148
148
  | Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages for three checkpoints are published on Hugging Face ([crit-labs/crit-eval-references](https://huggingface.co/datasets/crit-labs/crit-eval-references)) and fetched by default. |
149
149
  | Rounding-sensitive models | Where a model amplifies rounding (MoE routing near-ties, sparse-attention ties, recurrent state), two valid FP32 runs of the reference disagree; quality is then judged against twice the reference's own spread instead of the budget, which is looser for those models. |
150
150
  | Speed | One request at a time: `crit-eval` measures single-stream decode, prefill and time to first token, not throughput under concurrent requests or batching. A noisy sampled decode is repeated at up to nine seeds per case, so an engine whose speed swings with the drawn text takes longer to measure (`--no-adaptive-sampled` keeps three, an override the result records). |
151
- | Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded, but no limit uses them yet, so engines that offload weights or compute on the CPU get GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
151
+ | Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded; an engine that declares the bytes it reads from host memory per pass (`host_bytes_per_pass`) gets them priced at those rates, every other engine gets GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
152
152
  | Long context | The hub profile measures the long-context curve at its two ends and interpolates the points between; only the full profile measures every point. |
153
153
  | Comparability | Results compare only under the same protocol series and profile, on prepared data of the same checkpoint. |
154
154
 
@@ -7,7 +7,7 @@ from pathlib import Path
7
7
  # The package holds its suite, protocol, probes and examples; a checkout of crit-eval is its parent.
8
8
  EVAL_DIR = Path(__file__).resolve().parent
9
9
  SUITE_DIR = EVAL_DIR / "suite"
10
- __version__ = "0.2.0"
10
+ __version__ = "0.3.0"
11
11
  CONTRACT = "eval-v1"
12
12
  PROTOCOL_ID = re.compile(r"crit-eval-v(\d+)\.(\d+)\.(\d+)")
13
13
  # The ids before crit-eval had its own repository (crit-eval-v4, crit-eval-v4.1, ... crit-eval-v8): major and minor.
@@ -48,7 +48,7 @@ def checkouts():
48
48
 
49
49
 
50
50
  def machine_settings():
51
- """Memory thresholds from settings.toml (vram_headroom_gib, context_reserve_gib), the wanted context
51
+ """Memory thresholds from settings.toml (vram_headroom_gib, ram_headroom_gib, context_reserve_gib), the wanted context
52
52
  (context_tokens) and the reference package source (eval_packages, crit_eval.quality.packages): crit-eval's checkout's,
53
53
  else the crit checkout's (checkouts); without one crit-eval uses the protocol's defaults and
54
54
  DEFAULT_CONTEXT_TOKENS."""
@@ -57,7 +57,7 @@ def machine_settings():
57
57
  return {}
58
58
  import tomllib
59
59
  value = tomllib.loads(path.read_text(encoding="utf-8"))
60
- found = {key: float(value[key]) for key in ("vram_headroom_gib", "context_reserve_gib")
60
+ found = {key: float(value[key]) for key in ("vram_headroom_gib", "ram_headroom_gib", "context_reserve_gib")
61
61
  if isinstance(value.get(key), (int, float)) and not isinstance(value.get(key), bool)}
62
62
  if type(value.get("context_tokens")) is int and value["context_tokens"] > 0:
63
63
  found["context_tokens"] = value["context_tokens"]
@@ -24,6 +24,17 @@ def json_option(text, option):
24
24
  raise SystemExit(f"{option} file {path} is not JSON: {error}")
25
25
 
26
26
 
27
+ def adapter_config(adapter, text, option):
28
+ """An adapter's configuration: the configuration.json packaged beside it, the one crit serve opens the candidate
29
+ with, then the option's overrides. An adapter outside a package gets the overrides alone."""
30
+ overrides = json_option(text, option)
31
+ packaged = Path(adapter).parent / "configuration.json" if adapter else None
32
+ if packaged is None or not packaged.is_file():
33
+ return overrides
34
+ print(f"{option}: {packaged} with {len(overrides)} override(s)", file=sys.stderr)
35
+ return {**json_option(str(packaged), option), **overrides}
36
+
37
+
27
38
  def main(argv=None):
28
39
  if hasattr(sys.stdout, "reconfigure"):
29
40
  sys.stdout.reconfigure(encoding="utf-8", errors="replace")
@@ -39,7 +50,7 @@ def main(argv=None):
39
50
  run.add_argument("--model", required=True, help="Model directory passed to the adapter")
40
51
  run.add_argument("--data", required=True, help="Prepared suite directory for this model")
41
52
  run.add_argument("--out", required=True, help="Result JSON path")
42
- run.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
53
+ run.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file, over the configuration.json packaged beside it")
43
54
  run.add_argument("--only", choices=list(PARTS), action="append",
44
55
  help="Run only these parts (repeatable); default is all. context: the long-context curve and "
45
56
  "retrieval up to the required context and the speed cases again at a lower context; "
@@ -55,7 +66,7 @@ def main(argv=None):
55
66
  help='Drift limits over the protocol defaults, e.g. \'{"ppl_ratio_max": 1.02}\'; '
56
67
  'keys ppl_ratio_max, kld_mean_max, kld_p99_max')
57
68
  run.add_argument("--quality-mode", choices=["preserve", "as_quantized"], default="preserve",
58
- help="preserve (default): quality is judged against the model's own arithmetic with exact activations; as_quantized: against the checkpoint run as quantized, its activations quantized as it declares (score that reference first: crit eval reference --quality-mode as_quantized)")
69
+ help="preserve (default): quality is judged against the model's own arithmetic with exact activations; as_quantized: against the same scores with each set's budget widened by how far the checkpoint's own activation quantization lands from them (the allowance crit eval reference measures)")
59
70
  run.add_argument("--exactness", choices=["reported", "required"],
60
71
  help="reported (default): agreement with plain greedy is shown and the quality budget judges "
61
72
  "the output; required: any difference fails the gate")
@@ -94,11 +105,17 @@ def main(argv=None):
94
105
  context.add_argument("--device", required=True, metavar="JSON", help="Device file from `crit eval probe`")
95
106
  context.add_argument("--vram-headroom-gib", type=float, default=None,
96
107
  help="Device memory left free (default: settings.toml vram_headroom_gib, else 1)")
108
+ context.add_argument("--ram-headroom-gib", type=float, default=None,
109
+ help="Host memory left free, for the placement scenario (default: settings.toml "
110
+ "ram_headroom_gib, else 2)")
97
111
  context.add_argument("--context-reserve-gib", type=float, default=None,
98
112
  help="Device memory kept out of the KV cache for workspaces and speed techniques "
99
113
  "(default: settings.toml context_reserve_gib, else the protocol's reserve_gib)")
100
114
  context.add_argument("--requested-context", type=int, metavar="TOKENS", default=None,
101
115
  help="The longest request wanted (default: settings.toml context_tokens, else 262,144)")
116
+ context.add_argument("--drafter", action="append", default=[], metavar="DIR",
117
+ help="A companion drafter the engine may load beside the model; its stored weights are kept "
118
+ "out of the KV cache (repeatable)")
102
119
  context.add_argument("--json", action="store_true", help="Print the computation as JSON")
103
120
 
104
121
  probe = subs.add_parser("probe", help="Measure device bandwidth and GEMM rates per weight format (needs nvcc)")
@@ -131,10 +148,10 @@ def main(argv=None):
131
148
  "transformers, GPU; stop any engine first)")
132
149
  reference.add_argument("--model", required=True, help="The checkpoint the data was prepared from")
133
150
  reference.add_argument("--data", required=True, help="Prepared suite directory")
134
- reference.add_argument("--quality-mode", choices=["preserve", "as_quantized"], default="preserve",
135
- help="preserve (default): exact activations; as_quantized: the activations quantized as the "
136
- "checkpoint declares, scored into a store of their own for runs with --quality-mode "
137
- "as_quantized")
151
+ reference.add_argument("--quality-mode", choices=["preserve", "as_quantized"], default=None,
152
+ help="Accepted from earlier versions and ignored: one run scores what both quality modes "
153
+ "need (the scores, their spread and, for a checkpoint that declares quantized "
154
+ "activations, the as_quantized allowance)")
138
155
 
139
156
  fetch = subs.add_parser("fetch", help="Fetch the checkpoint's reference package (prepared data and reference "
140
157
  "scores) into the data, so a run needs only the engine")
@@ -157,14 +174,14 @@ def main(argv=None):
157
174
  pin.add_argument("--adapter", required=True, help="Python file exporting create_eval_engine(context)")
158
175
  pin.add_argument("--model", required=True, help="Model directory passed to the adapter")
159
176
  pin.add_argument("--data", required=True, help="Prepared data directory to pin the continuation in")
160
- pin.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
177
+ pin.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file, over the configuration.json packaged beside it")
161
178
  pin.add_argument("--context", type=int, metavar="TOKENS", help="Context the engine is built for (the run's kv_tokens)")
162
179
 
163
180
  ab = subs.add_parser("ab", help="Alternate two engines' speed runs in one session and report paired ratios")
164
181
  ab.add_argument("--adapter", required=True, help="Adapter of engine A")
165
- ab.add_argument("--config", default="{}", help="JSON configuration of engine A (text or a file)")
182
+ ab.add_argument("--config", default="{}", help="JSON configuration overrides of engine A (text or a file), over its packaged configuration.json")
166
183
  ab.add_argument("--adapter-b", help="Adapter of engine B (default: engine A's adapter)")
167
- ab.add_argument("--config-b", default="{}", help="JSON configuration of engine B (text or a file)")
184
+ ab.add_argument("--config-b", default="{}", help="JSON configuration overrides of engine B (text or a file), over its packaged configuration.json")
168
185
  ab.add_argument("--model", required=True)
169
186
  ab.add_argument("--data", required=True)
170
187
  ab.add_argument("--out", required=True, help="Result JSON path")
@@ -179,7 +196,7 @@ def main(argv=None):
179
196
  kernels.add_argument("--model", required=True, help="Model directory passed to the adapter")
180
197
  kernels.add_argument("--data", required=True, help="Prepared suite directory for this model")
181
198
  kernels.add_argument("--out", required=True, help="Kernel report JSON path")
182
- kernels.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
199
+ kernels.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file, over the configuration.json packaged beside it")
183
200
  kernels.add_argument("--case", action="append", help="Trace only these speed cases (repeatable); default all")
184
201
  kernels.add_argument("--device", metavar="JSON", help="Device ceilings from `crit eval probe` for share of peak")
185
202
  kernels.add_argument("--nvcc", help="nvcc of a CUDA toolkit that ships CUPTI")
@@ -189,6 +206,17 @@ def main(argv=None):
189
206
  "other|draft, \"format\": nvfp4|fp8|bf16, ...}]")
190
207
  kernels.add_argument("--context", type=int, metavar="TOKENS", help="Context the engine is built for (the run's kv_tokens)")
191
208
 
209
+ workbench = subs.add_parser("workbench", help="Check, time or profile a kernel on its workbench folder "
210
+ "(crit kernels new): reference, floor, yardstick")
211
+ workbench.add_argument("action", choices=("check", "bench", "profile"))
212
+ workbench.add_argument("folder", help="The workbench folder")
213
+ workbench.add_argument("--python", help="The engine's Python (torch and the GPU); default this one")
214
+ workbench.add_argument("--shape", type=int, help="One shape of kernel.json by index (profile: default the first)")
215
+ workbench.add_argument("--iters", type=int, help="bench: timed calls per shape and column count")
216
+ workbench.add_argument("--device", metavar="JSON", help="bench: a `crit eval probe` result, for the floor")
217
+ workbench.add_argument("--out", help="check and bench: the result JSON (default FOLDER/check.json, bench.json)")
218
+ workbench.add_argument("--ncu", help="profile: Nsight Compute's ncu, to add its sections")
219
+
192
220
  args = parser.parse_args(argv)
193
221
  # An engine behind a server is the server's to load (crit serve holds the lock while it serves).
194
222
  if args.command in ENGINE_COMMANDS and not getattr(args, "url", None):
@@ -204,6 +232,17 @@ ENGINE_COMMANDS = {"run", "pin", "ab", "kernels", "reference"}
204
232
 
205
233
 
206
234
  def _command(args):
235
+ if args.command == "workbench":
236
+ from .workbench import profile, run_harness
237
+ if args.action == "profile":
238
+ result = profile(args.folder, args.python, args.ncu, args.shape or 0)
239
+ print(json.dumps(result, indent=2))
240
+ return 0
241
+ options = {"iters": args.iters, "device": args.device, "out": args.out, "shape": args.shape} if args.action == "bench" else {"out": args.out, "shape": args.shape}
242
+ code, result = run_harness(args.folder, args.action, args.python, **options)
243
+ if result is None:
244
+ raise SystemExit(f"the workbench harness exited {code} without a result")
245
+ return 0 if code == 0 else 1
207
246
  if args.command == "kernels":
208
247
  from . import load_json as load
209
248
  from .limits.kernels import render as render_kernels, trace
@@ -212,7 +251,7 @@ def _command(args):
212
251
  from .quality.prepare import prepare as do_prepare
213
252
  do_prepare(args.model, args.data)
214
253
  work = json_option(args.work, "--work") if args.work else None
215
- result = trace(args.adapter, args.model, args.data, args.out, json_option(args.config, "--config"), args.case,
254
+ result = trace(args.adapter, args.model, args.data, args.out, adapter_config(args.adapter, args.config, "--config"), args.case,
216
255
  args.nvcc,
217
256
  load(args.device) if args.device else None, context_tokens=args.context, work=work)
218
257
  print(f"wrote {args.out}")
@@ -234,12 +273,14 @@ def _command(args):
234
273
  return 0
235
274
  if args.command == "reference":
236
275
  from .quality.reference import score_data
237
- from .quality.fixed import STORES
238
- how = score_data(args.model, args.data, log=lambda line: print(line, flush=True), quality_mode=args.quality_mode)
239
- print(f"reference scores of {how['positions']:,} positions {how['method']} in "
240
- f"{Path(args.data) / STORES[args.quality_mode]}"
276
+ from .quality.fixed import STORE
277
+ how = score_data(args.model, args.data, log=lambda line: print(line, flush=True))
278
+ print(f"reference scores of {how['positions']:,} positions {how['method']} in {Path(args.data) / STORE}"
241
279
  + (f" ({how['seconds']:.0f} s, {how['bytes'] / 2**20:.1f} MiB)" if how["method"] == "scored" else "")
242
- + (f"; the reference's own spread on the long texts {how['spread']}" if how.get("spread") else ""))
280
+ + (f"; the reference's own spread {how['spread']}" if how.get("spread") else "")
281
+ + {"none": "; the checkpoint declares no quantized activations (as_quantized judges as preserve)",
282
+ "unsupported": "; the as_quantized allowance not measured (see above)"}.get(
283
+ how.get("allowance"), f"; the as_quantized allowance {how['allowance']}" if how.get("allowance") else ""))
243
284
  return 0
244
285
  if args.command == "fetch":
245
286
  from .quality.packages import fetch as do_fetch, source
@@ -266,8 +307,15 @@ def _command(args):
266
307
  from .limits.context import GIB, max_context, render as render_context
267
308
  from . import EVAL_DIR
268
309
  settings = context_settings(load(EVAL_DIR / "protocol.json")["context"], args)
269
- value = max_context(args.model, load(args.device), headroom_gib(args) * GIB, settings, requested(args))
270
- print(json.dumps(value, indent=1) if args.json else render_context(value))
310
+ value = max_context(args.model, load(args.device), headroom_gib(args) * GIB, settings, requested(args),
311
+ drafters=args.drafter)
312
+ from .limits.context import placement, render_placement
313
+ from . import machine_settings
314
+ ram = args.ram_headroom_gib if args.ram_headroom_gib is not None else machine_settings().get("ram_headroom_gib", 2.0)
315
+ where = placement(args.model, load(args.device), headroom_gib(args) * GIB, ram * GIB, settings, requested(args),
316
+ drafters=args.drafter)
317
+ print(json.dumps({**value, "placement": where}, indent=1) if args.json
318
+ else render_context(value) + "\n" + render_placement(where))
271
319
  return 0 if value["tokens"] else 1
272
320
  if args.command == "compare":
273
321
  from .reports.compare import compare as do_compare
@@ -279,14 +327,14 @@ def _command(args):
279
327
  if not (Path(args.data) / PREPARED).is_file():
280
328
  from .quality.prepare import prepare as do_prepare
281
329
  do_prepare(args.model, args.data)
282
- pinned = do_pin(args.adapter, args.model, args.data, json_option(args.config, "--config"),
330
+ pinned = do_pin(args.adapter, args.model, args.data, adapter_config(args.adapter, args.config, "--config"),
283
331
  context_tokens=args.context)
284
332
  print(f"pinned {len(pinned['cases'])} speed cases in {Path(args.data) / CONTINUATIONS} (sha256 {pinned['sha256'][:12]})")
285
333
  return 0
286
334
  if args.command == "ab":
287
335
  from .run.ab import render as render_ab, run_ab
288
- result = run_ab(args.adapter, json_option(args.config, "--config"), args.adapter_b or args.adapter,
289
- json_option(args.config_b, "--config-b"),
336
+ result = run_ab(args.adapter, adapter_config(args.adapter, args.config, "--config"), args.adapter_b or args.adapter,
337
+ adapter_config(args.adapter_b or args.adapter, args.config_b, "--config-b"),
290
338
  args.model, args.data, args.out, args.rounds, args.warmups, args.repetitions,
291
339
  context_tokens=args.context)
292
340
  print(f"wrote {args.out}")
@@ -327,7 +375,7 @@ def _command(args):
327
375
  from .reports.status import render
328
376
  device = load(args.device) if args.device else None
329
377
  baseline = load(args.baseline) if args.baseline else None
330
- result = do_run(args.adapter, args.model, data, args.out, json_option(args.config, "--config"), parts,
378
+ result = do_run(args.adapter, args.model, data, args.out, adapter_config(args.adapter, args.config, "--config"), parts,
331
379
  args.warmups, args.repetitions, args.label, bandwidth_gbs=args.bandwidth, device=device,
332
380
  baseline=baseline, quality_budget=(json_option(args.quality_budget, "--quality-budget")
333
381
  if args.quality_budget else None),
@@ -112,11 +112,40 @@ def draft_step_bytes(model):
112
112
  return int(layers + experts + (draft_head or head or (embedding if config.tie_word_embeddings else 0)))
113
113
 
114
114
 
115
- def max_context(model, device, vram_headroom_bytes, settings, requested=None):
115
+ def companion_drafter_bytes(drafter, model):
116
+ """What a companion drafter (a published draft model in its own folder: EAGLE-3, DFlash, DSpark) costs beside
117
+ MODEL, from the weight headers: `weight_bytes`, every stored tensor but an embedding table (gathered by rows,
118
+ host memory is enough), the device memory it holds as stored; and `pass_bytes`, what one drafter pass reads:
119
+ those weights plus the output head its logits go through, its own where it stores one, else MODEL's (the
120
+ embedding table when MODEL ties its head to it). None without weight headers."""
121
+ shards, _ = weight_files(drafter)
122
+ if not shards:
123
+ return None
124
+ weights = own_head = 0
125
+ for shard in shards:
126
+ for tensor in shard.tensors:
127
+ role = tensor_role(tensor.name, shard.path.name)
128
+ if role.kind in ("embedding", "other_modality"):
129
+ continue
130
+ weights += tensor.nbytes
131
+ own_head += tensor.nbytes if role.kind == "head" else 0
132
+ head = 0
133
+ if not own_head:
134
+ target, _ = weight_files(model)
135
+ roles = [(tensor_role(t.name, s.path.name).kind, t.nbytes) for s in target for t in s.tensors]
136
+ head = sum(size for kind, size in roles if kind == "head")
137
+ if not head and model_config(model).tie_word_embeddings:
138
+ head = sum(size for kind, size in roles if kind == "embedding")
139
+ return {"weight_bytes": int(weights), "pass_bytes": int(weights + head)}
140
+
141
+
142
+ def max_context(model, device, vram_headroom_bytes, settings, requested=None, drafters=()):
116
143
  """The required context of an engine for MODEL on DEVICE (a `crit eval probe` result) with the facts
117
144
  behind it. `tokens` is None when an input is missing (`missing`) or the weights read on every pass do
118
145
  not fit (`reason`). settings: protocol.json's `context` section. requested: the user's wanted context; the
119
- model's positions and the device's memory may cap it (`bound`, and `note` says by how much)."""
146
+ model's positions and the device's memory may cap it (`bound`, and `note` says by how much). drafters: companion
147
+ drafter folders the engine may load beside the model; their stored weights (companion_drafter_bytes) are kept
148
+ out of the KV cache, so a candidate that speculates with one still fits the required context."""
120
149
  config = model_config(model)
121
150
  limit, source = config.position_limit()
122
151
  element = float(settings["kv_bytes_per_element"])
@@ -140,21 +169,32 @@ def max_context(model, device, vram_headroom_bytes, settings, requested=None):
140
169
  "the model's weight headers (safetensors files or a GGUF)")
141
170
  if not free:
142
171
  out["missing"].append("free device memory (re-run crit eval probe)")
172
+ companions = []
173
+ for path in drafters or ():
174
+ found = companion_drafter_bytes(path, model)
175
+ if found is None:
176
+ out["missing"].append(f"the companion drafter's weight headers ({path})")
177
+ else:
178
+ companions.append({"path": str(path), **found})
143
179
  if out["missing"]:
144
180
  return out
145
181
  fixed = config.fixed_state_bytes(element)
146
182
  reserve = int(float(settings["reserve_gib"]) * GIB)
147
- budget = free - int(vram_headroom_bytes) - reserve - split["resident"] - fixed
183
+ drafter_bytes = sum(row["weight_bytes"] for row in companions)
184
+ budget = free - int(vram_headroom_bytes) - reserve - split["resident"] - fixed - drafter_bytes
148
185
  out["basis"] = {"device": (device or {}).get("device"), "memory_free_bytes": free,
149
186
  "memory_total_bytes": (device or {}).get("memory_total_bytes"),
150
187
  "vram_headroom_bytes": int(vram_headroom_bytes), "reserve_bytes": reserve,
151
188
  "resident_weight_bytes": split["resident"], "draft_weight_bytes": split["draft"],
152
189
  "expert_bytes": split["experts"], "embedding_bytes": split["embedding"],
153
190
  "fixed_state_bytes": fixed, "kv_cache_bytes_available": max(0, budget)}
191
+ if companions:
192
+ out["basis"] |= {"companion_drafter_bytes": drafter_bytes, "companion_drafters": companions}
154
193
  if budget <= 0:
155
194
  out["device_limit"] = 0
156
- out["reason"] = (f"the weights read on every pass ({split['resident'] / 1e9:.1f} GB) and fixed state do not fit "
157
- f"the device's {free / 1e9:.1f} GB free memory with {vram_headroom_bytes / 1e9:.1f} GB headroom "
195
+ out["reason"] = (f"the weights read on every pass ({split['resident'] / 1e9:.1f} GB)"
196
+ + (f", the companion drafters ({drafter_bytes / 1e9:.1f} GB)" if drafter_bytes else "")
197
+ + f" and fixed state do not fit the device's {free / 1e9:.1f} GB free memory with {vram_headroom_bytes / 1e9:.1f} GB headroom "
158
198
  f"and a {reserve / 1e9:.1f} GB reserve")
159
199
  return out
160
200
  if kv:
@@ -175,6 +215,95 @@ def max_context(model, device, vram_headroom_bytes, settings, requested=None):
175
215
  return out
176
216
 
177
217
 
218
+ # Where an engine's weights can live, by what fits: the scenario a build starts from (crit recon records it). Lookup
219
+ # tables (gathered embedding and n-gram rows) are a modifier on any scenario: a pass reads a few rows of them, so
220
+ # they live wherever room is left, down to the disk.
221
+ SCENARIOS = {
222
+ "fits_device": (1, "every weight read on a pass fits device memory next to the KV cache at the required context"),
223
+ "sparse_device_host": (2, "routed experts beyond device memory fit host memory"),
224
+ "sparse_device_host_disk": (3, "routed experts beyond device memory exceed host memory: the rest stays on disk"),
225
+ "dense_device_host": (5, "the weights read on every pass do not fit device memory; a pass streams or computes the "
226
+ "rest from host memory"),
227
+ "beyond_host": (None, "the weights read on every pass exceed device and host memory together"),
228
+ }
229
+
230
+
231
+ def link_gbs(device):
232
+ """The device's host-to-device rate in GB/s: its pinned copies at the largest size the probe timed, else None."""
233
+ curve = ((device or {}).get("host_to_device_gbs") or {}).get("pinned") or {}
234
+ return curve[max(curve, key=int)] if curve else None
235
+
236
+
237
+ def placement(model, device, vram_headroom_bytes, ram_headroom_bytes, settings, tokens, drafters=()):
238
+ """The placement scenario of MODEL on DEVICE (a `crit eval probe` result) at `tokens` of required context: which of
239
+ SCENARIOS applies, the bytes each memory tier holds, where the lookup tables go, and the bound one plain decode
240
+ pass (one token, no speculation, routing spread evenly over the experts so a device cache holds its share of them)
241
+ has in each expert source: on the device, misses copied over the host link, misses computed on the CPU from host
242
+ memory. None when Eval cannot size the model (max_context's `missing`)."""
243
+ value = max_context(model, device, vram_headroom_bytes, settings, tokens, drafters)
244
+ basis = value.get("basis")
245
+ if not basis:
246
+ return None
247
+ config = model_config(model)
248
+ resident, experts, tables = basis["resident_weight_bytes"], basis["expert_bytes"], basis["embedding_bytes"]
249
+ room = basis["kv_cache_bytes_available"] - (value.get("kv_bytes_per_token") or 0) * value["tokens"] if value["tokens"] else 0
250
+ total, available = (device or {}).get("host_memory_total_bytes"), (device or {}).get("host_memory_available_bytes")
251
+ host = max(0, (total or available or 0) - int(ram_headroom_bytes)) if (total or available) else None
252
+ on_device = max(0, min(experts, room))
253
+ off_device = experts - on_device
254
+ if not value["tokens"]:
255
+ name = "dense_device_host" if host is None or resident + experts <= host else "beyond_host"
256
+ elif not off_device:
257
+ name = "fits_device"
258
+ else:
259
+ name = "sparse_device_host" if host is None or off_device <= host else "sparse_device_host_disk"
260
+ left = max(0, room - on_device)
261
+ host_left = None if host is None else max(0, host - (off_device if value["tokens"] else resident + experts))
262
+ tier = ("device" if tables <= left else "host" if host_left is None or tables <= host_left else "disk") if tables else None
263
+ moe = config.moe
264
+ share = min(1.0, moe.experts_per_token / moe.experts) if experts and moe and moe.experts and moe.experts_per_token else 0
265
+ active, missed = experts * share, (experts * share * off_device / experts) if experts else 0
266
+ bandwidth, link, memory = (device or {}).get("bandwidth_gbs"), link_gbs(device), ((device or {}).get("host_memory_gbs") or {}).get("read")
267
+ floors = {}
268
+ if bandwidth and value["tokens"]:
269
+ own = (resident - basis["draft_weight_bytes"] + active - missed) / (bandwidth * 1e9) * 1000
270
+ floors["device"] = (resident - basis["draft_weight_bytes"] + active) / (bandwidth * 1e9) * 1000
271
+ if missed and link:
272
+ floors["link"] = max(own, missed / (link * 1e9) * 1000)
273
+ if missed and memory:
274
+ floors["cpu"] = max(own, missed / (memory * 1e9) * 1000)
275
+ return {"scenario": name, "number": SCENARIOS[name][0], "meaning": SCENARIOS[name][1], "tokens": int(tokens),
276
+ "bytes": {"resident": resident, "experts": experts, "experts_on_device": on_device,
277
+ "experts_off_device": off_device, "lookup_tables": tables, "host_memory_for_weights": host},
278
+ "lookup_tables": {"bytes": tables, "tier": tier} if tables else None,
279
+ "active_expert_bytes_per_token": int(active), "missed_expert_bytes_per_token": int(missed),
280
+ "rates_gbs": {"device": bandwidth, "link": link, "host_memory_read": memory},
281
+ "plain_pass_floor_ms": floors,
282
+ "plain_pass_ceiling_tps": {k: round(1000 / v, 1) for k, v in floors.items() if v}}
283
+
284
+
285
+ def render_placement(value):
286
+ if not value:
287
+ return "placement unknown: Eval cannot size the model on this device"
288
+ b, gb = value["bytes"], 1e9
289
+ number = f" ({value['number']})" if value["number"] else ""
290
+ lines = [f"placement {value['scenario']}{number}: {value['meaning']}",
291
+ f" resident weights {b['resident'] / gb:.1f} GB; experts {b['experts'] / gb:.1f} GB, "
292
+ f"{b['experts_on_device'] / gb:.1f} GB on the device, {b['experts_off_device'] / gb:.1f} GB off it"
293
+ + (f"; host memory for weights {b['host_memory_for_weights'] / gb:.1f} GB" if b["host_memory_for_weights"] is not None else "")]
294
+ if value["lookup_tables"]:
295
+ lines.append(f" lookup tables {value['lookup_tables']['bytes'] / gb:.1f} GB (gathered rows) in {value['lookup_tables']['tier']} memory"
296
+ if value["lookup_tables"]["tier"] != "disk" else
297
+ f" lookup tables {value['lookup_tables']['bytes'] / gb:.1f} GB (gathered rows) on disk: read rows on demand, no read-ahead")
298
+ if value["missed_expert_bytes_per_token"]:
299
+ lines.append(f" experts per token {value['active_expert_bytes_per_token'] / gb:.2f} GB, "
300
+ f"{value['missed_expert_bytes_per_token'] / gb:.2f} GB of them off the device at even routing")
301
+ names = {"device": "all on the device", "link": "misses copied over the host link", "cpu": "misses computed on the CPU"}
302
+ for key, tps in value["plain_pass_ceiling_tps"].items():
303
+ lines.append(f" plain pass, {names[key]}: {value['plain_pass_floor_ms'][key]:.2f} ms floor, {tps:,.1f} tok/s")
304
+ return "\n".join(lines)
305
+
306
+
178
307
  def lower_context(required, longest_case_tokens, settings):
179
308
  """The lower context the speed cases are measured again at, or None when no lower context still holds
180
309
  every speed case."""
@@ -290,7 +419,9 @@ def render(value):
290
419
  " device limit none: no cache grows with the context")
291
420
  lines.append(f" {basis['memory_free_bytes'] / 1e9:.2f} GB free on {basis['device']} - {basis['vram_headroom_bytes'] / 1e9:.2f} "
292
421
  f"headroom - {basis['reserve_bytes'] / 1e9:.2f} reserve - {basis['resident_weight_bytes'] / 1e9:.2f} weights read "
293
- f"every pass - {basis['fixed_state_bytes'] / 1e9:.3f} fixed state")
422
+ f"every pass - {basis['fixed_state_bytes'] / 1e9:.3f} fixed state"
423
+ + (f" - {basis['companion_drafter_bytes'] / 1e9:.2f} companion drafters"
424
+ if basis.get("companion_drafter_bytes") else ""))
294
425
  if basis["expert_bytes"]:
295
426
  lines.append(f" experts {basis['expert_bytes'] / 1e9:.2f} GB, "
296
427
  f"{basis.get('expert_bytes_in_device_memory_at_max', 0) / 1e9:.2f} GB of them in device memory at the max")