crit-eval 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {crit_eval-0.2.0 → crit_eval-0.3.0}/PKG-INFO +3 -3
- {crit_eval-0.2.0 → crit_eval-0.3.0}/README.md +2 -2
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/__init__.py +3 -3
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/cli.py +70 -22
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/context.py +137 -6
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/headroom.py +75 -21
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/probe.py +18 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/protocol.json +21 -4
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/fixed.py +33 -5
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/packages.py +12 -21
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/prepare.py +22 -5
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/reference.py +282 -43
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/reports/status.py +65 -6
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/chat.py +5 -2
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/long_context.py +9 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/quality_checks.py +35 -13
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/runner.py +28 -13
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/timing.py +61 -4
- crit_eval-0.3.0/crit_eval/workbench/__init__.py +71 -0
- crit_eval-0.3.0/crit_eval/workbench/harness.py +166 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/PKG-INFO +3 -3
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/SOURCES.txt +4 -1
- {crit_eval-0.2.0 → crit_eval-0.3.0}/pyproject.toml +2 -2
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval.py +325 -8
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_packages.py +17 -17
- crit_eval-0.3.0/tests/test_eval_quality_modes.py +408 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_server.py +5 -2
- crit_eval-0.3.0/tests/test_eval_workbench.py +54 -0
- crit_eval-0.2.0/tests/test_eval_quality_modes.py +0 -240
- {crit_eval-0.2.0 → crit_eval-0.3.0}/LICENSE +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/__main__.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/__init__.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/gguf_deepseek4.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/gguf_layout.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/gguf_qwen4exp.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/architectures/indexers.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/examples/mock_adapter.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/__init__.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/ceiling.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/kernels.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/schemes.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/limits/tensors.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/probe/device_probe.cu +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/probe/kernel_trace.cpp +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/probe/logprob_compare.cu +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/probe/logprob_compare.h +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/__init__.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/attention.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/checkpoints.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/logprob.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/quality/prompt_state.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/reports/__init__.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/reports/compare.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/reports/metrics.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/__init__.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/ab.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/adapters.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/deadline.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/estimate.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/history.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/host.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/hostload.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/outcomes.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/pin.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/run/speed.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/README.md.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/cli.py.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/headroom.py.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/metrics.py.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/prepare.py.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/reference.py.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/report.py.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/runner.py.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/agent/tools.json +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/followup.json +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/heldout.json +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/long.json +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/long_document.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/quality.json +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval/suite/speed.json +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/dependency_links.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/entry_points.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/requires.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/crit_eval.egg-info/top_level.txt +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/setup.cfg +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_chat_template.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_deadline.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_distributions.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_estimate.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_followup.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_formats.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_gguf.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_gguf_deepseek4.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_gguf_qwen4exp.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_indexers.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_kernels.py +0 -0
- {crit_eval-0.2.0 → crit_eval-0.3.0}/tests/test_eval_tensors.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: crit-eval
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: crit-eval: one speed, quality and exactness suite that every inference engine runs the same way
|
|
5
5
|
License-Expression: Apache-2.0
|
|
6
6
|
Project-URL: Repository, https://github.com/crit-labs/crit-eval
|
|
@@ -25,7 +25,7 @@ Dynamic: license-file
|
|
|
25
25
|
|
|
26
26
|
### Speed and quality measurement tool to compare inference engines honestly.
|
|
27
27
|
|
|
28
|
-
[](CHANGELOG.md)
|
|
29
29
|
[](LICENSE)
|
|
30
30
|
[](pyproject.toml)
|
|
31
31
|
|
|
@@ -165,7 +165,7 @@ plugs in through a Python [adapter](docs/ADAPTER.md) instead of `--url`.
|
|
|
165
165
|
| Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages for three checkpoints are published on Hugging Face ([crit-labs/crit-eval-references](https://huggingface.co/datasets/crit-labs/crit-eval-references)) and fetched by default. |
|
|
166
166
|
| Rounding-sensitive models | Where a model amplifies rounding (MoE routing near-ties, sparse-attention ties, recurrent state), two valid FP32 runs of the reference disagree; quality is then judged against twice the reference's own spread instead of the budget, which is looser for those models. |
|
|
167
167
|
| Speed | One request at a time: `crit-eval` measures single-stream decode, prefill and time to first token, not throughput under concurrent requests or batching. A noisy sampled decode is repeated at up to nine seeds per case, so an engine whose speed swings with the drawn text takes longer to measure (`--no-adaptive-sampled` keeps three, an override the result records). |
|
|
168
|
-
| Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded
|
|
168
|
+
| Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded; an engine that declares the bytes it reads from host memory per pass (`host_bytes_per_pass`) gets them priced at those rates, every other engine gets GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
|
|
169
169
|
| Long context | The hub profile measures the long-context curve at its two ends and interpolates the points between; only the full profile measures every point. |
|
|
170
170
|
| Comparability | Results compare only under the same protocol series and profile, on prepared data of the same checkpoint. |
|
|
171
171
|
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
### Speed and quality measurement tool to compare inference engines honestly.
|
|
10
10
|
|
|
11
|
-
[](CHANGELOG.md)
|
|
12
12
|
[](LICENSE)
|
|
13
13
|
[](pyproject.toml)
|
|
14
14
|
|
|
@@ -148,7 +148,7 @@ plugs in through a Python [adapter](docs/ADAPTER.md) instead of `--url`.
|
|
|
148
148
|
| Reference coverage | The reference runs the model through transformers' modeling code in FP32, so it needs an architecture transformers implements and a CUDA GPU with the machine to itself for 5 to 25 minutes. Checkpoints in exl2 or exl3 are refused. Reference packages for three checkpoints are published on Hugging Face ([crit-labs/crit-eval-references](https://huggingface.co/datasets/crit-labs/crit-eval-references)) and fetched by default. |
|
|
149
149
|
| Rounding-sensitive models | Where a model amplifies rounding (MoE routing near-ties, sparse-attention ties, recurrent state), two valid FP32 runs of the reference disagree; quality is then judged against twice the reference's own spread instead of the budget, which is looser for those models. |
|
|
150
150
|
| Speed | One request at a time: `crit-eval` measures single-stream decode, prefill and time to first token, not throughput under concurrent requests or batching. A noisy sampled decode is repeated at up to nine seeds per case, so an engine whose speed swings with the drawn text takes longer to measure (`--no-adaptive-sampled` keeps three, an override the result records). |
|
|
151
|
-
| Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded
|
|
151
|
+
| Hardware | NVIDIA GPUs only: the probe builds with nvcc and measures one GPU. Multi-GPU engines are not measured as such. PCIe, host memory and CPU rates are recorded; an engine that declares the bytes it reads from host memory per pass (`host_bytes_per_pass`) gets them priced at those rates, every other engine gets GPU-only limits. On Windows the CPU probe uses AVX2, which under-reports an AVX-512 CPU. |
|
|
152
152
|
| Long context | The hub profile measures the long-context curve at its two ends and interpolates the points between; only the full profile measures every point. |
|
|
153
153
|
| Comparability | Results compare only under the same protocol series and profile, on prepared data of the same checkpoint. |
|
|
154
154
|
|
|
@@ -7,7 +7,7 @@ from pathlib import Path
|
|
|
7
7
|
# The package holds its suite, protocol, probes and examples; a checkout of crit-eval is its parent.
|
|
8
8
|
EVAL_DIR = Path(__file__).resolve().parent
|
|
9
9
|
SUITE_DIR = EVAL_DIR / "suite"
|
|
10
|
-
__version__ = "0.
|
|
10
|
+
__version__ = "0.3.0"
|
|
11
11
|
CONTRACT = "eval-v1"
|
|
12
12
|
PROTOCOL_ID = re.compile(r"crit-eval-v(\d+)\.(\d+)\.(\d+)")
|
|
13
13
|
# The ids before crit-eval had its own repository (crit-eval-v4, crit-eval-v4.1, ... crit-eval-v8): major and minor.
|
|
@@ -48,7 +48,7 @@ def checkouts():
|
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
def machine_settings():
|
|
51
|
-
"""Memory thresholds from settings.toml (vram_headroom_gib, context_reserve_gib), the wanted context
|
|
51
|
+
"""Memory thresholds from settings.toml (vram_headroom_gib, ram_headroom_gib, context_reserve_gib), the wanted context
|
|
52
52
|
(context_tokens) and the reference package source (eval_packages, crit_eval.quality.packages): crit-eval's checkout's,
|
|
53
53
|
else the crit checkout's (checkouts); without one crit-eval uses the protocol's defaults and
|
|
54
54
|
DEFAULT_CONTEXT_TOKENS."""
|
|
@@ -57,7 +57,7 @@ def machine_settings():
|
|
|
57
57
|
return {}
|
|
58
58
|
import tomllib
|
|
59
59
|
value = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
60
|
-
found = {key: float(value[key]) for key in ("vram_headroom_gib", "context_reserve_gib")
|
|
60
|
+
found = {key: float(value[key]) for key in ("vram_headroom_gib", "ram_headroom_gib", "context_reserve_gib")
|
|
61
61
|
if isinstance(value.get(key), (int, float)) and not isinstance(value.get(key), bool)}
|
|
62
62
|
if type(value.get("context_tokens")) is int and value["context_tokens"] > 0:
|
|
63
63
|
found["context_tokens"] = value["context_tokens"]
|
|
@@ -24,6 +24,17 @@ def json_option(text, option):
|
|
|
24
24
|
raise SystemExit(f"{option} file {path} is not JSON: {error}")
|
|
25
25
|
|
|
26
26
|
|
|
27
|
+
def adapter_config(adapter, text, option):
|
|
28
|
+
"""An adapter's configuration: the configuration.json packaged beside it, the one crit serve opens the candidate
|
|
29
|
+
with, then the option's overrides. An adapter outside a package gets the overrides alone."""
|
|
30
|
+
overrides = json_option(text, option)
|
|
31
|
+
packaged = Path(adapter).parent / "configuration.json" if adapter else None
|
|
32
|
+
if packaged is None or not packaged.is_file():
|
|
33
|
+
return overrides
|
|
34
|
+
print(f"{option}: {packaged} with {len(overrides)} override(s)", file=sys.stderr)
|
|
35
|
+
return {**json_option(str(packaged), option), **overrides}
|
|
36
|
+
|
|
37
|
+
|
|
27
38
|
def main(argv=None):
|
|
28
39
|
if hasattr(sys.stdout, "reconfigure"):
|
|
29
40
|
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
@@ -39,7 +50,7 @@ def main(argv=None):
|
|
|
39
50
|
run.add_argument("--model", required=True, help="Model directory passed to the adapter")
|
|
40
51
|
run.add_argument("--data", required=True, help="Prepared suite directory for this model")
|
|
41
52
|
run.add_argument("--out", required=True, help="Result JSON path")
|
|
42
|
-
run.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
|
|
53
|
+
run.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file, over the configuration.json packaged beside it")
|
|
43
54
|
run.add_argument("--only", choices=list(PARTS), action="append",
|
|
44
55
|
help="Run only these parts (repeatable); default is all. context: the long-context curve and "
|
|
45
56
|
"retrieval up to the required context and the speed cases again at a lower context; "
|
|
@@ -55,7 +66,7 @@ def main(argv=None):
|
|
|
55
66
|
help='Drift limits over the protocol defaults, e.g. \'{"ppl_ratio_max": 1.02}\'; '
|
|
56
67
|
'keys ppl_ratio_max, kld_mean_max, kld_p99_max')
|
|
57
68
|
run.add_argument("--quality-mode", choices=["preserve", "as_quantized"], default="preserve",
|
|
58
|
-
help="preserve (default): quality is judged against the model's own arithmetic with exact activations; as_quantized: against the
|
|
69
|
+
help="preserve (default): quality is judged against the model's own arithmetic with exact activations; as_quantized: against the same scores with each set's budget widened by how far the checkpoint's own activation quantization lands from them (the allowance crit eval reference measures)")
|
|
59
70
|
run.add_argument("--exactness", choices=["reported", "required"],
|
|
60
71
|
help="reported (default): agreement with plain greedy is shown and the quality budget judges "
|
|
61
72
|
"the output; required: any difference fails the gate")
|
|
@@ -94,11 +105,17 @@ def main(argv=None):
|
|
|
94
105
|
context.add_argument("--device", required=True, metavar="JSON", help="Device file from `crit eval probe`")
|
|
95
106
|
context.add_argument("--vram-headroom-gib", type=float, default=None,
|
|
96
107
|
help="Device memory left free (default: settings.toml vram_headroom_gib, else 1)")
|
|
108
|
+
context.add_argument("--ram-headroom-gib", type=float, default=None,
|
|
109
|
+
help="Host memory left free, for the placement scenario (default: settings.toml "
|
|
110
|
+
"ram_headroom_gib, else 2)")
|
|
97
111
|
context.add_argument("--context-reserve-gib", type=float, default=None,
|
|
98
112
|
help="Device memory kept out of the KV cache for workspaces and speed techniques "
|
|
99
113
|
"(default: settings.toml context_reserve_gib, else the protocol's reserve_gib)")
|
|
100
114
|
context.add_argument("--requested-context", type=int, metavar="TOKENS", default=None,
|
|
101
115
|
help="The longest request wanted (default: settings.toml context_tokens, else 262,144)")
|
|
116
|
+
context.add_argument("--drafter", action="append", default=[], metavar="DIR",
|
|
117
|
+
help="A companion drafter the engine may load beside the model; its stored weights are kept "
|
|
118
|
+
"out of the KV cache (repeatable)")
|
|
102
119
|
context.add_argument("--json", action="store_true", help="Print the computation as JSON")
|
|
103
120
|
|
|
104
121
|
probe = subs.add_parser("probe", help="Measure device bandwidth and GEMM rates per weight format (needs nvcc)")
|
|
@@ -131,10 +148,10 @@ def main(argv=None):
|
|
|
131
148
|
"transformers, GPU; stop any engine first)")
|
|
132
149
|
reference.add_argument("--model", required=True, help="The checkpoint the data was prepared from")
|
|
133
150
|
reference.add_argument("--data", required=True, help="Prepared suite directory")
|
|
134
|
-
reference.add_argument("--quality-mode", choices=["preserve", "as_quantized"], default=
|
|
135
|
-
help="
|
|
136
|
-
"
|
|
137
|
-
"as_quantized")
|
|
151
|
+
reference.add_argument("--quality-mode", choices=["preserve", "as_quantized"], default=None,
|
|
152
|
+
help="Accepted from earlier versions and ignored: one run scores what both quality modes "
|
|
153
|
+
"need (the scores, their spread and, for a checkpoint that declares quantized "
|
|
154
|
+
"activations, the as_quantized allowance)")
|
|
138
155
|
|
|
139
156
|
fetch = subs.add_parser("fetch", help="Fetch the checkpoint's reference package (prepared data and reference "
|
|
140
157
|
"scores) into the data, so a run needs only the engine")
|
|
@@ -157,14 +174,14 @@ def main(argv=None):
|
|
|
157
174
|
pin.add_argument("--adapter", required=True, help="Python file exporting create_eval_engine(context)")
|
|
158
175
|
pin.add_argument("--model", required=True, help="Model directory passed to the adapter")
|
|
159
176
|
pin.add_argument("--data", required=True, help="Prepared data directory to pin the continuation in")
|
|
160
|
-
pin.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
|
|
177
|
+
pin.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file, over the configuration.json packaged beside it")
|
|
161
178
|
pin.add_argument("--context", type=int, metavar="TOKENS", help="Context the engine is built for (the run's kv_tokens)")
|
|
162
179
|
|
|
163
180
|
ab = subs.add_parser("ab", help="Alternate two engines' speed runs in one session and report paired ratios")
|
|
164
181
|
ab.add_argument("--adapter", required=True, help="Adapter of engine A")
|
|
165
|
-
ab.add_argument("--config", default="{}", help="JSON configuration of engine A (text or a file)")
|
|
182
|
+
ab.add_argument("--config", default="{}", help="JSON configuration overrides of engine A (text or a file), over its packaged configuration.json")
|
|
166
183
|
ab.add_argument("--adapter-b", help="Adapter of engine B (default: engine A's adapter)")
|
|
167
|
-
ab.add_argument("--config-b", default="{}", help="JSON configuration of engine B (text or a file)")
|
|
184
|
+
ab.add_argument("--config-b", default="{}", help="JSON configuration overrides of engine B (text or a file), over its packaged configuration.json")
|
|
168
185
|
ab.add_argument("--model", required=True)
|
|
169
186
|
ab.add_argument("--data", required=True)
|
|
170
187
|
ab.add_argument("--out", required=True, help="Result JSON path")
|
|
@@ -179,7 +196,7 @@ def main(argv=None):
|
|
|
179
196
|
kernels.add_argument("--model", required=True, help="Model directory passed to the adapter")
|
|
180
197
|
kernels.add_argument("--data", required=True, help="Prepared suite directory for this model")
|
|
181
198
|
kernels.add_argument("--out", required=True, help="Kernel report JSON path")
|
|
182
|
-
kernels.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file")
|
|
199
|
+
kernels.add_argument("--config", default="{}", help="JSON configuration overrides for the adapter, text or a file, over the configuration.json packaged beside it")
|
|
183
200
|
kernels.add_argument("--case", action="append", help="Trace only these speed cases (repeatable); default all")
|
|
184
201
|
kernels.add_argument("--device", metavar="JSON", help="Device ceilings from `crit eval probe` for share of peak")
|
|
185
202
|
kernels.add_argument("--nvcc", help="nvcc of a CUDA toolkit that ships CUPTI")
|
|
@@ -189,6 +206,17 @@ def main(argv=None):
|
|
|
189
206
|
"other|draft, \"format\": nvfp4|fp8|bf16, ...}]")
|
|
190
207
|
kernels.add_argument("--context", type=int, metavar="TOKENS", help="Context the engine is built for (the run's kv_tokens)")
|
|
191
208
|
|
|
209
|
+
workbench = subs.add_parser("workbench", help="Check, time or profile a kernel on its workbench folder "
|
|
210
|
+
"(crit kernels new): reference, floor, yardstick")
|
|
211
|
+
workbench.add_argument("action", choices=("check", "bench", "profile"))
|
|
212
|
+
workbench.add_argument("folder", help="The workbench folder")
|
|
213
|
+
workbench.add_argument("--python", help="The engine's Python (torch and the GPU); default this one")
|
|
214
|
+
workbench.add_argument("--shape", type=int, help="One shape of kernel.json by index (profile: default the first)")
|
|
215
|
+
workbench.add_argument("--iters", type=int, help="bench: timed calls per shape and column count")
|
|
216
|
+
workbench.add_argument("--device", metavar="JSON", help="bench: a `crit eval probe` result, for the floor")
|
|
217
|
+
workbench.add_argument("--out", help="check and bench: the result JSON (default FOLDER/check.json, bench.json)")
|
|
218
|
+
workbench.add_argument("--ncu", help="profile: Nsight Compute's ncu, to add its sections")
|
|
219
|
+
|
|
192
220
|
args = parser.parse_args(argv)
|
|
193
221
|
# An engine behind a server is the server's to load (crit serve holds the lock while it serves).
|
|
194
222
|
if args.command in ENGINE_COMMANDS and not getattr(args, "url", None):
|
|
@@ -204,6 +232,17 @@ ENGINE_COMMANDS = {"run", "pin", "ab", "kernels", "reference"}
|
|
|
204
232
|
|
|
205
233
|
|
|
206
234
|
def _command(args):
|
|
235
|
+
if args.command == "workbench":
|
|
236
|
+
from .workbench import profile, run_harness
|
|
237
|
+
if args.action == "profile":
|
|
238
|
+
result = profile(args.folder, args.python, args.ncu, args.shape or 0)
|
|
239
|
+
print(json.dumps(result, indent=2))
|
|
240
|
+
return 0
|
|
241
|
+
options = {"iters": args.iters, "device": args.device, "out": args.out, "shape": args.shape} if args.action == "bench" else {"out": args.out, "shape": args.shape}
|
|
242
|
+
code, result = run_harness(args.folder, args.action, args.python, **options)
|
|
243
|
+
if result is None:
|
|
244
|
+
raise SystemExit(f"the workbench harness exited {code} without a result")
|
|
245
|
+
return 0 if code == 0 else 1
|
|
207
246
|
if args.command == "kernels":
|
|
208
247
|
from . import load_json as load
|
|
209
248
|
from .limits.kernels import render as render_kernels, trace
|
|
@@ -212,7 +251,7 @@ def _command(args):
|
|
|
212
251
|
from .quality.prepare import prepare as do_prepare
|
|
213
252
|
do_prepare(args.model, args.data)
|
|
214
253
|
work = json_option(args.work, "--work") if args.work else None
|
|
215
|
-
result = trace(args.adapter, args.model, args.data, args.out,
|
|
254
|
+
result = trace(args.adapter, args.model, args.data, args.out, adapter_config(args.adapter, args.config, "--config"), args.case,
|
|
216
255
|
args.nvcc,
|
|
217
256
|
load(args.device) if args.device else None, context_tokens=args.context, work=work)
|
|
218
257
|
print(f"wrote {args.out}")
|
|
@@ -234,12 +273,14 @@ def _command(args):
|
|
|
234
273
|
return 0
|
|
235
274
|
if args.command == "reference":
|
|
236
275
|
from .quality.reference import score_data
|
|
237
|
-
from .quality.fixed import
|
|
238
|
-
how = score_data(args.model, args.data, log=lambda line: print(line, flush=True)
|
|
239
|
-
print(f"reference scores of {how['positions']:,} positions {how['method']} in "
|
|
240
|
-
f"{Path(args.data) / STORES[args.quality_mode]}"
|
|
276
|
+
from .quality.fixed import STORE
|
|
277
|
+
how = score_data(args.model, args.data, log=lambda line: print(line, flush=True))
|
|
278
|
+
print(f"reference scores of {how['positions']:,} positions {how['method']} in {Path(args.data) / STORE}"
|
|
241
279
|
+ (f" ({how['seconds']:.0f} s, {how['bytes'] / 2**20:.1f} MiB)" if how["method"] == "scored" else "")
|
|
242
|
-
+ (f"; the reference's own spread
|
|
280
|
+
+ (f"; the reference's own spread {how['spread']}" if how.get("spread") else "")
|
|
281
|
+
+ {"none": "; the checkpoint declares no quantized activations (as_quantized judges as preserve)",
|
|
282
|
+
"unsupported": "; the as_quantized allowance not measured (see above)"}.get(
|
|
283
|
+
how.get("allowance"), f"; the as_quantized allowance {how['allowance']}" if how.get("allowance") else ""))
|
|
243
284
|
return 0
|
|
244
285
|
if args.command == "fetch":
|
|
245
286
|
from .quality.packages import fetch as do_fetch, source
|
|
@@ -266,8 +307,15 @@ def _command(args):
|
|
|
266
307
|
from .limits.context import GIB, max_context, render as render_context
|
|
267
308
|
from . import EVAL_DIR
|
|
268
309
|
settings = context_settings(load(EVAL_DIR / "protocol.json")["context"], args)
|
|
269
|
-
value = max_context(args.model, load(args.device), headroom_gib(args) * GIB, settings, requested(args)
|
|
270
|
-
|
|
310
|
+
value = max_context(args.model, load(args.device), headroom_gib(args) * GIB, settings, requested(args),
|
|
311
|
+
drafters=args.drafter)
|
|
312
|
+
from .limits.context import placement, render_placement
|
|
313
|
+
from . import machine_settings
|
|
314
|
+
ram = args.ram_headroom_gib if args.ram_headroom_gib is not None else machine_settings().get("ram_headroom_gib", 2.0)
|
|
315
|
+
where = placement(args.model, load(args.device), headroom_gib(args) * GIB, ram * GIB, settings, requested(args),
|
|
316
|
+
drafters=args.drafter)
|
|
317
|
+
print(json.dumps({**value, "placement": where}, indent=1) if args.json
|
|
318
|
+
else render_context(value) + "\n" + render_placement(where))
|
|
271
319
|
return 0 if value["tokens"] else 1
|
|
272
320
|
if args.command == "compare":
|
|
273
321
|
from .reports.compare import compare as do_compare
|
|
@@ -279,14 +327,14 @@ def _command(args):
|
|
|
279
327
|
if not (Path(args.data) / PREPARED).is_file():
|
|
280
328
|
from .quality.prepare import prepare as do_prepare
|
|
281
329
|
do_prepare(args.model, args.data)
|
|
282
|
-
pinned = do_pin(args.adapter, args.model, args.data,
|
|
330
|
+
pinned = do_pin(args.adapter, args.model, args.data, adapter_config(args.adapter, args.config, "--config"),
|
|
283
331
|
context_tokens=args.context)
|
|
284
332
|
print(f"pinned {len(pinned['cases'])} speed cases in {Path(args.data) / CONTINUATIONS} (sha256 {pinned['sha256'][:12]})")
|
|
285
333
|
return 0
|
|
286
334
|
if args.command == "ab":
|
|
287
335
|
from .run.ab import render as render_ab, run_ab
|
|
288
|
-
result = run_ab(args.adapter,
|
|
289
|
-
|
|
336
|
+
result = run_ab(args.adapter, adapter_config(args.adapter, args.config, "--config"), args.adapter_b or args.adapter,
|
|
337
|
+
adapter_config(args.adapter_b or args.adapter, args.config_b, "--config-b"),
|
|
290
338
|
args.model, args.data, args.out, args.rounds, args.warmups, args.repetitions,
|
|
291
339
|
context_tokens=args.context)
|
|
292
340
|
print(f"wrote {args.out}")
|
|
@@ -327,7 +375,7 @@ def _command(args):
|
|
|
327
375
|
from .reports.status import render
|
|
328
376
|
device = load(args.device) if args.device else None
|
|
329
377
|
baseline = load(args.baseline) if args.baseline else None
|
|
330
|
-
result = do_run(args.adapter, args.model, data, args.out,
|
|
378
|
+
result = do_run(args.adapter, args.model, data, args.out, adapter_config(args.adapter, args.config, "--config"), parts,
|
|
331
379
|
args.warmups, args.repetitions, args.label, bandwidth_gbs=args.bandwidth, device=device,
|
|
332
380
|
baseline=baseline, quality_budget=(json_option(args.quality_budget, "--quality-budget")
|
|
333
381
|
if args.quality_budget else None),
|
|
@@ -112,11 +112,40 @@ def draft_step_bytes(model):
|
|
|
112
112
|
return int(layers + experts + (draft_head or head or (embedding if config.tie_word_embeddings else 0)))
|
|
113
113
|
|
|
114
114
|
|
|
115
|
-
def
|
|
115
|
+
def companion_drafter_bytes(drafter, model):
|
|
116
|
+
"""What a companion drafter (a published draft model in its own folder: EAGLE-3, DFlash, DSpark) costs beside
|
|
117
|
+
MODEL, from the weight headers: `weight_bytes`, every stored tensor but an embedding table (gathered by rows,
|
|
118
|
+
host memory is enough), the device memory it holds as stored; and `pass_bytes`, what one drafter pass reads:
|
|
119
|
+
those weights plus the output head its logits go through, its own where it stores one, else MODEL's (the
|
|
120
|
+
embedding table when MODEL ties its head to it). None without weight headers."""
|
|
121
|
+
shards, _ = weight_files(drafter)
|
|
122
|
+
if not shards:
|
|
123
|
+
return None
|
|
124
|
+
weights = own_head = 0
|
|
125
|
+
for shard in shards:
|
|
126
|
+
for tensor in shard.tensors:
|
|
127
|
+
role = tensor_role(tensor.name, shard.path.name)
|
|
128
|
+
if role.kind in ("embedding", "other_modality"):
|
|
129
|
+
continue
|
|
130
|
+
weights += tensor.nbytes
|
|
131
|
+
own_head += tensor.nbytes if role.kind == "head" else 0
|
|
132
|
+
head = 0
|
|
133
|
+
if not own_head:
|
|
134
|
+
target, _ = weight_files(model)
|
|
135
|
+
roles = [(tensor_role(t.name, s.path.name).kind, t.nbytes) for s in target for t in s.tensors]
|
|
136
|
+
head = sum(size for kind, size in roles if kind == "head")
|
|
137
|
+
if not head and model_config(model).tie_word_embeddings:
|
|
138
|
+
head = sum(size for kind, size in roles if kind == "embedding")
|
|
139
|
+
return {"weight_bytes": int(weights), "pass_bytes": int(weights + head)}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def max_context(model, device, vram_headroom_bytes, settings, requested=None, drafters=()):
|
|
116
143
|
"""The required context of an engine for MODEL on DEVICE (a `crit eval probe` result) with the facts
|
|
117
144
|
behind it. `tokens` is None when an input is missing (`missing`) or the weights read on every pass do
|
|
118
145
|
not fit (`reason`). settings: protocol.json's `context` section. requested: the user's wanted context; the
|
|
119
|
-
model's positions and the device's memory may cap it (`bound`, and `note` says by how much).
|
|
146
|
+
model's positions and the device's memory may cap it (`bound`, and `note` says by how much). drafters: companion
|
|
147
|
+
drafter folders the engine may load beside the model; their stored weights (companion_drafter_bytes) are kept
|
|
148
|
+
out of the KV cache, so a candidate that speculates with one still fits the required context."""
|
|
120
149
|
config = model_config(model)
|
|
121
150
|
limit, source = config.position_limit()
|
|
122
151
|
element = float(settings["kv_bytes_per_element"])
|
|
@@ -140,21 +169,32 @@ def max_context(model, device, vram_headroom_bytes, settings, requested=None):
|
|
|
140
169
|
"the model's weight headers (safetensors files or a GGUF)")
|
|
141
170
|
if not free:
|
|
142
171
|
out["missing"].append("free device memory (re-run crit eval probe)")
|
|
172
|
+
companions = []
|
|
173
|
+
for path in drafters or ():
|
|
174
|
+
found = companion_drafter_bytes(path, model)
|
|
175
|
+
if found is None:
|
|
176
|
+
out["missing"].append(f"the companion drafter's weight headers ({path})")
|
|
177
|
+
else:
|
|
178
|
+
companions.append({"path": str(path), **found})
|
|
143
179
|
if out["missing"]:
|
|
144
180
|
return out
|
|
145
181
|
fixed = config.fixed_state_bytes(element)
|
|
146
182
|
reserve = int(float(settings["reserve_gib"]) * GIB)
|
|
147
|
-
|
|
183
|
+
drafter_bytes = sum(row["weight_bytes"] for row in companions)
|
|
184
|
+
budget = free - int(vram_headroom_bytes) - reserve - split["resident"] - fixed - drafter_bytes
|
|
148
185
|
out["basis"] = {"device": (device or {}).get("device"), "memory_free_bytes": free,
|
|
149
186
|
"memory_total_bytes": (device or {}).get("memory_total_bytes"),
|
|
150
187
|
"vram_headroom_bytes": int(vram_headroom_bytes), "reserve_bytes": reserve,
|
|
151
188
|
"resident_weight_bytes": split["resident"], "draft_weight_bytes": split["draft"],
|
|
152
189
|
"expert_bytes": split["experts"], "embedding_bytes": split["embedding"],
|
|
153
190
|
"fixed_state_bytes": fixed, "kv_cache_bytes_available": max(0, budget)}
|
|
191
|
+
if companions:
|
|
192
|
+
out["basis"] |= {"companion_drafter_bytes": drafter_bytes, "companion_drafters": companions}
|
|
154
193
|
if budget <= 0:
|
|
155
194
|
out["device_limit"] = 0
|
|
156
|
-
out["reason"] = (f"the weights read on every pass ({split['resident'] / 1e9:.1f} GB)
|
|
157
|
-
f"the
|
|
195
|
+
out["reason"] = (f"the weights read on every pass ({split['resident'] / 1e9:.1f} GB)"
|
|
196
|
+
+ (f", the companion drafters ({drafter_bytes / 1e9:.1f} GB)" if drafter_bytes else "")
|
|
197
|
+
+ f" and fixed state do not fit the device's {free / 1e9:.1f} GB free memory with {vram_headroom_bytes / 1e9:.1f} GB headroom "
|
|
158
198
|
f"and a {reserve / 1e9:.1f} GB reserve")
|
|
159
199
|
return out
|
|
160
200
|
if kv:
|
|
@@ -175,6 +215,95 @@ def max_context(model, device, vram_headroom_bytes, settings, requested=None):
|
|
|
175
215
|
return out
|
|
176
216
|
|
|
177
217
|
|
|
218
|
+
# Where an engine's weights can live, by what fits: the scenario a build starts from (crit recon records it). Lookup
|
|
219
|
+
# tables (gathered embedding and n-gram rows) are a modifier on any scenario: a pass reads a few rows of them, so
|
|
220
|
+
# they live wherever room is left, down to the disk.
|
|
221
|
+
SCENARIOS = {
|
|
222
|
+
"fits_device": (1, "every weight read on a pass fits device memory next to the KV cache at the required context"),
|
|
223
|
+
"sparse_device_host": (2, "routed experts beyond device memory fit host memory"),
|
|
224
|
+
"sparse_device_host_disk": (3, "routed experts beyond device memory exceed host memory: the rest stays on disk"),
|
|
225
|
+
"dense_device_host": (5, "the weights read on every pass do not fit device memory; a pass streams or computes the "
|
|
226
|
+
"rest from host memory"),
|
|
227
|
+
"beyond_host": (None, "the weights read on every pass exceed device and host memory together"),
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def link_gbs(device):
|
|
232
|
+
"""The device's host-to-device rate in GB/s: its pinned copies at the largest size the probe timed, else None."""
|
|
233
|
+
curve = ((device or {}).get("host_to_device_gbs") or {}).get("pinned") or {}
|
|
234
|
+
return curve[max(curve, key=int)] if curve else None
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def placement(model, device, vram_headroom_bytes, ram_headroom_bytes, settings, tokens, drafters=()):
|
|
238
|
+
"""The placement scenario of MODEL on DEVICE (a `crit eval probe` result) at `tokens` of required context: which of
|
|
239
|
+
SCENARIOS applies, the bytes each memory tier holds, where the lookup tables go, and the bound one plain decode
|
|
240
|
+
pass (one token, no speculation, routing spread evenly over the experts so a device cache holds its share of them)
|
|
241
|
+
has in each expert source: on the device, misses copied over the host link, misses computed on the CPU from host
|
|
242
|
+
memory. None when Eval cannot size the model (max_context's `missing`)."""
|
|
243
|
+
value = max_context(model, device, vram_headroom_bytes, settings, tokens, drafters)
|
|
244
|
+
basis = value.get("basis")
|
|
245
|
+
if not basis:
|
|
246
|
+
return None
|
|
247
|
+
config = model_config(model)
|
|
248
|
+
resident, experts, tables = basis["resident_weight_bytes"], basis["expert_bytes"], basis["embedding_bytes"]
|
|
249
|
+
room = basis["kv_cache_bytes_available"] - (value.get("kv_bytes_per_token") or 0) * value["tokens"] if value["tokens"] else 0
|
|
250
|
+
total, available = (device or {}).get("host_memory_total_bytes"), (device or {}).get("host_memory_available_bytes")
|
|
251
|
+
host = max(0, (total or available or 0) - int(ram_headroom_bytes)) if (total or available) else None
|
|
252
|
+
on_device = max(0, min(experts, room))
|
|
253
|
+
off_device = experts - on_device
|
|
254
|
+
if not value["tokens"]:
|
|
255
|
+
name = "dense_device_host" if host is None or resident + experts <= host else "beyond_host"
|
|
256
|
+
elif not off_device:
|
|
257
|
+
name = "fits_device"
|
|
258
|
+
else:
|
|
259
|
+
name = "sparse_device_host" if host is None or off_device <= host else "sparse_device_host_disk"
|
|
260
|
+
left = max(0, room - on_device)
|
|
261
|
+
host_left = None if host is None else max(0, host - (off_device if value["tokens"] else resident + experts))
|
|
262
|
+
tier = ("device" if tables <= left else "host" if host_left is None or tables <= host_left else "disk") if tables else None
|
|
263
|
+
moe = config.moe
|
|
264
|
+
share = min(1.0, moe.experts_per_token / moe.experts) if experts and moe and moe.experts and moe.experts_per_token else 0
|
|
265
|
+
active, missed = experts * share, (experts * share * off_device / experts) if experts else 0
|
|
266
|
+
bandwidth, link, memory = (device or {}).get("bandwidth_gbs"), link_gbs(device), ((device or {}).get("host_memory_gbs") or {}).get("read")
|
|
267
|
+
floors = {}
|
|
268
|
+
if bandwidth and value["tokens"]:
|
|
269
|
+
own = (resident - basis["draft_weight_bytes"] + active - missed) / (bandwidth * 1e9) * 1000
|
|
270
|
+
floors["device"] = (resident - basis["draft_weight_bytes"] + active) / (bandwidth * 1e9) * 1000
|
|
271
|
+
if missed and link:
|
|
272
|
+
floors["link"] = max(own, missed / (link * 1e9) * 1000)
|
|
273
|
+
if missed and memory:
|
|
274
|
+
floors["cpu"] = max(own, missed / (memory * 1e9) * 1000)
|
|
275
|
+
return {"scenario": name, "number": SCENARIOS[name][0], "meaning": SCENARIOS[name][1], "tokens": int(tokens),
|
|
276
|
+
"bytes": {"resident": resident, "experts": experts, "experts_on_device": on_device,
|
|
277
|
+
"experts_off_device": off_device, "lookup_tables": tables, "host_memory_for_weights": host},
|
|
278
|
+
"lookup_tables": {"bytes": tables, "tier": tier} if tables else None,
|
|
279
|
+
"active_expert_bytes_per_token": int(active), "missed_expert_bytes_per_token": int(missed),
|
|
280
|
+
"rates_gbs": {"device": bandwidth, "link": link, "host_memory_read": memory},
|
|
281
|
+
"plain_pass_floor_ms": floors,
|
|
282
|
+
"plain_pass_ceiling_tps": {k: round(1000 / v, 1) for k, v in floors.items() if v}}
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def render_placement(value):
|
|
286
|
+
if not value:
|
|
287
|
+
return "placement unknown: Eval cannot size the model on this device"
|
|
288
|
+
b, gb = value["bytes"], 1e9
|
|
289
|
+
number = f" ({value['number']})" if value["number"] else ""
|
|
290
|
+
lines = [f"placement {value['scenario']}{number}: {value['meaning']}",
|
|
291
|
+
f" resident weights {b['resident'] / gb:.1f} GB; experts {b['experts'] / gb:.1f} GB, "
|
|
292
|
+
f"{b['experts_on_device'] / gb:.1f} GB on the device, {b['experts_off_device'] / gb:.1f} GB off it"
|
|
293
|
+
+ (f"; host memory for weights {b['host_memory_for_weights'] / gb:.1f} GB" if b["host_memory_for_weights"] is not None else "")]
|
|
294
|
+
if value["lookup_tables"]:
|
|
295
|
+
lines.append(f" lookup tables {value['lookup_tables']['bytes'] / gb:.1f} GB (gathered rows) in {value['lookup_tables']['tier']} memory"
|
|
296
|
+
if value["lookup_tables"]["tier"] != "disk" else
|
|
297
|
+
f" lookup tables {value['lookup_tables']['bytes'] / gb:.1f} GB (gathered rows) on disk: read rows on demand, no read-ahead")
|
|
298
|
+
if value["missed_expert_bytes_per_token"]:
|
|
299
|
+
lines.append(f" experts per token {value['active_expert_bytes_per_token'] / gb:.2f} GB, "
|
|
300
|
+
f"{value['missed_expert_bytes_per_token'] / gb:.2f} GB of them off the device at even routing")
|
|
301
|
+
names = {"device": "all on the device", "link": "misses copied over the host link", "cpu": "misses computed on the CPU"}
|
|
302
|
+
for key, tps in value["plain_pass_ceiling_tps"].items():
|
|
303
|
+
lines.append(f" plain pass, {names[key]}: {value['plain_pass_floor_ms'][key]:.2f} ms floor, {tps:,.1f} tok/s")
|
|
304
|
+
return "\n".join(lines)
|
|
305
|
+
|
|
306
|
+
|
|
178
307
|
def lower_context(required, longest_case_tokens, settings):
|
|
179
308
|
"""The lower context the speed cases are measured again at, or None when no lower context still holds
|
|
180
309
|
every speed case."""
|
|
@@ -290,7 +419,9 @@ def render(value):
|
|
|
290
419
|
" device limit none: no cache grows with the context")
|
|
291
420
|
lines.append(f" {basis['memory_free_bytes'] / 1e9:.2f} GB free on {basis['device']} - {basis['vram_headroom_bytes'] / 1e9:.2f} "
|
|
292
421
|
f"headroom - {basis['reserve_bytes'] / 1e9:.2f} reserve - {basis['resident_weight_bytes'] / 1e9:.2f} weights read "
|
|
293
|
-
f"every pass - {basis['fixed_state_bytes'] / 1e9:.3f} fixed state"
|
|
422
|
+
f"every pass - {basis['fixed_state_bytes'] / 1e9:.3f} fixed state"
|
|
423
|
+
+ (f" - {basis['companion_drafter_bytes'] / 1e9:.2f} companion drafters"
|
|
424
|
+
if basis.get("companion_drafter_bytes") else ""))
|
|
294
425
|
if basis["expert_bytes"]:
|
|
295
426
|
lines.append(f" experts {basis['expert_bytes'] / 1e9:.2f} GB, "
|
|
296
427
|
f"{basis.get('expert_bytes_in_device_memory_at_max', 0) / 1e9:.2f} GB of them in device memory at the max")
|