quf 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- quf/__init__.py +28 -0
- quf/bench/__init__.py +1 -0
- quf/bench/lm_eval_adapter.py +203 -0
- quf/bench/suite.py +306 -0
- quf/cli.py +235 -0
- quf/convert/__init__.py +1 -0
- quf/convert/converter.py +280 -0
- quf/convert/gptq.py +109 -0
- quf/device.py +41 -0
- quf/format/__init__.py +1 -0
- quf/format/constants.py +24 -0
- quf/format/errors.py +39 -0
- quf/format/reader.py +136 -0
- quf/format/schema.py +112 -0
- quf/format/writer.py +106 -0
- quf/quant/__init__.py +1 -0
- quf/quant/dequant.py +46 -0
- quf/quant/gptq.py +384 -0
- quf/quant/pack.py +41 -0
- quf/quant/quantize.py +106 -0
- quf/runtime/__init__.py +1 -0
- quf/runtime/backends/__init__.py +1 -0
- quf/runtime/backends/base.py +22 -0
- quf/runtime/backends/cpu.py +31 -0
- quf/runtime/backends/marlin.py +159 -0
- quf/runtime/backends/torch_fused.py +161 -0
- quf/runtime/backends/torch_gpu.py +42 -0
- quf/runtime/errors.py +20 -0
- quf/runtime/generate.py +113 -0
- quf/runtime/linear.py +33 -0
- quf/runtime/model.py +316 -0
- quf/runtime/selector.py +98 -0
- quf/tokenization/__init__.py +6 -0
- quf/tokenization/calibration.py +54 -0
- quf/tokenization/wrapper.py +83 -0
- quf-0.1.0.data/data/share/doc/quf/ENV.md +38 -0
- quf-0.1.0.data/data/share/doc/quf/FORMAT.md +173 -0
- quf-0.1.0.data/data/share/doc/quf/README.md +137 -0
- quf-0.1.0.dist-info/METADATA +177 -0
- quf-0.1.0.dist-info/RECORD +44 -0
- quf-0.1.0.dist-info/WHEEL +5 -0
- quf-0.1.0.dist-info/entry_points.txt +2 -0
- quf-0.1.0.dist-info/licenses/LICENSE +21 -0
- quf-0.1.0.dist-info/top_level.txt +1 -0
quf/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""QUF: Quantized Unified Format.
|
|
2
|
+
|
|
3
|
+
Importing this package never requires optional extras (torch, gptqmodel, ...).
|
|
4
|
+
Submodules are imported lazily on first attribute access.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import importlib
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
__version__ = "0.1.0"
|
|
13
|
+
|
|
14
|
+
_LAZY_SUBMODULES = frozenset(
|
|
15
|
+
{"device", "format", "quant", "tokenization", "convert", "runtime", "bench", "cli"}
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__all__ = ["__version__", *sorted(_LAZY_SUBMODULES)]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def __getattr__(name: str) -> Any:
|
|
22
|
+
if name in _LAZY_SUBMODULES:
|
|
23
|
+
return importlib.import_module(f"{__name__}.{name}")
|
|
24
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def __dir__() -> list[str]:
|
|
28
|
+
return sorted(set(globals()) | _LAZY_SUBMODULES)
|
quf/bench/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Benchmark_Suite: perplexity, size, speed, and results JSON."""
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""lm-evaluation-harness adapter for QUF files.
|
|
2
|
+
|
|
3
|
+
``lm_eval`` is imported lazily, so importing this module never requires it.
|
|
4
|
+
The adapter class ``QUFLM`` subclasses ``lm_eval.api.model.LM`` and is built on
|
|
5
|
+
first attribute access (``from quf.bench.lm_eval_adapter import QUFLM``) or via
|
|
6
|
+
:func:`get_adapter_class`. It is registered with the harness as ``"quf"``::
|
|
7
|
+
|
|
8
|
+
lm_eval --model quf --model_args path=model.quf,device=cpu --tasks hellaswag
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import torch
|
|
16
|
+
import torch.nn.functional as F
|
|
17
|
+
|
|
18
|
+
_LM_CLASS: type | None = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _import_lm_base():
|
|
22
|
+
try:
|
|
23
|
+
from lm_eval.api.model import LM
|
|
24
|
+
except ImportError as exc:
|
|
25
|
+
raise ImportError(
|
|
26
|
+
"lm-evaluation-harness is required for the QUF adapter. "
|
|
27
|
+
"Install it with: pip install lm_eval"
|
|
28
|
+
) from exc
|
|
29
|
+
return LM
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _eos_id(model) -> int | None:
|
|
33
|
+
eos = (model.header.tokenizer or {}).get("eos_token_id")
|
|
34
|
+
return int(eos) if eos is not None else None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _stop_at(text: str, until: list[str]) -> str:
|
|
38
|
+
cut = len(text)
|
|
39
|
+
for s in until:
|
|
40
|
+
if s:
|
|
41
|
+
idx = text.find(s)
|
|
42
|
+
if idx != -1:
|
|
43
|
+
cut = min(cut, idx)
|
|
44
|
+
return text[:cut]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def get_adapter_class() -> type:
|
|
48
|
+
"""Return the ``QUFLM`` class, importing ``lm_eval`` on first use."""
|
|
49
|
+
global _LM_CLASS
|
|
50
|
+
if _LM_CLASS is not None:
|
|
51
|
+
return _LM_CLASS
|
|
52
|
+
LM = _import_lm_base()
|
|
53
|
+
|
|
54
|
+
class QUFLM(LM):
|
|
55
|
+
"""Evaluate a QUF_File with lm-evaluation-harness."""
|
|
56
|
+
|
|
57
|
+
def __init__(
|
|
58
|
+
self,
|
|
59
|
+
path: str | None = None,
|
|
60
|
+
model: Any = None,
|
|
61
|
+
tokenizer: Any = None,
|
|
62
|
+
device: str = "cpu",
|
|
63
|
+
backend: str | None = None,
|
|
64
|
+
tokenizer_path: str | None = None,
|
|
65
|
+
max_gen_toks: int = 256,
|
|
66
|
+
**kwargs: Any,
|
|
67
|
+
) -> None:
|
|
68
|
+
super().__init__()
|
|
69
|
+
from quf.runtime.model import QUFModel
|
|
70
|
+
from quf.tokenization.wrapper import TokenizerWrapper
|
|
71
|
+
|
|
72
|
+
if model is None:
|
|
73
|
+
if path is None:
|
|
74
|
+
raise ValueError("Provide 'path' to a .quf file or a loaded 'model'")
|
|
75
|
+
model = QUFModel.load(path, device_option=device, backend=backend)
|
|
76
|
+
if tokenizer is None:
|
|
77
|
+
tok_src = tokenizer_path or (model.header.tokenizer or {}).get("path")
|
|
78
|
+
if tok_src is None:
|
|
79
|
+
raise ValueError("Provide 'tokenizer' or 'tokenizer_path'")
|
|
80
|
+
tokenizer = TokenizerWrapper(tok_src)
|
|
81
|
+
self.model = model
|
|
82
|
+
self.tok = tokenizer
|
|
83
|
+
self._max_gen_toks = int(max_gen_toks)
|
|
84
|
+
|
|
85
|
+
# -- properties used by the harness ----------------------------------
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def eot_token_id(self) -> int | None:
|
|
89
|
+
return _eos_id(self.model)
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def max_length(self) -> int:
|
|
93
|
+
return int(self.model.max_position_embeddings)
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def max_gen_toks(self) -> int:
|
|
97
|
+
return self._max_gen_toks
|
|
98
|
+
|
|
99
|
+
# -- helpers ---------------------------------------------------------
|
|
100
|
+
|
|
101
|
+
def _logprobs(self, ids: list[int]) -> torch.Tensor:
|
|
102
|
+
"""Log-probabilities ``[len(ids), vocab]``; row i predicts ids[i + 1]."""
|
|
103
|
+
logits = self.model.forward(torch.tensor([ids], dtype=torch.long))[0]
|
|
104
|
+
return F.log_softmax(logits.to(torch.float32), dim=-1)
|
|
105
|
+
|
|
106
|
+
def _score(self, context_ids: list[int], cont_ids: list[int]) -> tuple[float, bool]:
|
|
107
|
+
if not cont_ids:
|
|
108
|
+
return 0.0, True
|
|
109
|
+
if not context_ids:
|
|
110
|
+
eos = self.eot_token_id
|
|
111
|
+
context_ids = [eos if eos is not None else 0]
|
|
112
|
+
full = (context_ids + cont_ids)[-self.max_length:]
|
|
113
|
+
n = len(cont_ids)
|
|
114
|
+
lp = self._logprobs(full[:-1])
|
|
115
|
+
cont = torch.tensor(full[-n:], dtype=torch.long)
|
|
116
|
+
rows = lp[-n:]
|
|
117
|
+
tok_lp = rows.gather(1, cont.unsqueeze(1)).squeeze(1)
|
|
118
|
+
greedy = bool(torch.equal(rows.argmax(dim=-1), cont))
|
|
119
|
+
return float(tok_lp.sum().item()), greedy
|
|
120
|
+
|
|
121
|
+
def _split(self, context: str, continuation: str) -> tuple[list[int], list[int]]:
|
|
122
|
+
# Encode context and continuation separately so the boundary is exact,
|
|
123
|
+
# moving trailing whitespace of the context onto the continuation.
|
|
124
|
+
n_spaces = len(context) - len(context.rstrip())
|
|
125
|
+
if n_spaces > 0:
|
|
126
|
+
continuation = context[-n_spaces:] + continuation
|
|
127
|
+
context = context[:-n_spaces]
|
|
128
|
+
return self.tok.encode(context), self.tok.encode(continuation)
|
|
129
|
+
|
|
130
|
+
# -- LM interface ----------------------------------------------------
|
|
131
|
+
|
|
132
|
+
def loglikelihood(self, requests) -> list[tuple[float, bool]]:
|
|
133
|
+
out: list[tuple[float, bool]] = []
|
|
134
|
+
for req in requests:
|
|
135
|
+
context, continuation = req.args[:2]
|
|
136
|
+
ctx_ids, cont_ids = self._split(context, continuation)
|
|
137
|
+
out.append(self._score(ctx_ids, cont_ids))
|
|
138
|
+
return out
|
|
139
|
+
|
|
140
|
+
def loglikelihood_rolling(self, requests) -> list[float]:
|
|
141
|
+
out: list[float] = []
|
|
142
|
+
window = self.max_length
|
|
143
|
+
eos = self.eot_token_id
|
|
144
|
+
prefix = eos if eos is not None else 0
|
|
145
|
+
for req in requests:
|
|
146
|
+
ids = self.tok.encode(req.args[0])
|
|
147
|
+
total = 0.0
|
|
148
|
+
pos = 0
|
|
149
|
+
prev: list[int] = [prefix]
|
|
150
|
+
while pos < len(ids):
|
|
151
|
+
chunk = ids[pos:pos + window - len(prev)]
|
|
152
|
+
lp, _ = self._score(prev, chunk)
|
|
153
|
+
total += lp
|
|
154
|
+
pos += len(chunk)
|
|
155
|
+
prev = [] # later windows condition on the start of the chunk only
|
|
156
|
+
if pos < len(ids):
|
|
157
|
+
prev = [ids[pos - 1]]
|
|
158
|
+
out.append(total)
|
|
159
|
+
return out
|
|
160
|
+
|
|
161
|
+
def generate_until(self, requests) -> list[str]:
|
|
162
|
+
from quf.runtime.generate import generate
|
|
163
|
+
|
|
164
|
+
out: list[str] = []
|
|
165
|
+
for req in requests:
|
|
166
|
+
context, gen_kwargs = req.args[0], dict(req.args[1] or {})
|
|
167
|
+
until = gen_kwargs.get("until") or []
|
|
168
|
+
if isinstance(until, str):
|
|
169
|
+
until = [until]
|
|
170
|
+
max_new = int(gen_kwargs.get("max_gen_toks", self.max_gen_toks))
|
|
171
|
+
max_new = max(1, min(max_new, self.max_length - 1))
|
|
172
|
+
temperature = float(gen_kwargs.get("temperature", 0.0) or 0.0)
|
|
173
|
+
ctx_ids = self.tok.encode(context)[-(self.max_length - max_new):]
|
|
174
|
+
ids = generate(
|
|
175
|
+
self.model,
|
|
176
|
+
self.tok,
|
|
177
|
+
ctx_ids,
|
|
178
|
+
max_new_tokens=max_new,
|
|
179
|
+
temperature=temperature,
|
|
180
|
+
top_k=gen_kwargs.get("top_k"),
|
|
181
|
+
top_p=gen_kwargs.get("top_p"),
|
|
182
|
+
)
|
|
183
|
+
eos = self.eot_token_id
|
|
184
|
+
if eos is not None and ids and ids[-1] == eos:
|
|
185
|
+
ids = ids[:-1]
|
|
186
|
+
out.append(_stop_at(self.tok.decode(ids), list(until)))
|
|
187
|
+
return out
|
|
188
|
+
|
|
189
|
+
try:
|
|
190
|
+
from lm_eval.api.registry import register_model
|
|
191
|
+
|
|
192
|
+
QUFLM = register_model("quf")(QUFLM)
|
|
193
|
+
except Exception: # registration is best effort (e.g. already registered)
|
|
194
|
+
pass
|
|
195
|
+
|
|
196
|
+
_LM_CLASS = QUFLM
|
|
197
|
+
return QUFLM
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def __getattr__(name: str):
|
|
201
|
+
if name == "QUFLM":
|
|
202
|
+
return get_adapter_class()
|
|
203
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
quf/bench/suite.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"""Benchmark_Suite: WikiText perplexity, file size, generation speed, results JSON.
|
|
2
|
+
|
|
3
|
+
The documented target is a relative perplexity increase of at most 3 percent
|
|
4
|
+
versus fp16. It is reported next to the measured value and never asserted.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
import math
|
|
12
|
+
import os
|
|
13
|
+
import platform
|
|
14
|
+
import time
|
|
15
|
+
from dataclasses import asdict, dataclass, field
|
|
16
|
+
from typing import Any, Callable, Optional
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger("quf.bench")
|
|
19
|
+
|
|
20
|
+
DEFAULT_DATASET = "wikitext-2-raw-v1"
|
|
21
|
+
PPL_TARGET_PERCENT = 3.0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class DatasetLoadError(RuntimeError):
|
|
25
|
+
"""The evaluation dataset could not be loaded."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, dataset: str, reason: str = ""):
|
|
28
|
+
self.dataset = dataset
|
|
29
|
+
msg = f"could not load dataset '{dataset}'"
|
|
30
|
+
if reason:
|
|
31
|
+
msg += f": {reason}"
|
|
32
|
+
super().__init__(msg)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class BenchResult:
|
|
37
|
+
backend: str
|
|
38
|
+
device_name: str
|
|
39
|
+
software_versions: dict[str, str]
|
|
40
|
+
perplexity: float
|
|
41
|
+
file_size_bytes: int
|
|
42
|
+
tokens_per_second: float
|
|
43
|
+
dataset: str = DEFAULT_DATASET
|
|
44
|
+
ppl_target_percent: float = PPL_TARGET_PERCENT
|
|
45
|
+
fp16_perplexity: Optional[float] = None
|
|
46
|
+
relative_ppl_increase: Optional[float] = None # fraction, 0.02 == 2 percent
|
|
47
|
+
gguf_size_bytes: Optional[int] = None
|
|
48
|
+
gguf_tokens_per_second: Optional[float] = None
|
|
49
|
+
|
|
50
|
+
def to_dict(self) -> dict[str, Any]:
|
|
51
|
+
return asdict(self)
|
|
52
|
+
|
|
53
|
+
def to_json(self) -> str:
|
|
54
|
+
return json.dumps(self.to_dict(), indent=2, sort_keys=True)
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_dict(cls, d: dict[str, Any]) -> "BenchResult":
|
|
58
|
+
return cls(**d)
|
|
59
|
+
|
|
60
|
+
@classmethod
|
|
61
|
+
def from_json(cls, text: str) -> "BenchResult":
|
|
62
|
+
return cls.from_dict(json.loads(text))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# -- dataset ---------------------------------------------------------------
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def load_wikitext(dataset: str = DEFAULT_DATASET, split: str = "test") -> str:
|
|
69
|
+
"""Return the WikiText ``split`` joined into one string.
|
|
70
|
+
|
|
71
|
+
Raises DatasetLoadError naming the dataset on any failure.
|
|
72
|
+
"""
|
|
73
|
+
try:
|
|
74
|
+
from datasets import load_dataset # type: ignore
|
|
75
|
+
|
|
76
|
+
ds = load_dataset("Salesforce/wikitext", dataset, split=split)
|
|
77
|
+
return "\n\n".join(ds["text"])
|
|
78
|
+
except Exception as exc: # noqa: BLE001 - missing package, network, or cache
|
|
79
|
+
raise DatasetLoadError(dataset, f"{type(exc).__name__}: {exc}") from exc
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# -- perplexity ------------------------------------------------------------
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def perplexity(
|
|
86
|
+
model,
|
|
87
|
+
tok,
|
|
88
|
+
dataset: str = DEFAULT_DATASET,
|
|
89
|
+
ctx_len: int = 2048,
|
|
90
|
+
*,
|
|
91
|
+
text: str | None = None,
|
|
92
|
+
max_windows: int | None = None,
|
|
93
|
+
) -> float:
|
|
94
|
+
"""Perplexity of ``model`` over non-overlapping windows of ``ctx_len`` tokens.
|
|
95
|
+
|
|
96
|
+
``model.forward(ids[1, n]) -> logits[1, n, vocab]``. ``text`` overrides
|
|
97
|
+
loading ``dataset`` (used by tests and offline runs).
|
|
98
|
+
"""
|
|
99
|
+
import torch
|
|
100
|
+
import torch.nn.functional as F
|
|
101
|
+
|
|
102
|
+
if text is None:
|
|
103
|
+
text = load_wikitext(dataset)
|
|
104
|
+
ids = tok.encode(text)
|
|
105
|
+
if ctx_len < 2:
|
|
106
|
+
raise ValueError("ctx_len must be at least 2")
|
|
107
|
+
n_windows = len(ids) // ctx_len
|
|
108
|
+
if n_windows == 0 and len(ids) >= 2:
|
|
109
|
+
ctx_len, n_windows = len(ids), 1
|
|
110
|
+
if n_windows == 0:
|
|
111
|
+
raise ValueError("evaluation text is too short to compute perplexity")
|
|
112
|
+
if max_windows is not None:
|
|
113
|
+
n_windows = min(n_windows, max_windows)
|
|
114
|
+
|
|
115
|
+
total_nll = 0.0
|
|
116
|
+
total_tokens = 0
|
|
117
|
+
for w in range(n_windows):
|
|
118
|
+
window = torch.tensor([ids[w * ctx_len : (w + 1) * ctx_len]], dtype=torch.long)
|
|
119
|
+
logits = model.forward(window)
|
|
120
|
+
logits = logits[0, :-1].to(torch.float32)
|
|
121
|
+
targets = window[0, 1:].to(logits.device)
|
|
122
|
+
nll = F.cross_entropy(logits, targets, reduction="sum")
|
|
123
|
+
total_nll += float(nll.item())
|
|
124
|
+
total_tokens += int(targets.numel())
|
|
125
|
+
return math.exp(total_nll / total_tokens)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# -- speed -----------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def measure_tokens_per_second(
|
|
132
|
+
model, tok, prompt: str = "The quick brown fox", max_new_tokens: int = 32
|
|
133
|
+
) -> float:
|
|
134
|
+
"""Greedy generation speed in new tokens per second."""
|
|
135
|
+
from quf.runtime.generate import generate
|
|
136
|
+
|
|
137
|
+
prompt_ids = tok.encode(prompt) or [0]
|
|
138
|
+
budget = max(1, min(max_new_tokens, model.max_position_embeddings - len(prompt_ids)))
|
|
139
|
+
start = time.perf_counter()
|
|
140
|
+
out = generate(model, tok, prompt_ids, max_new_tokens=budget, temperature=0.0)
|
|
141
|
+
elapsed = time.perf_counter() - start
|
|
142
|
+
return len(out) / elapsed if elapsed > 0 and out else 0.0
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class _HFCausalLM:
|
|
146
|
+
"""Adapts a transformers causal LM to the ``forward(ids)`` used here."""
|
|
147
|
+
|
|
148
|
+
def __init__(self, path: str, device: str):
|
|
149
|
+
import torch
|
|
150
|
+
from transformers import AutoModelForCausalLM # type: ignore
|
|
151
|
+
|
|
152
|
+
self._device = torch.device(device)
|
|
153
|
+
self._m = AutoModelForCausalLM.from_pretrained(
|
|
154
|
+
path, torch_dtype=torch.float16 if device != "cpu" else torch.float32
|
|
155
|
+
).to(self._device)
|
|
156
|
+
self._m.eval()
|
|
157
|
+
|
|
158
|
+
def forward(self, ids):
|
|
159
|
+
import torch
|
|
160
|
+
|
|
161
|
+
with torch.no_grad():
|
|
162
|
+
return self._m(ids.to(self._device)).logits
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _gguf_speed(path: str, tok_text: str = "The quick brown fox", n: int = 32) -> Optional[float]:
|
|
166
|
+
try:
|
|
167
|
+
from llama_cpp import Llama # type: ignore
|
|
168
|
+
except Exception as exc: # noqa: BLE001
|
|
169
|
+
logger.warning("llama-cpp-python unavailable, GGUF speed not measured: %r", exc)
|
|
170
|
+
return None
|
|
171
|
+
llm = Llama(model_path=path, verbose=False)
|
|
172
|
+
start = time.perf_counter()
|
|
173
|
+
out = llm(tok_text, max_tokens=n, temperature=0.0)
|
|
174
|
+
elapsed = time.perf_counter() - start
|
|
175
|
+
produced = int(out["usage"]["completion_tokens"])
|
|
176
|
+
return produced / elapsed if elapsed > 0 and produced else 0.0
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# -- environment -----------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def software_versions() -> dict[str, str]:
|
|
183
|
+
from importlib import metadata
|
|
184
|
+
|
|
185
|
+
import quf
|
|
186
|
+
|
|
187
|
+
versions = {"python": platform.python_version(), "quf": getattr(quf, "__version__", "unknown")}
|
|
188
|
+
for pkg in ("torch", "gptqmodel", "gigatoken", "tokenizers", "transformers", "datasets"):
|
|
189
|
+
try:
|
|
190
|
+
versions[pkg] = metadata.version(pkg)
|
|
191
|
+
except Exception: # noqa: BLE001
|
|
192
|
+
versions[pkg] = "not installed"
|
|
193
|
+
try:
|
|
194
|
+
import torch
|
|
195
|
+
|
|
196
|
+
versions["cuda"] = str(torch.version.cuda)
|
|
197
|
+
except Exception: # noqa: BLE001
|
|
198
|
+
versions["cuda"] = "not installed"
|
|
199
|
+
return versions
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
# -- driver ----------------------------------------------------------------
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _tokenizer_from_header(header):
|
|
206
|
+
from tokenizers import Tokenizer # type: ignore
|
|
207
|
+
|
|
208
|
+
from quf.tokenization.wrapper import TokenizerWrapper
|
|
209
|
+
|
|
210
|
+
raw = (header.tokenizer or {}).get("files", {}).get("tokenizer.json")
|
|
211
|
+
if raw is None:
|
|
212
|
+
raise ValueError("QUF header has no embedded tokenizer.json")
|
|
213
|
+
return TokenizerWrapper(hf_tokenizer=Tokenizer.from_str(raw))
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def run_bench(
|
|
217
|
+
quf_path,
|
|
218
|
+
*,
|
|
219
|
+
fp16_ref: str | None = None,
|
|
220
|
+
gguf: str | None = None,
|
|
221
|
+
out_json,
|
|
222
|
+
device: str = "auto",
|
|
223
|
+
backend_option: str = "auto",
|
|
224
|
+
prefer_fused: bool = False,
|
|
225
|
+
dataset: str = DEFAULT_DATASET,
|
|
226
|
+
ctx_len: int = 2048,
|
|
227
|
+
max_windows: int | None = None,
|
|
228
|
+
max_new_tokens: int = 32,
|
|
229
|
+
text: str | None = None,
|
|
230
|
+
model_loader: Callable[..., Any] | None = None,
|
|
231
|
+
tok=None,
|
|
232
|
+
) -> BenchResult:
|
|
233
|
+
"""Run the benchmarks and write the results JSON to ``out_json``.
|
|
234
|
+
|
|
235
|
+
Raises DatasetLoadError (naming the dataset) before any model work when the
|
|
236
|
+
dataset cannot be loaded.
|
|
237
|
+
"""
|
|
238
|
+
from quf.device import detect_devices
|
|
239
|
+
from quf.runtime.selector import select_backend
|
|
240
|
+
|
|
241
|
+
if text is None:
|
|
242
|
+
text = load_wikitext(dataset) # fail early, naming the dataset
|
|
243
|
+
|
|
244
|
+
info = detect_devices()
|
|
245
|
+
|
|
246
|
+
cached_bytes = free_bytes = None
|
|
247
|
+
if info.cuda_available:
|
|
248
|
+
from quf.runtime.model import _available_gpu_bytes, estimate_cached_bytes
|
|
249
|
+
|
|
250
|
+
cached_bytes, free_bytes = estimate_cached_bytes(quf_path), _available_gpu_bytes() or None
|
|
251
|
+
from quf.runtime.backends.torch_fused import fused_self_test
|
|
252
|
+
|
|
253
|
+
choice = select_backend(
|
|
254
|
+
device,
|
|
255
|
+
info,
|
|
256
|
+
backend_option=backend_option,
|
|
257
|
+
cached_bytes=cached_bytes,
|
|
258
|
+
free_bytes=free_bytes,
|
|
259
|
+
prefer_fused=prefer_fused,
|
|
260
|
+
fused_self_test=fused_self_test,
|
|
261
|
+
)
|
|
262
|
+
if model_loader is None:
|
|
263
|
+
from quf.runtime.model import QUFModel
|
|
264
|
+
|
|
265
|
+
model_loader = lambda p, backend: QUFModel.load( # noqa: E731
|
|
266
|
+
p, device_option="cpu" if backend == "cpu" else "cuda", backend=backend
|
|
267
|
+
)
|
|
268
|
+
model = model_loader(os.fspath(quf_path), choice.backend)
|
|
269
|
+
if tok is None:
|
|
270
|
+
tok = _tokenizer_from_header(model.header)
|
|
271
|
+
|
|
272
|
+
ppl = perplexity(model, tok, dataset, ctx_len, text=text, max_windows=max_windows)
|
|
273
|
+
tps = measure_tokens_per_second(model, tok, max_new_tokens=max_new_tokens)
|
|
274
|
+
|
|
275
|
+
result = BenchResult(
|
|
276
|
+
backend=choice.backend,
|
|
277
|
+
device_name=(info.name if choice.backend != "cpu" and info.name else "cpu"),
|
|
278
|
+
software_versions=software_versions(),
|
|
279
|
+
perplexity=ppl,
|
|
280
|
+
file_size_bytes=os.path.getsize(quf_path),
|
|
281
|
+
tokens_per_second=tps,
|
|
282
|
+
dataset=dataset,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
if fp16_ref is not None:
|
|
286
|
+
ref_device = "cpu" if choice.backend == "cpu" else "cuda"
|
|
287
|
+
ref = _HFCausalLM(fp16_ref, ref_device)
|
|
288
|
+
result.fp16_perplexity = perplexity(
|
|
289
|
+
ref, tok, dataset, ctx_len, text=text, max_windows=max_windows
|
|
290
|
+
)
|
|
291
|
+
result.relative_ppl_increase = ppl / result.fp16_perplexity - 1.0
|
|
292
|
+
logger.info(
|
|
293
|
+
"relative perplexity increase %.2f%% (documented target: at most %.0f%%)",
|
|
294
|
+
100 * result.relative_ppl_increase,
|
|
295
|
+
PPL_TARGET_PERCENT,
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
if gguf is not None:
|
|
299
|
+
result.gguf_size_bytes = os.path.getsize(gguf)
|
|
300
|
+
result.gguf_tokens_per_second = _gguf_speed(gguf, n=max_new_tokens)
|
|
301
|
+
|
|
302
|
+
parent = os.path.dirname(os.path.abspath(os.fspath(out_json)))
|
|
303
|
+
os.makedirs(parent, exist_ok=True)
|
|
304
|
+
with open(out_json, "w", encoding="utf-8") as f:
|
|
305
|
+
f.write(result.to_json())
|
|
306
|
+
return result
|