quf 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
quf/__init__.py ADDED
@@ -0,0 +1,28 @@
1
+ """QUF: Quantized Unified Format.
2
+
3
+ Importing this package never requires optional extras (torch, gptqmodel, ...).
4
+ Submodules are imported lazily on first attribute access.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import importlib
10
+ from typing import Any
11
+
12
+ __version__ = "0.1.0"
13
+
14
+ _LAZY_SUBMODULES = frozenset(
15
+ {"device", "format", "quant", "tokenization", "convert", "runtime", "bench", "cli"}
16
+ )
17
+
18
+ __all__ = ["__version__", *sorted(_LAZY_SUBMODULES)]
19
+
20
+
21
+ def __getattr__(name: str) -> Any:
22
+ if name in _LAZY_SUBMODULES:
23
+ return importlib.import_module(f"{__name__}.{name}")
24
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
25
+
26
+
27
+ def __dir__() -> list[str]:
28
+ return sorted(set(globals()) | _LAZY_SUBMODULES)
quf/bench/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """Benchmark_Suite: perplexity, size, speed, and results JSON."""
@@ -0,0 +1,203 @@
1
+ """lm-evaluation-harness adapter for QUF files.
2
+
3
+ ``lm_eval`` is imported lazily, so importing this module never requires it.
4
+ The adapter class ``QUFLM`` subclasses ``lm_eval.api.model.LM`` and is built on
5
+ first attribute access (``from quf.bench.lm_eval_adapter import QUFLM``) or via
6
+ :func:`get_adapter_class`. It is registered with the harness as ``"quf"``::
7
+
8
+ lm_eval --model quf --model_args path=model.quf,device=cpu --tasks hellaswag
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from typing import Any
14
+
15
+ import torch
16
+ import torch.nn.functional as F
17
+
18
+ _LM_CLASS: type | None = None
19
+
20
+
21
+ def _import_lm_base():
22
+ try:
23
+ from lm_eval.api.model import LM
24
+ except ImportError as exc:
25
+ raise ImportError(
26
+ "lm-evaluation-harness is required for the QUF adapter. "
27
+ "Install it with: pip install lm_eval"
28
+ ) from exc
29
+ return LM
30
+
31
+
32
+ def _eos_id(model) -> int | None:
33
+ eos = (model.header.tokenizer or {}).get("eos_token_id")
34
+ return int(eos) if eos is not None else None
35
+
36
+
37
+ def _stop_at(text: str, until: list[str]) -> str:
38
+ cut = len(text)
39
+ for s in until:
40
+ if s:
41
+ idx = text.find(s)
42
+ if idx != -1:
43
+ cut = min(cut, idx)
44
+ return text[:cut]
45
+
46
+
47
+ def get_adapter_class() -> type:
48
+ """Return the ``QUFLM`` class, importing ``lm_eval`` on first use."""
49
+ global _LM_CLASS
50
+ if _LM_CLASS is not None:
51
+ return _LM_CLASS
52
+ LM = _import_lm_base()
53
+
54
+ class QUFLM(LM):
55
+ """Evaluate a QUF_File with lm-evaluation-harness."""
56
+
57
+ def __init__(
58
+ self,
59
+ path: str | None = None,
60
+ model: Any = None,
61
+ tokenizer: Any = None,
62
+ device: str = "cpu",
63
+ backend: str | None = None,
64
+ tokenizer_path: str | None = None,
65
+ max_gen_toks: int = 256,
66
+ **kwargs: Any,
67
+ ) -> None:
68
+ super().__init__()
69
+ from quf.runtime.model import QUFModel
70
+ from quf.tokenization.wrapper import TokenizerWrapper
71
+
72
+ if model is None:
73
+ if path is None:
74
+ raise ValueError("Provide 'path' to a .quf file or a loaded 'model'")
75
+ model = QUFModel.load(path, device_option=device, backend=backend)
76
+ if tokenizer is None:
77
+ tok_src = tokenizer_path or (model.header.tokenizer or {}).get("path")
78
+ if tok_src is None:
79
+ raise ValueError("Provide 'tokenizer' or 'tokenizer_path'")
80
+ tokenizer = TokenizerWrapper(tok_src)
81
+ self.model = model
82
+ self.tok = tokenizer
83
+ self._max_gen_toks = int(max_gen_toks)
84
+
85
+ # -- properties used by the harness ----------------------------------
86
+
87
+ @property
88
+ def eot_token_id(self) -> int | None:
89
+ return _eos_id(self.model)
90
+
91
+ @property
92
+ def max_length(self) -> int:
93
+ return int(self.model.max_position_embeddings)
94
+
95
+ @property
96
+ def max_gen_toks(self) -> int:
97
+ return self._max_gen_toks
98
+
99
+ # -- helpers ---------------------------------------------------------
100
+
101
+ def _logprobs(self, ids: list[int]) -> torch.Tensor:
102
+ """Log-probabilities ``[len(ids), vocab]``; row i predicts ids[i + 1]."""
103
+ logits = self.model.forward(torch.tensor([ids], dtype=torch.long))[0]
104
+ return F.log_softmax(logits.to(torch.float32), dim=-1)
105
+
106
+ def _score(self, context_ids: list[int], cont_ids: list[int]) -> tuple[float, bool]:
107
+ if not cont_ids:
108
+ return 0.0, True
109
+ if not context_ids:
110
+ eos = self.eot_token_id
111
+ context_ids = [eos if eos is not None else 0]
112
+ full = (context_ids + cont_ids)[-self.max_length:]
113
+ n = len(cont_ids)
114
+ lp = self._logprobs(full[:-1])
115
+ cont = torch.tensor(full[-n:], dtype=torch.long)
116
+ rows = lp[-n:]
117
+ tok_lp = rows.gather(1, cont.unsqueeze(1)).squeeze(1)
118
+ greedy = bool(torch.equal(rows.argmax(dim=-1), cont))
119
+ return float(tok_lp.sum().item()), greedy
120
+
121
+ def _split(self, context: str, continuation: str) -> tuple[list[int], list[int]]:
122
+ # Encode context and continuation separately so the boundary is exact,
123
+ # moving trailing whitespace of the context onto the continuation.
124
+ n_spaces = len(context) - len(context.rstrip())
125
+ if n_spaces > 0:
126
+ continuation = context[-n_spaces:] + continuation
127
+ context = context[:-n_spaces]
128
+ return self.tok.encode(context), self.tok.encode(continuation)
129
+
130
+ # -- LM interface ----------------------------------------------------
131
+
132
+ def loglikelihood(self, requests) -> list[tuple[float, bool]]:
133
+ out: list[tuple[float, bool]] = []
134
+ for req in requests:
135
+ context, continuation = req.args[:2]
136
+ ctx_ids, cont_ids = self._split(context, continuation)
137
+ out.append(self._score(ctx_ids, cont_ids))
138
+ return out
139
+
140
+ def loglikelihood_rolling(self, requests) -> list[float]:
141
+ out: list[float] = []
142
+ window = self.max_length
143
+ eos = self.eot_token_id
144
+ prefix = eos if eos is not None else 0
145
+ for req in requests:
146
+ ids = self.tok.encode(req.args[0])
147
+ total = 0.0
148
+ pos = 0
149
+ prev: list[int] = [prefix]
150
+ while pos < len(ids):
151
+ chunk = ids[pos:pos + window - len(prev)]
152
+ lp, _ = self._score(prev, chunk)
153
+ total += lp
154
+ pos += len(chunk)
155
+ prev = [] # later windows condition on the start of the chunk only
156
+ if pos < len(ids):
157
+ prev = [ids[pos - 1]]
158
+ out.append(total)
159
+ return out
160
+
161
+ def generate_until(self, requests) -> list[str]:
162
+ from quf.runtime.generate import generate
163
+
164
+ out: list[str] = []
165
+ for req in requests:
166
+ context, gen_kwargs = req.args[0], dict(req.args[1] or {})
167
+ until = gen_kwargs.get("until") or []
168
+ if isinstance(until, str):
169
+ until = [until]
170
+ max_new = int(gen_kwargs.get("max_gen_toks", self.max_gen_toks))
171
+ max_new = max(1, min(max_new, self.max_length - 1))
172
+ temperature = float(gen_kwargs.get("temperature", 0.0) or 0.0)
173
+ ctx_ids = self.tok.encode(context)[-(self.max_length - max_new):]
174
+ ids = generate(
175
+ self.model,
176
+ self.tok,
177
+ ctx_ids,
178
+ max_new_tokens=max_new,
179
+ temperature=temperature,
180
+ top_k=gen_kwargs.get("top_k"),
181
+ top_p=gen_kwargs.get("top_p"),
182
+ )
183
+ eos = self.eot_token_id
184
+ if eos is not None and ids and ids[-1] == eos:
185
+ ids = ids[:-1]
186
+ out.append(_stop_at(self.tok.decode(ids), list(until)))
187
+ return out
188
+
189
+ try:
190
+ from lm_eval.api.registry import register_model
191
+
192
+ QUFLM = register_model("quf")(QUFLM)
193
+ except Exception: # registration is best effort (e.g. already registered)
194
+ pass
195
+
196
+ _LM_CLASS = QUFLM
197
+ return QUFLM
198
+
199
+
200
+ def __getattr__(name: str):
201
+ if name == "QUFLM":
202
+ return get_adapter_class()
203
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
quf/bench/suite.py ADDED
@@ -0,0 +1,306 @@
1
+ """Benchmark_Suite: WikiText perplexity, file size, generation speed, results JSON.
2
+
3
+ The documented target is a relative perplexity increase of at most 3 percent
4
+ versus fp16. It is reported next to the measured value and never asserted.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import logging
11
+ import math
12
+ import os
13
+ import platform
14
+ import time
15
+ from dataclasses import asdict, dataclass, field
16
+ from typing import Any, Callable, Optional
17
+
18
+ logger = logging.getLogger("quf.bench")
19
+
20
+ DEFAULT_DATASET = "wikitext-2-raw-v1"
21
+ PPL_TARGET_PERCENT = 3.0
22
+
23
+
24
+ class DatasetLoadError(RuntimeError):
25
+ """The evaluation dataset could not be loaded."""
26
+
27
+ def __init__(self, dataset: str, reason: str = ""):
28
+ self.dataset = dataset
29
+ msg = f"could not load dataset '{dataset}'"
30
+ if reason:
31
+ msg += f": {reason}"
32
+ super().__init__(msg)
33
+
34
+
35
+ @dataclass
36
+ class BenchResult:
37
+ backend: str
38
+ device_name: str
39
+ software_versions: dict[str, str]
40
+ perplexity: float
41
+ file_size_bytes: int
42
+ tokens_per_second: float
43
+ dataset: str = DEFAULT_DATASET
44
+ ppl_target_percent: float = PPL_TARGET_PERCENT
45
+ fp16_perplexity: Optional[float] = None
46
+ relative_ppl_increase: Optional[float] = None # fraction, 0.02 == 2 percent
47
+ gguf_size_bytes: Optional[int] = None
48
+ gguf_tokens_per_second: Optional[float] = None
49
+
50
+ def to_dict(self) -> dict[str, Any]:
51
+ return asdict(self)
52
+
53
+ def to_json(self) -> str:
54
+ return json.dumps(self.to_dict(), indent=2, sort_keys=True)
55
+
56
+ @classmethod
57
+ def from_dict(cls, d: dict[str, Any]) -> "BenchResult":
58
+ return cls(**d)
59
+
60
+ @classmethod
61
+ def from_json(cls, text: str) -> "BenchResult":
62
+ return cls.from_dict(json.loads(text))
63
+
64
+
65
+ # -- dataset ---------------------------------------------------------------
66
+
67
+
68
+ def load_wikitext(dataset: str = DEFAULT_DATASET, split: str = "test") -> str:
69
+ """Return the WikiText ``split`` joined into one string.
70
+
71
+ Raises DatasetLoadError naming the dataset on any failure.
72
+ """
73
+ try:
74
+ from datasets import load_dataset # type: ignore
75
+
76
+ ds = load_dataset("Salesforce/wikitext", dataset, split=split)
77
+ return "\n\n".join(ds["text"])
78
+ except Exception as exc: # noqa: BLE001 - missing package, network, or cache
79
+ raise DatasetLoadError(dataset, f"{type(exc).__name__}: {exc}") from exc
80
+
81
+
82
+ # -- perplexity ------------------------------------------------------------
83
+
84
+
85
+ def perplexity(
86
+ model,
87
+ tok,
88
+ dataset: str = DEFAULT_DATASET,
89
+ ctx_len: int = 2048,
90
+ *,
91
+ text: str | None = None,
92
+ max_windows: int | None = None,
93
+ ) -> float:
94
+ """Perplexity of ``model`` over non-overlapping windows of ``ctx_len`` tokens.
95
+
96
+ ``model.forward(ids[1, n]) -> logits[1, n, vocab]``. ``text`` overrides
97
+ loading ``dataset`` (used by tests and offline runs).
98
+ """
99
+ import torch
100
+ import torch.nn.functional as F
101
+
102
+ if text is None:
103
+ text = load_wikitext(dataset)
104
+ ids = tok.encode(text)
105
+ if ctx_len < 2:
106
+ raise ValueError("ctx_len must be at least 2")
107
+ n_windows = len(ids) // ctx_len
108
+ if n_windows == 0 and len(ids) >= 2:
109
+ ctx_len, n_windows = len(ids), 1
110
+ if n_windows == 0:
111
+ raise ValueError("evaluation text is too short to compute perplexity")
112
+ if max_windows is not None:
113
+ n_windows = min(n_windows, max_windows)
114
+
115
+ total_nll = 0.0
116
+ total_tokens = 0
117
+ for w in range(n_windows):
118
+ window = torch.tensor([ids[w * ctx_len : (w + 1) * ctx_len]], dtype=torch.long)
119
+ logits = model.forward(window)
120
+ logits = logits[0, :-1].to(torch.float32)
121
+ targets = window[0, 1:].to(logits.device)
122
+ nll = F.cross_entropy(logits, targets, reduction="sum")
123
+ total_nll += float(nll.item())
124
+ total_tokens += int(targets.numel())
125
+ return math.exp(total_nll / total_tokens)
126
+
127
+
128
+ # -- speed -----------------------------------------------------------------
129
+
130
+
131
+ def measure_tokens_per_second(
132
+ model, tok, prompt: str = "The quick brown fox", max_new_tokens: int = 32
133
+ ) -> float:
134
+ """Greedy generation speed in new tokens per second."""
135
+ from quf.runtime.generate import generate
136
+
137
+ prompt_ids = tok.encode(prompt) or [0]
138
+ budget = max(1, min(max_new_tokens, model.max_position_embeddings - len(prompt_ids)))
139
+ start = time.perf_counter()
140
+ out = generate(model, tok, prompt_ids, max_new_tokens=budget, temperature=0.0)
141
+ elapsed = time.perf_counter() - start
142
+ return len(out) / elapsed if elapsed > 0 and out else 0.0
143
+
144
+
145
+ class _HFCausalLM:
146
+ """Adapts a transformers causal LM to the ``forward(ids)`` used here."""
147
+
148
+ def __init__(self, path: str, device: str):
149
+ import torch
150
+ from transformers import AutoModelForCausalLM # type: ignore
151
+
152
+ self._device = torch.device(device)
153
+ self._m = AutoModelForCausalLM.from_pretrained(
154
+ path, torch_dtype=torch.float16 if device != "cpu" else torch.float32
155
+ ).to(self._device)
156
+ self._m.eval()
157
+
158
+ def forward(self, ids):
159
+ import torch
160
+
161
+ with torch.no_grad():
162
+ return self._m(ids.to(self._device)).logits
163
+
164
+
165
+ def _gguf_speed(path: str, tok_text: str = "The quick brown fox", n: int = 32) -> Optional[float]:
166
+ try:
167
+ from llama_cpp import Llama # type: ignore
168
+ except Exception as exc: # noqa: BLE001
169
+ logger.warning("llama-cpp-python unavailable, GGUF speed not measured: %r", exc)
170
+ return None
171
+ llm = Llama(model_path=path, verbose=False)
172
+ start = time.perf_counter()
173
+ out = llm(tok_text, max_tokens=n, temperature=0.0)
174
+ elapsed = time.perf_counter() - start
175
+ produced = int(out["usage"]["completion_tokens"])
176
+ return produced / elapsed if elapsed > 0 and produced else 0.0
177
+
178
+
179
+ # -- environment -----------------------------------------------------------
180
+
181
+
182
+ def software_versions() -> dict[str, str]:
183
+ from importlib import metadata
184
+
185
+ import quf
186
+
187
+ versions = {"python": platform.python_version(), "quf": getattr(quf, "__version__", "unknown")}
188
+ for pkg in ("torch", "gptqmodel", "gigatoken", "tokenizers", "transformers", "datasets"):
189
+ try:
190
+ versions[pkg] = metadata.version(pkg)
191
+ except Exception: # noqa: BLE001
192
+ versions[pkg] = "not installed"
193
+ try:
194
+ import torch
195
+
196
+ versions["cuda"] = str(torch.version.cuda)
197
+ except Exception: # noqa: BLE001
198
+ versions["cuda"] = "not installed"
199
+ return versions
200
+
201
+
202
+ # -- driver ----------------------------------------------------------------
203
+
204
+
205
+ def _tokenizer_from_header(header):
206
+ from tokenizers import Tokenizer # type: ignore
207
+
208
+ from quf.tokenization.wrapper import TokenizerWrapper
209
+
210
+ raw = (header.tokenizer or {}).get("files", {}).get("tokenizer.json")
211
+ if raw is None:
212
+ raise ValueError("QUF header has no embedded tokenizer.json")
213
+ return TokenizerWrapper(hf_tokenizer=Tokenizer.from_str(raw))
214
+
215
+
216
+ def run_bench(
217
+ quf_path,
218
+ *,
219
+ fp16_ref: str | None = None,
220
+ gguf: str | None = None,
221
+ out_json,
222
+ device: str = "auto",
223
+ backend_option: str = "auto",
224
+ prefer_fused: bool = False,
225
+ dataset: str = DEFAULT_DATASET,
226
+ ctx_len: int = 2048,
227
+ max_windows: int | None = None,
228
+ max_new_tokens: int = 32,
229
+ text: str | None = None,
230
+ model_loader: Callable[..., Any] | None = None,
231
+ tok=None,
232
+ ) -> BenchResult:
233
+ """Run the benchmarks and write the results JSON to ``out_json``.
234
+
235
+ Raises DatasetLoadError (naming the dataset) before any model work when the
236
+ dataset cannot be loaded.
237
+ """
238
+ from quf.device import detect_devices
239
+ from quf.runtime.selector import select_backend
240
+
241
+ if text is None:
242
+ text = load_wikitext(dataset) # fail early, naming the dataset
243
+
244
+ info = detect_devices()
245
+
246
+ cached_bytes = free_bytes = None
247
+ if info.cuda_available:
248
+ from quf.runtime.model import _available_gpu_bytes, estimate_cached_bytes
249
+
250
+ cached_bytes, free_bytes = estimate_cached_bytes(quf_path), _available_gpu_bytes() or None
251
+ from quf.runtime.backends.torch_fused import fused_self_test
252
+
253
+ choice = select_backend(
254
+ device,
255
+ info,
256
+ backend_option=backend_option,
257
+ cached_bytes=cached_bytes,
258
+ free_bytes=free_bytes,
259
+ prefer_fused=prefer_fused,
260
+ fused_self_test=fused_self_test,
261
+ )
262
+ if model_loader is None:
263
+ from quf.runtime.model import QUFModel
264
+
265
+ model_loader = lambda p, backend: QUFModel.load( # noqa: E731
266
+ p, device_option="cpu" if backend == "cpu" else "cuda", backend=backend
267
+ )
268
+ model = model_loader(os.fspath(quf_path), choice.backend)
269
+ if tok is None:
270
+ tok = _tokenizer_from_header(model.header)
271
+
272
+ ppl = perplexity(model, tok, dataset, ctx_len, text=text, max_windows=max_windows)
273
+ tps = measure_tokens_per_second(model, tok, max_new_tokens=max_new_tokens)
274
+
275
+ result = BenchResult(
276
+ backend=choice.backend,
277
+ device_name=(info.name if choice.backend != "cpu" and info.name else "cpu"),
278
+ software_versions=software_versions(),
279
+ perplexity=ppl,
280
+ file_size_bytes=os.path.getsize(quf_path),
281
+ tokens_per_second=tps,
282
+ dataset=dataset,
283
+ )
284
+
285
+ if fp16_ref is not None:
286
+ ref_device = "cpu" if choice.backend == "cpu" else "cuda"
287
+ ref = _HFCausalLM(fp16_ref, ref_device)
288
+ result.fp16_perplexity = perplexity(
289
+ ref, tok, dataset, ctx_len, text=text, max_windows=max_windows
290
+ )
291
+ result.relative_ppl_increase = ppl / result.fp16_perplexity - 1.0
292
+ logger.info(
293
+ "relative perplexity increase %.2f%% (documented target: at most %.0f%%)",
294
+ 100 * result.relative_ppl_increase,
295
+ PPL_TARGET_PERCENT,
296
+ )
297
+
298
+ if gguf is not None:
299
+ result.gguf_size_bytes = os.path.getsize(gguf)
300
+ result.gguf_tokens_per_second = _gguf_speed(gguf, n=max_new_tokens)
301
+
302
+ parent = os.path.dirname(os.path.abspath(os.fspath(out_json)))
303
+ os.makedirs(parent, exist_ok=True)
304
+ with open(out_json, "w", encoding="utf-8") as f:
305
+ f.write(result.to_json())
306
+ return result