quant-regress 0.1.0__py3-none-any.whl

quant_regress/__init__.py

@@ -0,0 +1,13 @@
+"""quant-regress — task-level regression testing for quantized models.
+
+Perplexity and token-level metrics do not reliably detect the damage
+quantization does at the margins. This package measures task accuracy
+before and after quantization so a regression fails a build instead of
+shipping.
+"""
+
+__version__ = "0.1.0"
+
+from .harness import QuantHarness, ComparisonResult, Verdict
+
+__all__ = ["QuantHarness", "ComparisonResult", "Verdict", "__version__"]

quant_regress/adapter.py

@@ -0,0 +1,70 @@
+"""Build real models at a requested precision.
+
+Only dynamic ``quantize_dynamic`` INT8 is supported. That is deliberate:
+
+* it needs no extra dependency (no autoawq / optimum / onnxruntime),
+* it emits CPU-only kernels, which is what CI runners have, and
+* it is the exact method that produced the motivating INT8 accuracy collapse.
+
+Attempting to move a quantized model to CUDA raises at forward time, so the
+device is pinned to CPU and we say so rather than letting it fail obscurely.
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+SUPPORTED = ("fp32", "int8")
+
+
+class AdapterError(RuntimeError):
+    """Raised with an actionable message when a model cannot be built."""
+
+
+def _quantize_int8(model):
+    """Dynamic INT8 on Linear layers; embeddings/LayerNorm stay FP32."""
+    return torch.ao.quantization.quantize_dynamic(
+        model, {torch.nn.Linear}, dtype=torch.qint8
+    )
+
+
+def build_model_factory(
+    model_name: str,
+    cache_dir: str | None = None,
+    max_new_tokens: int = 32,
+) -> Callable[[str], object]:
+    if not model_name or not model_name.strip():
+        raise AdapterError(
+            "no model specified. Pass model: to the action, e.g. "
+            "model: meta-llama/Llama-3.2-1B-Instruct"
+        )
+    name = model_name.strip()
+
+    def factory(precision: str):
+        p = precision.lower().strip()
+        if p not in SUPPORTED:
+            raise ValueError(
+                f"unsupported precision {precision!r}; supported: {SUPPORTED}"
+            )
+        kwargs = {"cache_dir": cache_dir} if cache_dir else {}
+        try:
+            tok = AutoTokenizer.from_pretrained(name, **kwargs)
+            model = AutoModelForCausalLM.from_pretrained(name, **kwargs)
+        except Exception as exc:
+            raise AdapterError(
+                f"could not load {name!r}: {type(exc).__name__}: {exc}. "
+                "Check the model id, and that the runner has disk space "
+                "(a 1B model is ~2 GB, a 7B model is ~14 GB)."
+            ) from exc
+        if p == "int8":
+            model = _quantize_int8(model)
+        # quantize_dynamic emits CPU-only kernels.
+        model = model.to("cpu").eval()
+        model.tokenizer = tok  # attached for the harness
+        model.max_new_tokens = max_new_tokens
+        return model
+
+    return factory

quant_regress/cli.py

@@ -0,0 +1,110 @@
+"""Command-line entry point.
+
+Exit codes are the contract with CI:
+
+* ``0`` — within tolerance
+* ``1`` — accuracy regressed past ``--max-drop-points``
+* ``2`` — bad usage / configuration (missing model, unreadable eval set)
+
+A JSON report is written when ``--report`` is given so a workflow can publish
+it as an artifact.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from .adapter import AdapterError, build_model_factory
+from .evalset import EvalSet, EvalSetError
+from .harness import QuantHarness, Verdict
+
+EXIT_OK, EXIT_REGRESSION, EXIT_USAGE = 0, 1, 2
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="quant-regress",
+        description="Fail CI when quantization costs you task accuracy.",
+    )
+    p.add_argument("--eval", required=True, help="path to a JSONL task eval set")
+    p.add_argument("--model", default=None, help="HF model id (required for a real run)")
+    p.add_argument("--precisions", default="int8",
+                   help="comma-separated candidate precisions (default: int8)")
+    p.add_argument("--baseline-precision", default="fp32")
+    p.add_argument("--max-drop-points", type=float, default=2.0,
+                   help="fail if any candidate drops more than this many points")
+    p.add_argument("--max-new-tokens", type=int, default=32)
+    p.add_argument("--report", default=None, help="write a JSON report here")
+    p.add_argument("--cache-dir", default=None)
+    return p
+
+
+def _markdown(res) -> str:
+    b = res.baseline
+    lines = [
+        "| precision | accuracy | correct | drop |",
+        "|---|---|---|---|",
+        f"| {b.precision} (baseline) | {b.accuracy*100:.1f}% | {b.correct}/{b.total} | — |",
+    ]
+    for c in res.candidates:
+        drop = (b.accuracy - c.accuracy) * 100.0
+        lines.append(
+            f"| {c.precision} | {c.accuracy*100:.1f}% | {c.correct}/{c.total} | "
+            f"{drop:+.1f} pts |"
+        )
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        eval_set = EvalSet.load(args.eval)
+    except EvalSetError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_USAGE
+    if not args.model:
+        print(
+            "error: --model is required (e.g. --model "
+            "meta-llama/Llama-3.2-1B-Instruct)", file=sys.stderr
+        )
+        return EXIT_USAGE
+
+    try:
+        factory = build_model_factory(
+            args.model, cache_dir=args.cache_dir,
+            max_new_tokens=args.max_new_tokens,
+        )
+    except AdapterError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_USAGE
+
+    harness = QuantHarness(model_factory=factory, max_new_tokens=args.max_new_tokens)
+    try:
+        res = harness.compare(
+            eval_set,
+            baseline_precision=args.baseline_precision,
+            candidate_precisions=[p.strip() for p in args.precisions.split(",") if p.strip()],
+            max_drop_points=args.max_drop_points,
+        )
+    except RuntimeError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_USAGE
+
+    print(_markdown(res))
+    print(f"\nworst drop: {res.worst_drop_points:+.2f} pts "
+          f"(threshold {args.max_drop_points:+.2f}) -> {res.verdict.value.upper()}")
+
+    if args.report:
+        Path(args.report).parent.mkdir(parents=True, exist_ok=True)
+        Path(args.report).write_text(
+            json.dumps(res.to_dict(), indent=2) + "\n", encoding="utf-8"
+        )
+
+    return EXIT_OK if res.verdict is Verdict.PASS else EXIT_REGRESSION
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

quant_regress/evalset.py

@@ -0,0 +1,69 @@
+"""Eval-set loading for quant-regress.
+
+Format: JSONL, one case per line, each ``{"id", "prompt", "expected"}``.
+Comparison is exact-after-normalisation by design: a fuzzy scorer would hide
+the marginal-decision damage this tool exists to detect.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Iterable
+
+
+class EvalSetError(ValueError):
+    """Raised when an eval set is malformed. Always names the offending row."""
+
+
+def normalise(text: str) -> str:
+    """Lowercase, strip, and collapse internal whitespace."""
+    return " ".join(str(text).split()).strip().lower()
+
+
+@dataclass(frozen=True)
+class Case:
+    id: str
+    prompt: str
+    expected: str
+
+
+class EvalSet:
+    def __init__(self, cases: Iterable[Case]):
+        self.cases = list(cases)
+
+    def __len__(self) -> int:
+        return len(self.cases)
+
+    def __iter__(self):
+        return iter(self.cases)
+
+    @classmethod
+    def load(cls, path: str | Path) -> "EvalSet":
+        p = Path(path)
+        if not p.exists():
+            raise EvalSetError(f"eval set not found: {p}")
+        cases: list[Case] = []
+        seen: set[str] = set()
+        for lineno, raw in enumerate(p.read_text(encoding="utf-8").splitlines(), 1):
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise EvalSetError(f"line {lineno}: invalid JSON ({exc})") from exc
+            missing = [k for k in ("id", "prompt", "expected") if k not in row]
+            if missing:
+                raise EvalSetError(f"line {lineno}: missing {', '.join(missing)}")
+            cid = str(row["id"])
+            if cid in seen:
+                raise EvalSetError(f"line {lineno}: duplicate id {cid!r}")
+            seen.add(cid)
+            cases.append(
+                Case(id=cid, prompt=str(row["prompt"]), expected=normalise(row["expected"]))
+            )
+        if not cases:
+            raise EvalSetError(f"eval set is empty: {p}")
+        return cls(cases)

quant_regress/harness.py

@@ -0,0 +1,209 @@
+"""Core harness: run a task eval at several precisions and compare accuracy.
+
+The point of this module is to make the marginal-decision damage visible.
+Token-level proxies (perplexity, KL) routinely miss it; task accuracy does
+not, provided the whole set is actually run — which the harness enforces by
+scoring every case at every precision and refusing to report a partial run.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Callable, Sequence
+
+from .evalset import EvalSet, normalise
+
+SUPPORTED_PRECISIONS = ("fp32", "int8")
+
+
+class Verdict(str, Enum):
+    PASS = "pass"
+    FAIL = "fail"
+
+
+@dataclass
+class PrecisionResult:
+    precision: str
+    correct: int
+    total: int
+    seconds: float
+    misclassified: list[str] = field(default_factory=list)
+
+    @property
+    def accuracy(self) -> float:
+        return (self.correct / self.total) if self.total else 0.0
+
+
+@dataclass
+class ComparisonResult:
+    baseline: PrecisionResult
+    candidates: list[PrecisionResult]
+    max_drop_points: float
+    verdict: Verdict
+
+    @property
+    def worst_drop_points(self) -> float:
+        drops = [
+            (self.baseline.accuracy - c.accuracy) * 100.0 for c in self.candidates
+        ]
+        return max(drops) if drops else 0.0
+
+    def to_dict(self) -> dict:
+        return {
+            "baseline": {
+                "precision": self.baseline.precision,
+                "accuracy": round(self.baseline.accuracy, 4),
+                "correct": self.baseline.correct,
+                "total": self.baseline.total,
+                "seconds": round(self.baseline.seconds, 3),
+            },
+            "candidates": [
+                {
+                    "precision": c.precision,
+                    "accuracy": round(c.accuracy, 4),
+                    "correct": c.correct,
+                    "total": c.total,
+                    "seconds": round(c.seconds, 3),
+                    "drop_points": round(
+                        (self.baseline.accuracy - c.accuracy) * 100.0, 3
+                    ),
+                }
+                for c in self.candidates
+            ],
+            "worst_drop_points": round(self.worst_drop_points, 3),
+            "max_drop_points": self.max_drop_points,
+            "verdict": self.verdict.value,
+        }
+
+
+class QuantHarness:
+    """Measures task accuracy for a model at each requested precision.
+
+    ``model_factory(precision) -> model`` is injected so tests can run with a
+    deterministic fake and CI can run with a real model.
+    """
+
+    def __init__(
+        self,
+        model_factory: Callable[[str], object],
+        max_new_tokens: int = 32,
+        scorer: Callable[[str, str], bool] | None = None,
+        system_prompt: str | None = None,
+        labels: Sequence[str] | None = None,
+    ):
+        if scorer is None:
+            def scorer(pred: str, expected: str) -> bool:
+                return normalise(pred) == expected
+        self._factory = model_factory
+        self.max_new_tokens = max_new_tokens
+        self._scorer = scorer
+        self.system_prompt = system_prompt
+        # Label set for models that expose only logits and carry no id2label.
+        self.labels = list(labels) if labels else []
+
+    # -- public -----------------------------------------------------------
+    def compare(
+        self,
+        eval_set: EvalSet,
+        baseline_precision: str = "fp32",
+        candidate_precisions: Sequence[str] = ("int8",),
+        max_drop_points: float = 2.0,
+    ) -> ComparisonResult:
+        for p in (baseline_precision, *candidate_precisions):
+            if p not in SUPPORTED_PRECISIONS:
+                raise ValueError(
+                    f"unsupported precision {p!r}; supported: {SUPPORTED_PRECISIONS}"
+                )
+        if not len(eval_set):
+            raise ValueError("eval set is empty")
+
+        baseline = self._measure(eval_set, baseline_precision)
+        candidates = [self._measure(eval_set, p) for p in candidate_precisions]
+
+        # Every precision must have scored the whole set, or the comparison
+        # is meaningless. Refuse rather than report a partial number.
+        for r in (baseline, *candidates):
+            if r.total != len(eval_set):
+                raise RuntimeError(
+                    f"incomplete run at {r.precision}: scored {r.total} of "
+                    f"{len(eval_set)} cases"
+                )
+
+        worst = max(
+            ((baseline.accuracy - c.accuracy) * 100.0 for c in candidates), default=0.0
+        )
+        return ComparisonResult(
+            baseline=baseline,
+            candidates=candidates,
+            max_drop_points=max_drop_points,
+            verdict=Verdict.FAIL if worst > max_drop_points else Verdict.PASS,
+        )
+
+    # -- internals --------------------------------------------------------
+    def _measure(self, eval_set: EvalSet, precision: str) -> PrecisionResult:
+        model = self._factory(precision)
+        tok = getattr(model, "tokenizer", None)
+        correct = 0
+        mis: list[str] = []
+        t0 = time.perf_counter()
+        for case in eval_set:
+            pred = self._predict(model, tok, case.prompt)
+            ok = self._scorer(pred, case.expected)
+            correct += int(ok)
+            if not ok:
+                mis.append(case.id)
+        return PrecisionResult(
+            precision=precision,
+            correct=correct,
+            total=len(eval_set),
+            seconds=time.perf_counter() - t0,
+            misclassified=mis,
+        )
+
+    def _predict(self, model, tok, prompt: str) -> str:
+        """Return the model's answer as text.
+
+        Three supported model shapes, in priority order:
+
+        1. A real HF model with ``generate`` + tokenizer — decode the output.
+        2. A model exposing ``.answer(prompt)`` — used by fakes and by any model
+           whose output is not free text (a classifier, a scorer).
+        3. A bare callable returning ``.logits`` — decoded via the model's own
+           ``id2label`` when it has one, else by argmax over a caller-supplied
+           label set.
+
+        Shape 3 deliberately does NOT stringify the raw logit. A logit is not an
+        answer: ``str(0.95)`` can never equal ``"yes"``, so stringifying it made
+        every such model score 0% and silently defeated the regression gate.
+        """
+        # 2. explicit text answer
+        answer = getattr(model, "answer", None)
+        if callable(answer):
+            return str(answer(prompt))
+
+        # 1. real HF generation path
+        if tok is not None and hasattr(model, "generate"):
+            text = prompt if self.system_prompt is None else f"{self.system_prompt}\n{prompt}"
+            inputs = tok(text, return_tensors="pt")
+            inputs = {k: v.to(getattr(model, "device", "cpu")) for k, v in inputs.items()}
+            out = model.generate(**inputs, max_new_tokens=self.max_new_tokens,
+                                 do_sample=False)
+            return tok.decode(out[0], skip_special_tokens=True)
+
+        # 3. logits -> label
+        logits = model(**{"input_ids": [[1]], "attention_mask": [[1]]}).logits
+        row = logits[0]
+        idx = int(max(range(len(row)), key=lambda i: float(row[i])))
+        id2label = getattr(model, "config", None)
+        id2label = getattr(id2label, "id2label", None) if id2label else None
+        if id2label:
+            return str(id2label.get(idx, idx))
+        if self.labels:
+            return str(self.labels[idx]) if idx < len(self.labels) else str(idx)
+        raise RuntimeError(
+            "cannot map model output to text: the model exposes no .answer(), "
+            "no generate()+tokenizer, and no id2label. Give the fake model an "
+            "answer(prompt) method, or pass labels=[...] to QuantHarness."
+        )

quant_regress-0.1.0.dist-info/METADATA

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: quant-regress
+Version: 0.1.0
+Summary: Fail CI when quantization costs you task accuracy. Task-level regression testing for quantized models.
+Author: DawnofGenX
+License: MIT
+Project-URL: Homepage, https://github.com/DawnofGenX/quant-regress
+Project-URL: Issues, https://github.com/DawnofGenX/quant-regress/issues
+Keywords: quantization,regression-testing,evaluation,ci,llm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: torch>=2.0
+Requires-Dist: transformers>=4.40
+Requires-Dist: numpy>=1.24
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+
+# quant-regress
+
+Fail CI when quantization costs you task accuracy.
+
+Token-level proxies — perplexity, KL divergence — routinely report that a
+quantized model is fine. Task accuracy does not, provided you actually measure
+it. This action measures a task eval at each precision and fails the build when
+accuracy drops past a threshold you set.
+
+## Why
+
+<!-- claim:int8-collapse -->
+On a cross-encoder claim verifier, dynamic INT8 quantization left aggregate
+accuracy looking survivable while destroying the capability that mattered:
+
+<!-- claim:int8-collapse -->
+| precision | accuracy | correct |
+|---|---|---|
+| fp32 (baseline) | 100.0% <!-- claim:int8-collapse --> | 20/20 |
+| int8 | 0.0% <!-- claim:int8-collapse --> | 0/20 |
+
+Regenerate with `python scripts/regen_int8_collapse.py` — every figure above
+comes from that script, and `tests/test_claims_are_reproducible.py` fails the
+build if a number in this file stops matching it.
+
+The pattern that matters: the quantized model still runs, still returns a valid
+answer, and still looks like a working model. It is simply wrong every time.
+That is why a token-level proxy can clear it and a task eval cannot.
+
+## Usage
+
+```yaml
+- uses: DawnofGenX/quant-regress@v1
+  with:
+    eval: evals/tasks.jsonl
+    model: meta-llama/Llama-3.2-1B-Instruct
+    max-drop-points: 2
+```
+
+Exit codes: `0` within tolerance, `1` accuracy regressed, `2` bad configuration.
+
+## Constraints worth knowing
+
+- **CPU only.** `quantize_dynamic` emits CPU-only quantized kernels; moving the
+  model to CUDA raises at forward time. GitHub-hosted runners have no GPU, so
+  this is not a limitation in CI — but it does mean GPU-only quantizers
+  (AWQ/GPTQ/GGUF) are out of scope for v1.
+- **The threshold unit is accuracy points**, not a ratio. `--max-drop-points: 2`
+  means "fail if any candidate is more than 2 points below the baseline".
+- Comparison is exact-after-normalisation by default. Pass a scorer if you need
+  semantic equivalence.
+- Models are downloaded at run time. A 1B model is ~2 GB; check runner disk.
+
+## Honest limits
+
+- Only `fp32` and dynamic `int8` are supported in v1.
+- The library runs generation, so it needs a task whose success is checkable in
+  code. It is not an LLM-judge harness.
+- **The bundled `evals/fixtures/collapse.jsonl` is model-specific.** Its cases
+  expect the answer `yes`, so it only demonstrates a collapse for a model that
+  answers `yes` when unquantized. Pointed at an untuned tiny LM it scores 0/N at
+  both precisions, produces no drop, and the gate correctly reports PASS. Use it
+  to see the output format; bring your own eval for a real check.
+- What the repo's own CI pins is the **exit-code contract** (0 holds, 1 regressed,
+  2 misconfigured), not any particular model's accuracy — see
+  `tests/test_selftest_gate.py`.
+- No published adoption yet. If you use it, an issue saying what you quantized
+  and what broke is genuinely useful.

quant_regress-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+quant_regress/__init__.py,sha256=FdX062901mNltHsvTWGpnAKR4MWqClsXuKihCBXlHYk,457
+quant_regress/adapter.py,sha256=iIBYKxRBUaWzRjrZcix10nZLJoNLWAngZggDjkpxlIE,2401
+quant_regress/cli.py,sha256=aTSbXbD_igfQwwULFm2lpuH_IYdjUYA824qKi6cLym8,3712
+quant_regress/evalset.py,sha256=C7RtyoijUOm4crpfe9V5Pg0NwDy_JOOSmDrBBZDrG40,2144
+quant_regress/harness.py,sha256=P3nPYRe99-Sh1Wb8SydGWFX0KFYm1CGiLUrcuAXFx4k,7852
+quant_regress-0.1.0.dist-info/METADATA,sha256=Hev2UbdF0d0nVwD4hQgHRb5eJoiPzMA0lM_ArSVgeAY,3752
+quant_regress-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+quant_regress-0.1.0.dist-info/entry_points.txt,sha256=oRiCjEh4r0wiHGdzNeGw8YktnknxPSujgpMjcDF4hCA,57
+quant_regress-0.1.0.dist-info/top_level.txt,sha256=uQCAYM3v-PLGcdsYS4qoFdvTl-nGvO9RXZvT9W-Wqig,14
+quant_regress-0.1.0.dist-info/RECORD,,

quant_regress-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

quant_regress-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+quant-regress = quant_regress.cli:main

quant_regress-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+quant_regress