quantgap 0.1.0__py3-none-any.whl

quantgap/hub/packs.py

@@ -0,0 +1,142 @@
+"""Find, download and publish packs (Hugging Face datasets tagged `quantgap-pack`)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from quantgap.config import Settings
+from quantgap.models import Manifest
+from quantgap.pack.reader import Pack
+
+PACK_TAG = "quantgap-pack"
+
+
+class PackNotFound(RuntimeError):
+    pass
+
+
+def _norm(s: str | None) -> str:
+    return (s or "").strip().lower().removeprefix("https://huggingface.co/")
+
+
+def list_packs(settings: Settings, model_query: str | None = None) -> list[tuple[str, Manifest]]:
+    """(dataset id, manifest) for every known pack, optionally filtered by model name."""
+    from huggingface_hub import HfApi, hf_hub_download
+
+    api = HfApi()
+    out: list[tuple[str, Manifest]] = []
+    for ds in api.list_datasets(filter=PACK_TAG, author=settings.hub.packs_org):
+        if model_query and model_query.lower() not in ds.id.lower():
+            continue
+        try:
+            path = hf_hub_download(ds.id, "manifest.json", repo_type="dataset")
+            out.append((ds.id, Manifest.model_validate(json.loads(Path(path).read_text("utf-8")))))
+        except Exception:
+            continue
+    return out
+
+
+def load_pack(ref: str, verify: bool = True) -> Pack:
+    """`ref` is a local folder or a Hub dataset id like `quantgap-packs/Qwen3-8B`."""
+    p = Path(ref).expanduser()
+    if p.is_dir():
+        return Pack.load(p, verify=verify)
+    from huggingface_hub import snapshot_download
+
+    try:
+        local = snapshot_download(ref, repo_type="dataset")
+    except Exception as e:
+        raise PackNotFound(
+            f"Could not find a pack at {ref!r} (not a folder, not a Hub dataset): {e}"
+        ) from e
+    return Pack.load(local, verify=verify)
+
+
+def auto_find_pack(base_model: str | None, settings: Settings) -> Pack:
+    """Find the pack that matches a quant's base model, or explain what to do (spec E1)."""
+    if not base_model:
+        raise PackNotFound(
+            "This quant does not say which model it was made from, so no pack can be chosen "
+            "automatically. Pass --pack PACK_REF (a pack folder or a Hub dataset id). "
+            "List packs with: quantgap packs list"
+        )
+    packs = list_packs(settings)
+    # Exact model name first. Many GGUFs name the parent of an instruct model as base_model
+    # (Qwen2.5-0.5B for Qwen2.5-0.5B-Instruct), so fall back to the same model family.
+    matches = [(ds, m) for ds, m in packs if _norm(m.model.repo) == _norm(base_model)]
+    if not matches:
+        matches = [(ds, m) for ds, m in packs if _same_family(base_model, m.model.repo)]
+    if not matches:
+        raise PackNotFound(
+            f"No pack found for {base_model}. Make one with: quantgap pack {base_model} "
+            "(see docs/packs.md), or pass --pack."
+        )
+    if len(matches) > 1:
+        options = "\n".join(f"  {ds}  ({m.short_name}, probes {m.probes.set})" for ds, m in matches)
+        raise PackNotFound(
+            f"More than one pack matches {base_model}. Pick one with --pack:\n{options}"
+        )
+    return load_pack(matches[0][0])
+
+
+def _stem(repo: str) -> str:
+    return _norm(repo).split("/")[-1]
+
+
+def _same_family(a: str, b: str) -> bool:
+    x, y = _stem(a), _stem(b)
+    return x == y or x.startswith(y) or y.startswith(x)
+
+
+def fine_tune_warning(base_model: str | None, pack: Pack) -> str | None:
+    """E2: the quant says it comes from a different model than the pack's.
+
+    GGUF files often name the *parent* of an instruct model (Qwen2.5-0.5B for
+    Qwen2.5-0.5B-Instruct), so names where one starts with the other count as the same family.
+    """
+    if not base_model:
+        return None
+    if _same_family(base_model, pack.manifest.model.repo):
+        return None
+    return (
+        f"This looks like a fine-tune ({base_model}, not {pack.manifest.model.repo}). "
+        "Its drift includes the fine-tuning, not only quantization."
+    )
+
+
+def dataset_card(m: Manifest) -> str:
+    lic = m.model.license or "unknown (check the model page before sharing)"
+    return f"""---
+license: cc-by-4.0
+tags:
+- {PACK_TAG}
+- quantization
+pretty_name: quantgap pack for {m.model.repo}
+---
+
+# quantgap pack: {m.model.repo}
+
+Pack id: `{m.pack_id}`. Made with quantgap {m.runner.quantgap}.
+
+This pack holds the reference model's top-{m.k} next-token log probabilities on the quantgap
+probe set `{m.probes.set}` ({m.probes.n_positions:,} positions). Use it with
+`quantgap score QUANT --pack <this dataset>` to measure how far a quantized copy of the model
+drifts from the original, without running the original.
+
+- Reference model: `{m.model.repo}` @ `{m.model.revision}` ({m.model.dtype})
+- Reference model license: **{lic}**
+- The pack contains model outputs (log probabilities) on openly licensed texts. It does not
+  contain the model weights.
+"""
+
+
+def push_pack(pack_dir: str | Path, repo_id: str, token: str | None = None) -> str:
+    from huggingface_hub import HfApi
+
+    pack = Pack.load(pack_dir)
+    api = HfApi(token=token)
+    api.create_repo(repo_id, repo_type="dataset", exist_ok=True)
+    (Path(pack_dir) / "README.md").write_text(dataset_card(pack.manifest), encoding="utf-8")
+    api.upload_folder(folder_path=str(pack_dir), repo_id=repo_id, repo_type="dataset")
+    return f"https://huggingface.co/datasets/{repo_id}"

quantgap/hub/resolve.py

@@ -0,0 +1,247 @@
+"""Turn what the user typed into a quant on disk: local paths, hf:// URLs, repo globs."""
+
+from __future__ import annotations
+
+import fnmatch
+import re
+import struct
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from quantgap.backends.base import ResolvedSource
+from quantgap.util import free_disk_bytes, human_size, sha256_file
+
+_LABEL_RE = re.compile(
+    r"((?:UD-)?(?:IQ\d(?:_[A-Z0-9]+)*|Q\d(?:_[A-Z0-9]+)*|MXFP4(?:_MOE)?|BF16|F16|F32|NVFP4))",
+    re.IGNORECASE,
+)
+
+
+class ResolveError(ValueError):
+    """The quant reference could not be understood or found. The message says what to try."""
+
+
+@dataclass
+class HFRef:
+    repo: str
+    path: str | None = None
+
+
+def parse_hf_url(url: str) -> HFRef:
+    """`hf://org/repo` or `hf://org/repo/sub/folder/file.gguf`."""
+    if not url.startswith("hf://"):
+        raise ResolveError(f"{url!r} is not an hf:// URL. Example: hf://org/repo/file.gguf")
+    parts = [p for p in url[len("hf://") :].split("/") if p]
+    if len(parts) < 2:
+        raise ResolveError(
+            f"{url!r} needs at least org/repo. Example: hf://unsloth/Qwen3-8B-GGUF/Qwen3-8B-Q4_K_M.gguf"
+        )
+    return HFRef(repo="/".join(parts[:2]), path="/".join(parts[2:]) or None)
+
+
+def label_from_filename(name: str) -> str:
+    stem = Path(name).stem
+    m = _LABEL_RE.search(stem)
+    return m.group(1).upper() if m else stem
+
+
+def expand_glob(repo: str, pattern: str, files: list[str] | None = None) -> list[str]:
+    """Repo file paths matching a glob. `files` can be given (tests); otherwise asks the Hub."""
+    if files is None:
+        from huggingface_hub import HfApi
+
+        files = HfApi().list_repo_files(repo)
+    hits = sorted(
+        f for f in files if fnmatch.fnmatch(f, pattern) or fnmatch.fnmatch(Path(f).name, pattern)
+    )
+    # Split GGUFs ("-00001-of-00003.gguf") are not supported yet: keep only single files or first parts.
+    out = []
+    for f in hits:
+        m = re.search(r"-(\d{5})-of-(\d{5})\.gguf$", f)
+        if m and m.group(1) != "00001":
+            continue
+        out.append(f)
+    if not out:
+        raise ResolveError(f"No files in {repo} match {pattern!r}.")
+    return out
+
+
+# ------------------------------------------------------------------ GGUF metadata (no deps)
+
+_GGUF_TYPES = {
+    0: "B",
+    1: "b",
+    2: "H",
+    3: "h",
+    4: "I",
+    5: "i",
+    6: "f",
+    7: "?",
+    10: "Q",
+    11: "q",
+    12: "d",
+}
+
+
+def read_gguf_metadata(path: str | Path, wanted: set[str] | None = None) -> dict[str, Any]:
+    """Read key/value metadata from a GGUF header. Arrays are skipped. Never loads tensors."""
+    out: dict[str, Any] = {}
+    with open(path, "rb") as f:
+        if f.read(4) != b"GGUF":
+            raise ResolveError(f"{path} is not a GGUF file.")
+        (_version,) = struct.unpack("<I", f.read(4))
+        _n_tensors, n_kv = struct.unpack("<QQ", f.read(16))
+
+        def read_str() -> str:
+            (n,) = struct.unpack("<Q", f.read(8))
+            return f.read(n).decode("utf-8", errors="replace")
+
+        def read_value(t: int) -> Any:
+            if t == 8:
+                return read_str()
+            if t == 9:  # array
+                (et,) = struct.unpack("<I", f.read(4))
+                (n,) = struct.unpack("<Q", f.read(8))
+                for _ in range(n):
+                    read_value(et)
+                return None
+            fmt = _GGUF_TYPES[t]
+            return struct.unpack("<" + fmt, f.read(struct.calcsize(fmt)))[0]
+
+        for _ in range(n_kv):
+            key = read_str()
+            (t,) = struct.unpack("<I", f.read(4))
+            val = read_value(t)
+            if val is not None and (wanted is None or key in wanted):
+                out[key] = val
+            if wanted is not None and wanted <= out.keys():
+                break
+    return out
+
+
+def gguf_base_model(path: str | Path) -> str | None:
+    """The `org/repo` of the model this GGUF was made from, when the file says so."""
+    keys = {"general.base_model.0.repo_url", "general.base_model.0.name"}
+    try:
+        meta = read_gguf_metadata(path, keys)
+    except (OSError, ResolveError, struct.error, KeyError):
+        return None
+    url = meta.get("general.base_model.0.repo_url", "")
+    m = re.search(r"huggingface\.co/([^/]+/[^/?#]+)", url)
+    if m:
+        return m.group(1)
+    return meta.get("general.base_model.0.name")
+
+
+# ------------------------------------------------------------------ resolve
+
+
+def _dir_format(path: Path) -> str:
+    cfg = path / "config.json"
+    if cfg.is_file() and '"quantization"' in cfg.read_text(encoding="utf-8", errors="ignore"):
+        return "mlx"
+    return "hf"
+
+
+def _check_disk(size: int, cache_dir: str | None) -> None:
+    from huggingface_hub import constants
+
+    where = cache_dir or constants.HF_HUB_CACHE
+    free = free_disk_bytes(where)
+    need = int(size * 1.1)
+    if free < need:
+        raise ResolveError(
+            f"Not enough disk space in {where}: need about {human_size(need)}, "
+            f"{human_size(free)} free. Set HF_HOME to a bigger drive."
+        )
+    print(f"Downloading to {where} ({human_size(size)}, {human_size(free)} free)")
+
+
+def resolve_quant(
+    quant: str,
+    *,
+    base_url: str | None = None,
+    model_name: str | None = None,
+    compute_hash: bool = True,
+) -> ResolvedSource:
+    """Find a quant. Downloads from the Hub when given an hf:// URL."""
+    if quant.startswith("openai://"):
+        if not base_url:
+            raise ResolveError("openai:// needs --base-url, for example http://localhost:8000/v1")
+        name = quant[len("openai://") :] or (model_name or "")
+        return ResolvedSource(
+            source=quant,
+            label=name or "server-model",
+            format="openai",
+            extra={"model_name": model_name or name},
+        )
+    if quant.startswith("hf://"):
+        ref = parse_hf_url(quant)
+        return _resolve_hf(ref, quant)
+    p = Path(quant).expanduser()
+    if p.is_file():
+        src = ResolvedSource(
+            source=str(p),
+            label=label_from_filename(p.name) if p.suffix == ".gguf" else p.stem,
+            path=p,
+            size_bytes=p.stat().st_size,
+            format="gguf" if p.suffix.lower() == ".gguf" else "hf",
+        )
+        if src.format == "gguf":
+            src.base_model = gguf_base_model(p)
+        if compute_hash:
+            src.sha256 = sha256_file(p)
+        return src
+    if p.is_dir():
+        size = sum(f.stat().st_size for f in p.rglob("*") if f.is_file())
+        return ResolvedSource(
+            source=str(p), label=p.name, path=p, size_bytes=size, format=_dir_format(p)
+        )
+    raise ResolveError(
+        f"Cannot find {quant!r}. Use a local file or folder, hf://org/repo/file.gguf, "
+        "or openai://NAME with --base-url."
+    )
+
+
+def _resolve_hf(ref: HFRef, original: str) -> ResolvedSource:
+    from huggingface_hub import HfApi, hf_hub_download, snapshot_download
+
+    api = HfApi()
+    if ref.path and ref.path.endswith(".gguf"):
+        info = api.get_paths_info(ref.repo, [ref.path])
+        if not info:
+            raise ResolveError(f"{ref.path} was not found in {ref.repo}.")
+        size = info[0].size or 0
+        sha = info[0].lfs.sha256 if getattr(info[0], "lfs", None) else None
+        _check_disk(size, None)
+        local = Path(hf_hub_download(ref.repo, ref.path))
+        if sha is None:
+            sha = sha256_file(local)
+        return ResolvedSource(
+            source=f"hf://{ref.repo}/{ref.path}",
+            label=label_from_filename(ref.path),
+            path=local,
+            size_bytes=size or local.stat().st_size,
+            format="gguf",
+            publisher=ref.repo.split("/")[0],
+            base_model=gguf_base_model(local),
+            sha256=sha,
+        )
+    if ref.path:
+        raise ResolveError(
+            f"{original!r}: only .gguf files or whole repos (for MLX or Transformers) are supported."
+        )
+    # whole repo: MLX or Transformers folder
+    files = api.model_info(ref.repo, files_metadata=True).siblings or []
+    size = sum(s.size or 0 for s in files if s.rfilename.endswith((".safetensors", ".bin")))
+    _check_disk(size, None)
+    local = Path(snapshot_download(ref.repo))
+    return ResolvedSource(
+        source=f"hf://{ref.repo}",
+        label=ref.repo.split("/")[-1],
+        path=local,
+        size_bytes=size,
+        format=_dir_format(local),
+        publisher=ref.repo.split("/")[0],
+    )

quantgap/hub/submit.py

@@ -0,0 +1,87 @@
+"""Send a result to the shared results dataset as a pull request."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from quantgap.models import RESULT_SCHEMA, ScoreResult, Submission
+
+
+class SubmitError(ValueError):
+    pass
+
+
+def load_result(path: str | Path) -> ScoreResult:
+    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    if data.get("schema") != RESULT_SCHEMA:
+        raise SubmitError(f"{path} is not a quantgap result (schema {data.get('schema')!r}).")
+    return ScoreResult.model_validate(data)
+
+
+def build_submission(result: ScoreResult) -> Submission:
+    """Check a result is submittable and describe exactly what would be sent."""
+    if result.quick:
+        raise SubmitError(
+            "This is a --quick result. Run without --quick to get a submittable result."
+        )
+    if not result.quant.sha256:
+        raise SubmitError(
+            "This result has no file hash, so nobody could re-check it. Score a GGUF file or a "
+            "downloaded model folder, not a bare server."
+        )
+    path = f"results/{result.pack_id}/{result.quant.sha256}.json"
+    m = result.metrics
+    d = m.damage
+    lines = [
+        f"Result for **{result.quant.label}** ({result.quant.source})",
+        "",
+        f"- pack: `{result.pack_name or result.pack_id}` (id `{result.pack_id}`)",
+        f"- quant file SHA-256: `{result.quant.sha256}`",
+        f"- backend: {result.backend.name} ({result.backend.version})",
+        f"- hardware: {result.hardware}",
+        f"- KLD mean: {m.kld_mean:.5f} (95% CI {m.kld_mean_ci95[0]:.5f} to {m.kld_mean_ci95[1]:.5f})",
+        f"- top-1 agreement: {m.top1_agree * 100:.2f}%",
+        f"- approximate (server top-N): {'yes' if result.approximate else 'no'}",
+    ]
+    if d:
+        lines.append(
+            f"- loops: {d.loops} (reference {d.loops_ref}), wrong script: {d.wrong_script}, "
+            f"valid JSON: {d.json_valid}/{d.json_n} (reference {d.json_valid_ref})"
+        )
+    lines += [
+        "",
+        "To re-check: `quantgap score <same file> --pack <same pack>` should give a KLD mean within 2%.",
+        "",
+        "Submitted with quantgap " + result.quantgap + ".",
+    ]
+    return Submission(
+        path=path,
+        pr_title=f"Add result: {result.quant.label} on {result.pack_name or result.pack_id}",
+        pr_body="\n".join(lines),
+        result=result,
+    )
+
+
+def submit(
+    result_file: str | Path,
+    results_repo: str,
+    dry_run: bool = False,
+    token: str | None = None,
+) -> tuple[Submission, str | None]:
+    """Returns (submission, PR url). With dry_run nothing is sent and the URL is None."""
+    sub = build_submission(load_result(result_file))
+    if dry_run:
+        return sub, None
+    from huggingface_hub import CommitOperationAdd, HfApi
+
+    api = HfApi(token=token)
+    info = api.create_commit(
+        repo_id=results_repo,
+        repo_type="dataset",
+        operations=[CommitOperationAdd(sub.path, sub.result.to_json().encode("utf-8"))],
+        commit_message=sub.pr_title,
+        commit_description=sub.pr_body,
+        create_pr=True,
+    )
+    return sub, info.pr_url

quantgap/models.py

@@ -0,0 +1,202 @@
+"""Typed data models. Pydantic validates JSON files and gives us JSON Schema for free."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+PACK_SCHEMA = "quantgap.pack/1"
+RESULT_SCHEMA = "quantgap.result/1"
+
+GROUPS = ("general", "chat", "code", "math", "multilingual", "json")
+
+
+class _Model(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+# --------------------------------------------------------------------------- probes
+
+
+class ProbeSeq(_Model):
+    """One row of the probe set (the texts every model is scored on)."""
+
+    seq_id: int
+    group: Literal["general", "chat", "code", "math", "multilingual", "json"]
+    kind: Literal["text", "chat"] = "text"
+    lang: str = "en"
+    text: str | None = None  # kind == "text"
+    messages: list[dict[str, str]] | None = None  # kind == "chat"
+    license: str
+    source: str
+    # Used by the damage checks. For text rows the prompt is the first words of the text.
+    greedy: bool = False
+    greedy_prompt: str | None = None
+
+
+# --------------------------------------------------------------------------- pack manifest
+
+
+class ModelRef(_Model):
+    repo: str
+    revision: str | None = None
+    dtype: str = "bfloat16"
+    license: str | None = None
+
+
+class TokenizerInfo(_Model):
+    sha256: str | None = None
+    vocab_size: int
+    bos: bool = True
+    bos_id: int | None = None
+    eos_id: int | None = None
+    check_ids: list[int] = Field(default_factory=list)
+
+
+class ProbesInfo(_Model):
+    set: str
+    sha256: str
+    n_sequences: int
+    n_positions: int
+
+
+class GreedyInfo(_Model):
+    n_prompts: int
+    max_new_tokens: int
+
+
+class RunnerInfo(_Model):
+    quantgap: str
+    torch: str | None = None
+    transformers: str | None = None
+    device: str | None = None
+
+
+class Manifest(_Model):
+    schema_: str = Field(default=PACK_SCHEMA, alias="schema")
+    model: ModelRef
+    tokenizer: TokenizerInfo
+    probes: ProbesInfo
+    chat_template_sha256: str | None = None
+    k: int
+    logp_dtype: Literal["float16", "float32"] = "float32"
+    greedy: GreedyInfo
+    runner: RunnerInfo
+    files: dict[str, str] = Field(default_factory=dict)
+    created_at: str
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    def canonical_json(self) -> str:
+        data = self.model_dump(by_alias=True, mode="json")
+        return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+
+    @property
+    def pack_id(self) -> str:
+        """First 16 hex chars of SHA-256 over the canonical manifest."""
+        return hashlib.sha256(self.canonical_json().encode("utf-8")).hexdigest()[:16]
+
+    @property
+    def short_name(self) -> str:
+        return f"{self.model.repo.split('/')[-1]}@{self.pack_id}"
+
+
+# --------------------------------------------------------------------------- results
+
+
+class QuantInfo(_Model):
+    source: str
+    sha256: str | None = None
+    size_bytes: int
+    label: str
+    publisher: str | None = None
+    format: str = "gguf"
+
+
+class BackendInfo(_Model):
+    name: str
+    version: str
+    settings: dict[str, Any] = Field(default_factory=dict)
+
+
+class GroupMetrics(_Model):
+    kld_mean: float
+    top1_agree: float
+    n_positions: int = 0
+
+
+class DamageMetrics(_Model):
+    """Every number is stored next to the reference's own number (section 7.4)."""
+
+    n_prompts: int
+    loops: int
+    loops_ref: int
+    loop_rate: float
+    wrong_script: int
+    wrong_script_ref: int
+    wrong_script_rate: float
+    json_valid: int
+    json_valid_ref: int
+    json_n: int
+    json_valid_rate: float
+    first_divergence_median: float
+
+
+class Metrics(_Model):
+    kld_mean: float
+    kld_mean_ci95: tuple[float, float]
+    kld_p99: float
+    kld_p999: float
+    top1_agree: float
+    ppl_ratio: float
+    n_positions: int
+    n_sequences: int
+    clamp_count: int = 0
+    by_group: dict[str, GroupMetrics] = Field(default_factory=dict)
+    damage: DamageMetrics | None = None
+
+
+class ScoreResult(_Model):
+    schema_: str = Field(default=RESULT_SCHEMA, alias="schema")
+    pack_id: str
+    pack_name: str | None = None
+    quant: QuantInfo
+    backend: BackendInfo
+    metrics: Metrics
+    hardware: str
+    quantgap: str
+    created_at: str
+    quick: bool = False
+    approximate: bool = False
+    nondeterministic: bool = False
+    warnings: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    def to_json(self) -> str:
+        return json.dumps(self.model_dump(by_alias=True, mode="json"), indent=2, ensure_ascii=False)
+
+    @property
+    def submittable(self) -> bool:
+        return not self.quick and self.quant.sha256 is not None
+
+
+class CompareReport(_Model):
+    pack_id: str
+    pack_name: str | None = None
+    results: list[ScoreResult]
+    budget_bytes: int | None = None
+    recommendation: str | None = None
+    assumptions: str | None = None
+
+
+class Submission(_Model):
+    """What `quantgap submit` writes into the results dataset."""
+
+    path: str
+    pr_title: str
+    pr_body: str
+    result: ScoreResult

quantgap/pack/__init__.py

@@ -0,0 +1,4 @@
+from quantgap.pack.reader import Pack, PackIntegrityError, SeqTopK
+from quantgap.pack.writer import PackWriter
+
+__all__ = ["Pack", "PackIntegrityError", "PackWriter", "SeqTopK"]