votum 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
votum/__init__.py ADDED
@@ -0,0 +1,12 @@
1
+ """Votum: single-pass decision model on Apertus-1.5-8B. One forward pass reads a state, a question and its options and
2
+ returns a probability for every option; no text is generated.
3
+
4
+ from votum import Votum
5
+ vm = Votum.from_pretrained("<org>/Votum-8B")
6
+ vm.decide({"state": "...", "questions": {"q1": {"type": "choice", "instructions": "...", "criteria": {"a": "...", "b": "..."}}}})
7
+ """
8
+ from .config import VotumConfig
9
+ from .modeling import Votum
10
+
11
+ __all__ = ["Votum", "VotumConfig"]
12
+ __version__ = "0.1.0"
votum/build.py ADDED
@@ -0,0 +1,135 @@
1
+ """Build a Votum checkpoint directory from the adapter form (LoRA adapter + pointer head + votum_config.json) and the
2
+ Apertus-1.5-8B base checkpoint. Both the PyTorch reference (votum.Votum) and the vLLM server (votum.vllm) load it.
3
+
4
+ votum build --adapter <dir | <org>/Votum-8B> --out <dir> [--base swiss-ai/Apertus-v1.5-8B | <dir>] [--exact] [--link]
5
+
6
+ Default (merged): W <- bf16(W + (alpha / r) * B @ A), computed in fp32 and stored in bf16. On the evaluation sets this
7
+ gives the same accuracy and benchmark scores as the unmerged model (98 % identical answers), is about 10 % faster on
8
+ vLLM and needs no LoRA support. --exact: the base weights are written unchanged and the adapter goes to lora/, served
9
+ through vLLM's LoRA kernels (the training-time computation exactly).
10
+ Written: the base's text stack only (model.language_model.* -> model.*; vision / audio tokenizers and lm_head are not
11
+ needed), the pointer head (pointer.o.weight, pointer.q0; fp32), the tokenizer files, votum_config.json, and a config.json
12
+ with model_type "apertus" and the decision_* fields read by votum.vllm. --link (with --exact): shards of a local base
13
+ that hold only text-stack tensors are symlinked instead of rewritten."""
14
+ from __future__ import annotations
15
+
16
+ import argparse, json, os, re, shutil
17
+ from pathlib import Path
18
+
19
+ from .config import CONFIG_NAME, VotumConfig
20
+
21
+ TEXT = "model.language_model."
22
+ TOKENIZER_FILES = ["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "chat_template.jinja"]
23
+ HEAD_NAME = "pointer_head.safetensors"
24
+ ADAPTER_FILES = [CONFIG_NAME, HEAD_NAME, "adapter_config.json", "adapter_model.safetensors"]
25
+ LORA = re.compile(r"base_model\.model\.model\.layers\.(\d+)\.(self_attn|mlp)\.(\w+)\.lora_([AB])\.weight")
26
+
27
+
28
+ def resolve(path_or_repo: str, revision: str | None = None, token=None, patterns: list[str] | None = None, subfolder: str = "") -> Path:
29
+ """a local directory, or a Hub repo downloaded with the given file patterns (inside subfolder)."""
30
+ p = Path(path_or_repo)
31
+ if p.exists(): return p / subfolder if subfolder else p
32
+ from huggingface_hub import snapshot_download
33
+ pats = [f"{subfolder}/{x}" if subfolder else x for x in (patterns or ["*"])]
34
+ d = Path(snapshot_download(path_or_repo, revision=revision, token=token, allow_patterns=pats))
35
+ return d / subfolder if subfolder else d
36
+
37
+
38
+ def adapter_dir(path_or_repo: str, token=None) -> Path:
39
+ """the adapter form: a directory with the four adapter files, or the adapter/ subfolder of a Votum repo."""
40
+ p = Path(path_or_repo)
41
+ if p.exists(): return p if (p / "adapter_model.safetensors").exists() else p / "adapter"
42
+ return resolve(path_or_repo, None, token, ADAPTER_FILES, subfolder="adapter")
43
+
44
+
45
+ def serving_config(base_cfg: dict, cfg: VotumConfig, tok) -> dict:
46
+ t = base_cfg.get("text_config", base_cfg)
47
+ rp = dict(t.get("rope_parameters") or {**(t.get("rope_scaling") or {}), "rope_theta": t["rope_theta"]})
48
+ h = cfg.header
49
+ return {
50
+ "architectures": ["VotumDecisionModel"], "model_type": "apertus", "torch_dtype": "bfloat16",
51
+ "hidden_size": t["hidden_size"], "intermediate_size": t["intermediate_size"], "num_hidden_layers": t["num_hidden_layers"],
52
+ "num_attention_heads": t["num_attention_heads"], "num_key_value_heads": t["num_key_value_heads"],
53
+ "hidden_act": t["hidden_act"], "rms_norm_eps": t["rms_norm_eps"], "vocab_size": t["vocab_size"],
54
+ "max_position_embeddings": t["max_position_embeddings"], "attention_bias": t.get("attention_bias", False),
55
+ "mlp_bias": t.get("mlp_bias", False), "qk_norm": t.get("qk_norm", True), "post_norm": t.get("post_norm", False),
56
+ "tie_word_embeddings": False, "bos_token_id": t["bos_token_id"], "eos_token_id": t["eos_token_id"], "pad_token_id": t["pad_token_id"],
57
+ "rope_theta": rp.pop("rope_theta"), "rope_scaling": rp,
58
+ "decision_opt_end_id": cfg.opt_end_id, "decision_bidirectional_layers": cfg.bidirectional_layers,
59
+ "decision_pointer_rank": cfg.pointer_rank, "decision_temperatures": cfg.temperatures,
60
+ "decision_shared_positions": cfg.shared_positions, "decision_close_last": cfg.close_last,
61
+ "decision_options_header_ids": tok("\n\nOptions:\n", add_special_tokens=False).input_ids[-2:], # "Options", ":\n"
62
+ "decision_question_header_ids": tok("\n\nQuestion:\n", add_special_tokens=False).input_ids[-2:], # "Question", ":\n"
63
+ "decision_header": {"max_q": int(h.get("max_q", 8)), "max_opts": h.get("max_opts"), "sort": bool(h.get("sort", True)),
64
+ "chunk": bool(h.get("chunk", False))},
65
+ }
66
+
67
+
68
+ def build(out: Path, adapter: str, base: str | None = None, exact: bool = False, link: bool = False, token=None) -> None:
69
+ import torch
70
+ from safetensors import safe_open
71
+ from safetensors.torch import load_file, save_file
72
+ from transformers import AutoTokenizer
73
+ out.mkdir(parents=True, exist_ok=True)
74
+ ad = adapter_dir(adapter, token); cfg = VotumConfig.load(ad)
75
+ bd = resolve(base or cfg.base_model, None if base else cfg.base_revision, token,
76
+ ["config.json", "model.safetensors.index.json", "*.safetensors", *TOKENIZER_FILES])
77
+ for f in TOKENIZER_FILES:
78
+ if (bd / f).exists(): shutil.copy(bd / f, out / f)
79
+ tok = AutoTokenizer.from_pretrained(str(out))
80
+ assert tok.convert_tokens_to_ids(cfg.opt_end_token) == cfg.opt_end_id
81
+ delta, scale = {}, None
82
+ if not exact:
83
+ acfg = json.loads((ad / "adapter_config.json").read_text()); scale = acfg["lora_alpha"] / acfg["r"]
84
+ for k, v in load_file(str(ad / "adapter_model.safetensors")).items():
85
+ m = LORA.fullmatch(k); assert m, k
86
+ delta.setdefault(f"model.layers.{m[1]}.{m[2]}.{m[3]}.weight", {})[m[4]] = v
87
+ link = False
88
+ wmap = json.loads((bd / "model.safetensors.index.json").read_text())["weight_map"]
89
+ plan = [] # (base shard, {base key: our key}, all keys of the shard)
90
+ for shard in sorted(set(wmap.values())):
91
+ with safe_open(str(bd / shard), framework="pt") as f: names = list(f.keys()) # every tensor in the file (vLLM reads them all)
92
+ text = {k: "model." + k[len(TEXT):] for k in names if k.startswith(TEXT)}
93
+ text.update({k: k for k in names if k.startswith("model.") and not k.startswith(TEXT) and k.split(".")[1] in ("layers", "embed_tokens", "norm")})
94
+ if text: plan.append((shard, text, names))
95
+ n = len(plan) + 1; index = {}
96
+ for i, (shard, text, names) in enumerate(plan, 1):
97
+ fn = f"model-{i:05d}-of-{n:05d}.safetensors"; dst = out / fn
98
+ if dst.exists() or dst.is_symlink(): dst.unlink()
99
+ if link and len(text) == len(names) and all(k == v for k, v in text.items()): # a pure text-stack shard: link it unchanged
100
+ os.symlink((bd / shard).resolve(), dst); index.update({k: fn for k in text}); print(f"{fn}: linked ({len(text)} tensors)", flush=True)
101
+ continue
102
+ tensors = {}
103
+ with safe_open(str(bd / shard), framework="pt") as f:
104
+ for k, name in text.items():
105
+ w = f.get_tensor(k)
106
+ if name in delta:
107
+ d = delta.pop(name); w = (w.float() + scale * (d["B"].float() @ d["A"].float())).to(torch.bfloat16)
108
+ tensors[name] = w.contiguous()
109
+ save_file(tensors, str(dst), metadata={"format": "pt"}); index.update({k: fn for k in tensors}); print(f"{fn}: {len(tensors)} tensors", flush=True)
110
+ assert any(k.startswith("model.layers.") for k in index), "no text-stack weights found in the base checkpoint"
111
+ assert not delta, f"{len(delta)} LoRA targets not found in the base"
112
+ head = load_file(str(ad / HEAD_NAME)); fn = f"model-{n:05d}-of-{n:05d}.safetensors"
113
+ save_file({"pointer.o.weight": head["o.weight"].float().contiguous(), "pointer.q0": head["q0"].float().contiguous()}, str(out / fn),
114
+ metadata={"format": "pt"})
115
+ index.update({"pointer.o.weight": fn, "pointer.q0": fn})
116
+ (out / "model.safetensors.index.json").write_text(json.dumps({"metadata": {}, "weight_map": index}, indent=1))
117
+ if exact:
118
+ (out / "lora").mkdir(exist_ok=True); shutil.copy(ad / "adapter_model.safetensors", out / "lora" / "adapter_model.safetensors")
119
+ acfg = json.loads((ad / "adapter_config.json").read_text()); acfg.update(base_model_name_or_path=out.name, task_type="CAUSAL_LM")
120
+ (out / "lora" / "adapter_config.json").write_text(json.dumps(acfg, indent=1))
121
+ shutil.copy(ad / CONFIG_NAME, out / CONFIG_NAME)
122
+ (out / "config.json").write_text(json.dumps(serving_config(json.loads((bd / "config.json").read_text()), cfg, tok), indent=1))
123
+ print(f"{out}: {'exact (LoRA under lora/)' if exact else 'merged'} checkpoint ready")
124
+
125
+
126
+ def main(argv=None) -> None:
127
+ ap = argparse.ArgumentParser(prog="votum build"); ap.add_argument("--out", required=True)
128
+ ap.add_argument("--adapter", required=True, help="adapter directory, or a Votum repo id / directory (uses its adapter/)")
129
+ ap.add_argument("--base", default=None); ap.add_argument("--exact", action="store_true"); ap.add_argument("--link", action="store_true")
130
+ a = ap.parse_args(argv)
131
+ build(Path(a.out), a.adapter, a.base, a.exact, a.link)
132
+
133
+
134
+ if __name__ == "__main__":
135
+ main()
votum/cli.py ADDED
@@ -0,0 +1,48 @@
1
+ """votum command line.
2
+
3
+ votum decide --repo <org>/Votum-8B [--request req.json | < req.json] [--exact] [--device cuda|cpu] PyTorch reference, one request
4
+ votum serve --model <org>/Votum-8B [--host 127.0.0.1] [--port 8011] vLLM HTTP server (/v1/systemone; more options: votum serve --help)
5
+ votum quantize --model <org>/Votum-8B --out <dir> FP8 copy of a checkpoint, about half the size (votum quantize --help)
6
+ votum build --adapter <org>/Votum-8B --out <dir> [--exact] [--link] [--base <dir>] rebuild a checkpoint from the adapter form
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import json, sys
11
+
12
+
13
+ def main(argv=None) -> None:
14
+ argv = list(sys.argv[1:] if argv is None else argv)
15
+ if not argv or argv[0] in ("-h", "--help"):
16
+ print(__doc__); return
17
+ cmd, rest = argv[0], argv[1:]
18
+ if cmd == "build":
19
+ from .build import main as m; m(rest)
20
+ elif cmd == "quantize":
21
+ from .fp8 import main as m; m(rest)
22
+ elif cmd == "serve":
23
+ from .vllm.server import main as m; m(rest)
24
+ elif cmd == "decide":
25
+ import argparse
26
+ ap = argparse.ArgumentParser(prog="votum decide", description="Answer one request with the PyTorch reference. The request JSON "
27
+ "({\"state\": ..., \"questions\": {...}}) comes from --request or stdin.")
28
+ ap.add_argument("--repo", required=True, help="Votum repo id or local checkpoint directory")
29
+ ap.add_argument("--request", help="JSON file with the request (default: read stdin)")
30
+ ap.add_argument("--exact", action="store_true", help="use the unmerged adapter in the repo's adapter/ folder on the pinned base")
31
+ ap.add_argument("--base", help="with --exact: base checkpoint (repo id or directory) instead of the pinned one")
32
+ ap.add_argument("--device", help="cuda (default when available) or cpu"); a = ap.parse_args(rest)
33
+ if a.request: req = json.load(open(a.request))
34
+ else:
35
+ if sys.stdin.isatty(): print("votum decide: reading the request JSON from stdin (pass --request FILE, or pipe it in)", file=sys.stderr)
36
+ raw = sys.stdin.read()
37
+ if not raw.strip(): sys.exit("votum decide: no request on stdin; pass --request FILE or pipe a JSON request")
38
+ try: req = json.loads(raw)
39
+ except json.JSONDecodeError as e: sys.exit(f"votum decide: stdin is not JSON ({e})")
40
+ from .modeling import Votum
41
+ vm = Votum.from_pretrained(a.repo, exact=a.exact, base=a.base, device=a.device)
42
+ print(json.dumps(vm.decide(req), indent=1))
43
+ else:
44
+ sys.exit(f"unknown command {cmd!r}\n{__doc__}")
45
+
46
+
47
+ if __name__ == "__main__":
48
+ main()
votum/config.py ADDED
@@ -0,0 +1,40 @@
1
+ """votum_config.json: everything besides the weights that a Votum checkpoint needs (layout, mask, readout, temperatures)."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ from dataclasses import asdict, dataclass, field
6
+ from pathlib import Path
7
+
8
+ CONFIG_NAME = "votum_config.json"
9
+ FORMAT = "votum/1"
10
+
11
+
12
+ @dataclass
13
+ class VotumConfig:
14
+ base_model: str = "swiss-ai/Apertus-v1.5-8B"
15
+ base_revision: str = "ef4bbc54d6fe658bc782280ff9f3b53a10a9758f"
16
+ opt_end_token: str = "<|fim_end|>"
17
+ opt_end_id: int = 60
18
+ bidirectional_layers: list = field(default_factory=lambda: list(range(3, 32, 4)))
19
+ pointer_rank: int = 256
20
+ shared_positions: bool = True # every option block starts at the same position id
21
+ close_last: bool = True # the prompt's final line break belongs to the last option block
22
+ # header: up to max_q questions of a request listed before the state; choice options listed (all if max_opts is None),
23
+ # sorted by text; chunk: split requests of more than max_q questions into separate headers
24
+ header: dict = field(default_factory=lambda: {"max_q": 8, "max_opts": None, "sort": True, "chunk": False})
25
+ temperatures: dict = field(default_factory=lambda: {"choice": 1.5164094828380006, "noul": 1.3096884485875626,
26
+ "score": 1.1819906765646513})
27
+ max_tokens: int = 8192
28
+ format: str = FORMAT
29
+
30
+ @classmethod
31
+ def load(cls, path: str | Path) -> "VotumConfig":
32
+ p = Path(path); p = p / CONFIG_NAME if p.is_dir() else p
33
+ d = json.loads(p.read_text())
34
+ if d.get("format") != FORMAT: raise ValueError(f"{p}: unsupported format {d.get('format')!r} (expected {FORMAT})")
35
+ known = set(cls.__dataclass_fields__)
36
+ return cls(**{k: v for k, v in d.items() if k in known})
37
+
38
+ def save(self, path: str | Path) -> None:
39
+ p = Path(path); p = p / CONFIG_NAME if p.is_dir() else p
40
+ p.write_text(json.dumps(asdict(self), indent=2) + "\n")
votum/fp8.py ADDED
@@ -0,0 +1,155 @@
1
+ """FP8 weights for Votum: make a quantized checkpoint, and load one in PyTorch.
2
+
3
+ votum quantize --model <org>/Votum-8B --out Votum-8B-FP8 # the published variant; also for your own fine-tune
4
+ votum quantize --model <dir> --out <dir> --activations [--ignore-layers 0,1] [--ignore-proj up_proj,o_proj]
5
+
6
+ The output is a checkpoint in the compressed-tensors format. `votum serve` and `Votum.from_pretrained` both load it and
7
+ need no flag. Weights of the attention and MLP projections become e4m3 with one fp32 scale per output channel
8
+ (scale = max |w| / 448); the activations stay bf16, so memory roughly halves and speed stays about the same.
9
+ --activations also quantizes the activations (faster on recent GPUs, but more answers change, and only vLLM runs it).
10
+ --ignore-layers / --ignore-proj keep layers or projection types in bf16. The pointer head, the norms, the xIELU
11
+ parameters and the embeddings are never quantized. The output keeps the decision_* fields of config.json,
12
+ votum_config.json and the tokenizer; it has no adapter/ folder. The conversion is deterministic and runs on the CPU,
13
+ shard by shard, in about a minute.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import argparse, json, shutil
18
+ from pathlib import Path
19
+
20
+ import torch
21
+ import torch.nn as nn
22
+ import torch.nn.functional as F
23
+
24
+ from .build import TOKENIZER_FILES, resolve
25
+
26
+ FP8_MAX = 448.0
27
+ PROJ = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", "mlp.up_proj", "mlp.down_proj")
28
+ COPY = ["votum_config.json", "LICENSE", "NOTICE", *TOKENIZER_FILES]
29
+
30
+
31
+ def quantize(model: str, out: str | Path, activations: bool = False, ignore_layers: set[int] | None = None,
32
+ ignore_proj: set[str] | None = None, token=None) -> Path:
33
+ from safetensors import safe_open
34
+ from safetensors.torch import save_file
35
+ ign_layers, ign_proj = set(ignore_layers or ()), set(ignore_proj or ())
36
+ src = resolve(model, None, token, ["config.json", "votum_config.json", "model*.safetensors", "model.safetensors.index.json", "LICENSE", "NOTICE", *TOKENIZER_FILES])
37
+ cfg = json.loads((src / "config.json").read_text())
38
+ if cfg.get("quantization_config"): raise SystemExit(f"votum quantize: {model} is already quantized; start from the full-precision checkpoint")
39
+ if (src / "lora" / "adapter_config.json").exists():
40
+ raise SystemExit(f"votum quantize: {model} keeps its adapter unmerged (votum build --exact); quantize a merged checkpoint")
41
+ out = Path(out); out.mkdir(parents=True, exist_ok=True)
42
+ idx = json.loads((src / "model.safetensors.index.json").read_text())["weight_map"]; shards = sorted(set(idx.values()))
43
+
44
+ def quantized(k: str) -> bool:
45
+ if not (k.startswith("model.layers.") and any(k.endswith(p + ".weight") for p in PROJ)): return False
46
+ return int(k.split(".")[2]) not in ign_layers and k.split(".")[-2] not in ign_proj
47
+
48
+ new_map, n_q, n_keep, total = {}, 0, 0, 0
49
+ for si, sh in enumerate(shards):
50
+ tensors = {}
51
+ with safe_open(str(src / sh), "pt") as f:
52
+ for k in f.keys():
53
+ w = f.get_tensor(k)
54
+ if quantized(k):
55
+ wf = w.float(); s = wf.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) / FP8_MAX
56
+ tensors[k] = (wf / s).to(torch.float8_e4m3fn).contiguous(); tensors[k[:-len("weight")] + "weight_scale"] = s.contiguous(); n_q += 1
57
+ else:
58
+ tensors[k] = w.contiguous(); n_keep += 1
59
+ name = f"model-{si + 1:05d}-of-{len(shards):05d}.safetensors"
60
+ save_file(tensors, str(out / name), metadata={"format": "pt"}); total += sum(t.numel() * t.element_size() for t in tensors.values())
61
+ new_map.update({k: name for k in tensors}); print(f"{name}: {len(tensors)} tensors", flush=True)
62
+ json.dump({"metadata": {"total_size": total}, "weight_map": new_map}, open(out / "model.safetensors.index.json", "w"), indent=1)
63
+
64
+ ignore = ["model.embed_tokens", "lm_head", "pointer.o"] + [f"model.layers.{i}.{p}" for i in sorted(ign_layers) for p in PROJ] + [f"re:.*\\.{p}$" for p in sorted(ign_proj)]
65
+ acts = {"num_bits": 8, "type": "float", "symmetric": True, "strategy": "token", "dynamic": True, "observer": None} if activations else None
66
+ cfg["quantization_config"] = {
67
+ "quant_method": "compressed-tensors", "format": "float-quantized", "quantization_status": "compressed", "kv_cache_scheme": None,
68
+ "sparsity_config": {}, "ignore": ignore,
69
+ "config_groups": {"group_0": {"targets": ["Linear"], "weights": {"num_bits": 8, "type": "float", "symmetric": True, "strategy": "channel", "dynamic": False, "observer": "minmax"},
70
+ "input_activations": acts, "output_activations": None}},
71
+ "votum_fp8": {"source": Path(model).name, # the name only: a local path must not end up in a published config
72
+ "weights": "fp8 e4m3, per output channel", "activations": "fp8 per token (dynamic)" if activations else "bf16",
73
+ "ignore_layers": sorted(ign_layers), "ignore_proj": sorted(ign_proj)}}
74
+ (out / "config.json").write_text(json.dumps(cfg, indent=1) + "\n")
75
+ for f in COPY:
76
+ if (src / f).exists(): shutil.copy(src / f, out / f)
77
+ print(f"{n_q} projections quantized, {n_keep} tensors kept in their dtype; {total / 1e9:.1f} GB -> {out}")
78
+ print(f"serve it: votum serve --model {out}")
79
+ return out
80
+
81
+
82
+ def main(argv=None) -> None:
83
+ ap = argparse.ArgumentParser(prog="votum quantize", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
84
+ ap.add_argument("--model", required=True, help="Votum checkpoint: Hub repo id or local directory (merged bf16 weights)")
85
+ ap.add_argument("--out", required=True, help="directory to write the FP8 checkpoint to")
86
+ ap.add_argument("--activations", action="store_true", help="also quantize activations (W8A8, dynamic per token); default weights only (W8A16)")
87
+ ap.add_argument("--ignore-layers", default="", help="comma-separated layer indices kept in bf16")
88
+ ap.add_argument("--ignore-proj", default="", help="comma-separated projections kept in bf16 in every layer, e.g. up_proj,o_proj")
89
+ ap.add_argument("--token", default=None)
90
+ a = ap.parse_args(argv)
91
+ quantize(a.model, a.out, a.activations, {int(x) for x in a.ignore_layers.split(",") if x.strip()},
92
+ {x.strip() for x in a.ignore_proj.split(",") if x.strip()}, a.token)
93
+
94
+
95
+ # ---------------------------------------------------------------------------------------------- PyTorch loading
96
+ class FP8Linear(nn.Module):
97
+ """a linear layer without bias whose weight stays in FP8 (e4m3, one fp32 scale per output channel). The weight is
98
+ expanded for each call and freed again, so the model keeps the memory footprint of the file."""
99
+
100
+ def __init__(self, weight: torch.Tensor, scale: torch.Tensor) -> None:
101
+ super().__init__()
102
+ self.register_buffer("weight", weight); self.register_buffer("weight_scale", scale.float())
103
+ self.out_features, self.in_features = weight.shape
104
+
105
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
106
+ return F.linear(x, (self.weight.float() * self.weight_scale).to(x.dtype))
107
+
108
+ def extra_repr(self) -> str:
109
+ return f"in_features={self.in_features}, out_features={self.out_features}, fp8"
110
+
111
+
112
+ def is_quantized(rd: Path) -> bool:
113
+ return bool(json.loads((Path(rd) / "config.json").read_text()).get("quantization_config"))
114
+
115
+
116
+ def load_backbone(rd: Path, tcfg, device, dtype=torch.bfloat16):
117
+ """the text stack of a weight-only FP8 Votum checkpoint: quantized projections as FP8Linear, the rest in dtype."""
118
+ from accelerate import init_empty_weights
119
+ from safetensors import safe_open
120
+ from transformers import ApertusModel
121
+ qc = json.loads((rd / "config.json").read_text())["quantization_config"]
122
+ groups = list((qc.get("config_groups") or {}).values())
123
+ if qc.get("format") != "float-quantized" or any(g.get("input_activations") or g.get("output_activations") for g in groups):
124
+ raise ValueError(f"{rd}: the PyTorch path loads weight-only FP8 checkpoints (votum quantize without --activations). "
125
+ "Serve this one with `votum serve`.")
126
+ reset = {c: c.reset_parameters for c in (nn.Linear, nn.Embedding)}
127
+ try: # the weights come from the file: skip the random initialisation of 8B values
128
+ for c in reset: c.reset_parameters = lambda self: None
129
+ with init_empty_weights(): bb = ApertusModel(tcfg)
130
+ finally:
131
+ for c, f in reset.items(): c.reset_parameters = f
132
+ wmap = json.loads((rd / "model.safetensors.index.json").read_text())["weight_map"]
133
+ sd, q, scales = {}, {}, {}
134
+ for sh in sorted(set(wmap.values())):
135
+ with safe_open(str(rd / sh), framework="pt", device=str(device)) as f:
136
+ for k in f.keys():
137
+ if not k.startswith("model."): continue # the pointer head is read by the caller
138
+ name, t = k[len("model."):], f.get_tensor(k)
139
+ if name.endswith(".weight_scale"): scales[name[:-len(".weight_scale")]] = t
140
+ elif t.dtype == torch.float8_e4m3fn: q[name[:-len(".weight")]] = t
141
+ else: sd[name] = t.to(dtype) if t.is_floating_point() else t
142
+ if set(q) != set(scales): raise ValueError(f"{rd}: FP8 weights and their scales do not match")
143
+ for name, w in q.items():
144
+ parent, _, leaf = name.rpartition(".")
145
+ setattr(bb.get_submodule(parent), leaf, FP8Linear(w, scales[name]))
146
+ res = bb.load_state_dict(sd, strict=False, assign=True)
147
+ missing = [k for k in res.missing_keys if not k.endswith((".weight_scale",)) and k.rpartition(".")[0] not in q]
148
+ if missing or res.unexpected_keys:
149
+ raise ValueError(f"{rd}: weights do not fit the model (missing {missing[:3]}, unexpected {list(res.unexpected_keys)[:3]})")
150
+ bb.eval(); bb.requires_grad_(False)
151
+ return bb
152
+
153
+
154
+ if __name__ == "__main__":
155
+ main()