dirichlet-regularization 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirichlet_regularization-1.0.0.dist-info/METADATA +292 -0
- dirichlet_regularization-1.0.0.dist-info/RECORD +11 -0
- dirichlet_regularization-1.0.0.dist-info/WHEEL +5 -0
- dirichlet_regularization-1.0.0.dist-info/licenses/LICENSE +21 -0
- dirichlet_regularization-1.0.0.dist-info/top_level.txt +1 -0
- dreg/__init__.py +45 -0
- dreg/baselines.py +93 -0
- dreg/model.py +189 -0
- dreg/quantization.py +419 -0
- dreg/spectral.py +164 -0
- dreg/topology.py +338 -0
dreg/quantization.py
ADDED
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Base-3 Quantum Trit Quantization & .tritq Binary Serialization
|
|
3
|
+
=============================================================
|
|
4
|
+
Implements hierarchical 4-band spectral quantization and base-3 packaging:
|
|
5
|
+
- 5 balanced trits {-1, 0, +1} packed into 1 byte (3^5 = 243 <= 256) -> 1.60 bits/trit.
|
|
6
|
+
- Effective bpp depends on matrix geometry (M×N) and quantizer radii (r0, r1, r2):
|
|
7
|
+
· Default (r0=0.10, r1=0.25, r2=0.50): ~0.47 bpp (~68× on 256×1024)
|
|
8
|
+
· Aggressive (r0=0.10, r1=0.30, r2=0.75): ~0.91 bpp (~35× on 256×1024)
|
|
9
|
+
· Calibrated TinyStories-10M / MNIST (r0=0.15, r1=0.40, r2=1.00): ~1.64 bpp (~19.5×)
|
|
10
|
+
· Mid-Band (r0=0.20, r1=0.55, r2=1.15): ~2.18 bpp (~14.7×)
|
|
11
|
+
See paper Table 5 and §3.3 for the full empirical rate–distortion frontier.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import struct
|
|
16
|
+
from typing import Any, BinaryIO, Dict, Optional, Tuple
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
import torch
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Base3TritQuantizer:
|
|
23
|
+
"""
|
|
24
|
+
Quantizes 2D-DCT spectral coefficients into 4 hierarchical frequency bands.
|
|
25
|
+
|
|
26
|
+
Default radii (r0=0.10, r1=0.25, r2=0.50) yield ~0.47 bpp (~68×).
|
|
27
|
+
The calibrated TinyStories-10M & MNIST setting (r0=0.15, r1=0.40, r2=1.00)
|
|
28
|
+
yields ~1.64 bpp (~19.5×). See paper Table 5 for the full rate–distortion sweep:
|
|
29
|
+
(0.08, 0.22, 0.50) -> 0.43 bpp, (0.10, 0.30, 0.75) -> 0.91 bpp,
|
|
30
|
+
(0.15, 0.40, 1.00) -> 1.64 bpp, (0.20, 0.55, 1.15) -> 2.18 bpp,
|
|
31
|
+
(0.30, 0.70, 1.30) -> 2.79 bpp, (0.45, 0.90, 1.41) -> 3.77 bpp.
|
|
32
|
+
|
|
33
|
+
- Band 0 (rho <= r0): DC / Basal harmonics -> 8-bit uint (8 bpp)
|
|
34
|
+
- Band 1 (r0 < rho <= r1): Mid frequencies -> 4-bit nibbles (4 bpp)
|
|
35
|
+
- Band 2 (r1 < rho <= r2): High-mid frequencies -> Base-3 Trits (1.6 bpp)
|
|
36
|
+
- Band 3 (rho > r2): High frequencies -> Truncated to zero (0 bpp)
|
|
37
|
+
"""
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
r0: float = 0.10,
|
|
41
|
+
r1: float = 0.25,
|
|
42
|
+
r2: float = 0.50,
|
|
43
|
+
band2_mode: str = "lloyd_max",
|
|
44
|
+
):
|
|
45
|
+
if not (0.0 <= r0 <= r1 <= r2):
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f"Quantizer radii must satisfy 0 <= r0 <= r1 <= r2, got ({r0}, {r1}, {r2})"
|
|
48
|
+
)
|
|
49
|
+
if band2_mode not in {"lloyd_max", "legacy"}:
|
|
50
|
+
raise ValueError(
|
|
51
|
+
f"Unknown band2_mode: {band2_mode}. Expected 'lloyd_max' or 'legacy'."
|
|
52
|
+
)
|
|
53
|
+
self.r0 = float(r0)
|
|
54
|
+
self.r1 = float(r1)
|
|
55
|
+
self.r2 = float(r2)
|
|
56
|
+
self.band2_mode = band2_mode
|
|
57
|
+
self._lut = self._build_trit_lut()
|
|
58
|
+
|
|
59
|
+
@staticmethod
|
|
60
|
+
def _build_trit_lut() -> np.ndarray:
|
|
61
|
+
"""
|
|
62
|
+
Builds a 256 x 5 float32 lookup table for instant O(1) byte-to-trit unpacking:
|
|
63
|
+
byte_val = t0 + t1*3 + t2*9 + t3*27 + t4*81, where t in {0, 1, 2} maps to {-1, 0, +1}
|
|
64
|
+
"""
|
|
65
|
+
lut = np.zeros((256, 5), dtype=np.float32)
|
|
66
|
+
for b in range(243):
|
|
67
|
+
val = b
|
|
68
|
+
for i in range(5):
|
|
69
|
+
trit = val % 3
|
|
70
|
+
val //= 3
|
|
71
|
+
lut[b, i] = float(trit - 1) # 0 -> -1.0, 1 -> 0.0, 2 -> +1.0
|
|
72
|
+
return lut
|
|
73
|
+
|
|
74
|
+
def compute_radial_grid(self, m: int, n: int, device: torch.device) -> torch.Tensor:
|
|
75
|
+
"""
|
|
76
|
+
Computes normalized Euclidean radial frequency coordinates in [0, sqrt(2)).
|
|
77
|
+
"""
|
|
78
|
+
u = torch.arange(m, device=device, dtype=torch.float32).unsqueeze(1) / float(m)
|
|
79
|
+
v = torch.arange(n, device=device, dtype=torch.float32).unsqueeze(0) / float(n)
|
|
80
|
+
return torch.sqrt(u.pow(2) + v.pow(2))
|
|
81
|
+
|
|
82
|
+
def quantize_matrix(self, spec: torch.Tensor) -> Dict[str, Any]:
|
|
83
|
+
"""
|
|
84
|
+
Quantizes a 2D spectral tensor into compressed packed buffers.
|
|
85
|
+
"""
|
|
86
|
+
if spec.dim() != 2:
|
|
87
|
+
raise ValueError(f"Expected 2D spectral tensor, got shape {tuple(spec.shape)}")
|
|
88
|
+
m, n = spec.shape
|
|
89
|
+
rho = self.compute_radial_grid(m, n, spec.device)
|
|
90
|
+
|
|
91
|
+
mask0 = rho <= self.r0
|
|
92
|
+
mask1 = (rho > self.r0) & (rho <= self.r1)
|
|
93
|
+
mask2 = (rho > self.r1) & (rho <= self.r2)
|
|
94
|
+
|
|
95
|
+
# Band 0: 8-bit
|
|
96
|
+
vals0 = spec[mask0]
|
|
97
|
+
if vals0.numel() > 0:
|
|
98
|
+
min0, max0 = vals0.min().item(), vals0.max().item()
|
|
99
|
+
scale0 = (max0 - min0) / 255.0 if max0 > min0 else 1.0
|
|
100
|
+
q0 = torch.clamp(((vals0 - min0) / (scale0 + 1e-12)).round(), 0, 255).to(torch.uint8)
|
|
101
|
+
else:
|
|
102
|
+
min0, scale0 = 0.0, 1.0
|
|
103
|
+
q0 = torch.empty(0, dtype=torch.uint8, device=spec.device)
|
|
104
|
+
|
|
105
|
+
# Band 1: 4-bit nibbles (2 values per byte)
|
|
106
|
+
vals1 = spec[mask1]
|
|
107
|
+
if vals1.numel() > 0:
|
|
108
|
+
min1, max1 = vals1.min().item(), vals1.max().item()
|
|
109
|
+
scale1 = (max1 - min1) / 15.0 if max1 > min1 else 1.0
|
|
110
|
+
q1_raw = torch.clamp(((vals1 - min1) / (scale1 + 1e-12)).round(), 0, 15).to(torch.uint8)
|
|
111
|
+
if len(q1_raw) % 2 != 0:
|
|
112
|
+
q1_raw = torch.cat([q1_raw, torch.zeros(1, dtype=torch.uint8, device=spec.device)])
|
|
113
|
+
q1_packed = (q1_raw[0::2] << 4) | (q1_raw[1::2] & 0x0F)
|
|
114
|
+
else:
|
|
115
|
+
min1, scale1 = 0.0, 1.0
|
|
116
|
+
q1_packed = torch.empty(0, dtype=torch.uint8, device=spec.device)
|
|
117
|
+
|
|
118
|
+
# Band 2: Base-3 trits (5 trits per byte)
|
|
119
|
+
vals2 = spec[mask2]
|
|
120
|
+
if vals2.numel() > 0:
|
|
121
|
+
if self.band2_mode == "lloyd_max":
|
|
122
|
+
# 1D Lloyd-Max optimal symmetric ternary quantization:
|
|
123
|
+
# strictly minimizes Band-2 MSE and guarantees reconstructed energy <= 100%.
|
|
124
|
+
abs_v = vals2.abs()
|
|
125
|
+
tau = 0.7 * abs_v.mean().item()
|
|
126
|
+
scale2 = 0.0
|
|
127
|
+
for _ in range(3):
|
|
128
|
+
active = abs_v > tau
|
|
129
|
+
if active.any():
|
|
130
|
+
scale2 = abs_v[active].mean().item()
|
|
131
|
+
tau = 0.5 * scale2
|
|
132
|
+
else:
|
|
133
|
+
scale2 = 0.0
|
|
134
|
+
break
|
|
135
|
+
threshold = tau
|
|
136
|
+
else:
|
|
137
|
+
scale2 = vals2.std().item() if vals2.numel() > 1 else (vals2.abs().mean().item() or 1.0)
|
|
138
|
+
threshold = 0.5 * scale2
|
|
139
|
+
|
|
140
|
+
# Map {-1, 0, +1} -> {0, 1, 2}
|
|
141
|
+
trits = torch.ones_like(vals2, dtype=torch.int64) # default 0 (code 1)
|
|
142
|
+
if scale2 > 0.0:
|
|
143
|
+
trits[vals2 > threshold] = 2 # +1
|
|
144
|
+
trits[vals2 < -threshold] = 0 # -1
|
|
145
|
+
|
|
146
|
+
# Pack into uint8 bytes
|
|
147
|
+
rem = (5 - (len(trits) % 5)) % 5
|
|
148
|
+
if rem > 0:
|
|
149
|
+
trits = torch.cat([trits, torch.ones(rem, dtype=torch.int64, device=spec.device)])
|
|
150
|
+
|
|
151
|
+
t0 = trits[0::5]
|
|
152
|
+
t1 = trits[1::5] * 3
|
|
153
|
+
t2 = trits[2::5] * 9
|
|
154
|
+
t3 = trits[3::5] * 27
|
|
155
|
+
t4 = trits[4::5] * 81
|
|
156
|
+
q2_packed = (t0 + t1 + t2 + t3 + t4).to(torch.uint8)
|
|
157
|
+
else:
|
|
158
|
+
scale2 = 1.0
|
|
159
|
+
q2_packed = torch.empty(0, dtype=torch.uint8, device=spec.device)
|
|
160
|
+
|
|
161
|
+
total_bytes = q0.numel() + q1_packed.numel() + q2_packed.numel()
|
|
162
|
+
bpp = (total_bytes * 8.0) / max(spec.numel(), 1)
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
"shape": (m, n),
|
|
166
|
+
"bpp": bpp,
|
|
167
|
+
"band0": {"data": q0.cpu().numpy(), "min": min0, "scale": scale0, "mask": mask0.cpu().numpy()},
|
|
168
|
+
"band1": {"data": q1_packed.cpu().numpy(), "min": min1, "scale": scale1, "mask": mask1.cpu().numpy(), "count": vals1.numel()},
|
|
169
|
+
"band2": {"data": q2_packed.cpu().numpy(), "scale": scale2, "mask": mask2.cpu().numpy(), "count": vals2.numel()},
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
def dequantize_matrix(self, packed: Dict[str, Any], device: torch.device = torch.device("cpu")) -> torch.Tensor:
|
|
173
|
+
"""
|
|
174
|
+
Unpacks and reconstructs the full 2D spectral tensor from compressed buffers.
|
|
175
|
+
"""
|
|
176
|
+
m, n = packed["shape"]
|
|
177
|
+
rec = torch.zeros((m, n), dtype=torch.float32, device=device)
|
|
178
|
+
|
|
179
|
+
# Band 0: Unpack 8-bit
|
|
180
|
+
b0 = packed["band0"]
|
|
181
|
+
if len(b0["data"]) > 0:
|
|
182
|
+
q0 = torch.from_numpy(b0["data"]).to(device=device, dtype=torch.float32)
|
|
183
|
+
vals0 = q0 * b0["scale"] + b0["min"]
|
|
184
|
+
mask0 = torch.from_numpy(b0["mask"]).to(device=device)
|
|
185
|
+
rec[mask0] = vals0
|
|
186
|
+
|
|
187
|
+
# Band 1: Unpack 4-bit nibbles
|
|
188
|
+
b1 = packed["band1"]
|
|
189
|
+
if b1["count"] > 0:
|
|
190
|
+
q1_packed = torch.from_numpy(b1["data"]).to(device=device)
|
|
191
|
+
high = (q1_packed >> 4) & 0x0F
|
|
192
|
+
low = q1_packed & 0x0F
|
|
193
|
+
q1_unpacked = torch.stack([high, low], dim=1).view(-1)[:b1["count"]].float()
|
|
194
|
+
vals1 = q1_unpacked * b1["scale"] + b1["min"]
|
|
195
|
+
mask1 = torch.from_numpy(b1["mask"]).to(device=device)
|
|
196
|
+
rec[mask1] = vals1
|
|
197
|
+
|
|
198
|
+
# Band 2: Unpack Base-3 trits via LUT
|
|
199
|
+
b2 = packed["band2"]
|
|
200
|
+
if b2["count"] > 0:
|
|
201
|
+
q2_bytes = b2["data"]
|
|
202
|
+
unpacked_trits = self._lut[q2_bytes].reshape(-1)[:b2["count"]]
|
|
203
|
+
vals2 = torch.from_numpy(unpacked_trits).to(device=device) * b2["scale"]
|
|
204
|
+
mask2 = torch.from_numpy(b2["mask"]).to(device=device)
|
|
205
|
+
rec[mask2] = vals2
|
|
206
|
+
|
|
207
|
+
return rec
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
class TritQFormat:
|
|
211
|
+
"""
|
|
212
|
+
Serializes and deserializes entire Transformer checkpoints in the .tritq binary format.
|
|
213
|
+
|
|
214
|
+
By default, embedding and vocabulary projection tensors (matching ``exclude_substrings``)
|
|
215
|
+
are stored uncompressed in FP16 rather than spectrally quantized, preserving token geometry.
|
|
216
|
+
"""
|
|
217
|
+
MAGIC = b"TRITQ_V1"
|
|
218
|
+
DEFAULT_EXCLUDE_SUBSTRINGS: Tuple[str, ...] = ("emb", "embed", "lm_head", "wte", "wpe")
|
|
219
|
+
MAX_HEADER_BYTES = 1_000_000
|
|
220
|
+
MAX_TENSORS = 10_000
|
|
221
|
+
MAX_MATRIX_ELEMENTS = 500_000_000
|
|
222
|
+
|
|
223
|
+
@staticmethod
|
|
224
|
+
def _read_exact(f: BinaryIO, nbytes: int) -> bytes:
|
|
225
|
+
data = f.read(nbytes)
|
|
226
|
+
if len(data) != nbytes:
|
|
227
|
+
raise ValueError(f"Truncated .tritq file: expected {nbytes} bytes, got {len(data)}")
|
|
228
|
+
return data
|
|
229
|
+
|
|
230
|
+
@classmethod
|
|
231
|
+
def _should_compress(cls, name: str, tensor: torch.Tensor, exclude_substrings: Tuple[str, ...]) -> bool:
|
|
232
|
+
if tensor.dim() != 2:
|
|
233
|
+
return False
|
|
234
|
+
lname = name.lower()
|
|
235
|
+
return not any(sub in lname for sub in exclude_substrings)
|
|
236
|
+
|
|
237
|
+
@classmethod
|
|
238
|
+
def save(
|
|
239
|
+
cls,
|
|
240
|
+
filepath: str,
|
|
241
|
+
model_weights: Dict[str, torch.Tensor],
|
|
242
|
+
config_dict: Dict[str, Any],
|
|
243
|
+
quantizer: Optional[Base3TritQuantizer] = None,
|
|
244
|
+
exclude_substrings: Optional[Tuple[str, ...]] = None,
|
|
245
|
+
):
|
|
246
|
+
if quantizer is None:
|
|
247
|
+
quantizer = Base3TritQuantizer()
|
|
248
|
+
excl = cls.DEFAULT_EXCLUDE_SUBSTRINGS if exclude_substrings is None else tuple(s.lower() for s in exclude_substrings)
|
|
249
|
+
|
|
250
|
+
with open(filepath, "wb") as f:
|
|
251
|
+
f.write(cls.MAGIC)
|
|
252
|
+
header_meta = {
|
|
253
|
+
"config": config_dict,
|
|
254
|
+
"r0": float(quantizer.r0),
|
|
255
|
+
"r1": float(quantizer.r1),
|
|
256
|
+
"r2": float(quantizer.r2),
|
|
257
|
+
"version": 1,
|
|
258
|
+
}
|
|
259
|
+
cfg_bytes = json.dumps(header_meta).encode("utf-8")
|
|
260
|
+
f.write(struct.pack("<I", len(cfg_bytes)))
|
|
261
|
+
f.write(cfg_bytes)
|
|
262
|
+
|
|
263
|
+
f.write(struct.pack("<I", len(model_weights)))
|
|
264
|
+
for name, w in model_weights.items():
|
|
265
|
+
name_bytes = name.encode("utf-8")
|
|
266
|
+
f.write(struct.pack("<H", len(name_bytes)))
|
|
267
|
+
f.write(name_bytes)
|
|
268
|
+
if cls._should_compress(name, w, excl):
|
|
269
|
+
from .spectral import dct2d
|
|
270
|
+
spec = dct2d(w.float())
|
|
271
|
+
packed = quantizer.quantize_matrix(spec)
|
|
272
|
+
# Serialized flag: 1 = compressed spectral
|
|
273
|
+
f.write(struct.pack("<B", 1))
|
|
274
|
+
f.write(struct.pack("<II", packed["shape"][0], packed["shape"][1]))
|
|
275
|
+
# Band 0
|
|
276
|
+
b0_data = packed["band0"]["data"].tobytes()
|
|
277
|
+
f.write(struct.pack("<Iff", len(b0_data), packed["band0"]["min"], packed["band0"]["scale"]))
|
|
278
|
+
f.write(b0_data)
|
|
279
|
+
# Band 1
|
|
280
|
+
b1_data = packed["band1"]["data"].tobytes()
|
|
281
|
+
f.write(struct.pack("<IffI", len(b1_data), packed["band1"]["min"], packed["band1"]["scale"], packed["band1"]["count"]))
|
|
282
|
+
f.write(b1_data)
|
|
283
|
+
# Band 2
|
|
284
|
+
b2_data = packed["band2"]["data"].tobytes()
|
|
285
|
+
f.write(struct.pack("<IfI", len(b2_data), packed["band2"]["scale"], packed["band2"]["count"]))
|
|
286
|
+
f.write(b2_data)
|
|
287
|
+
else:
|
|
288
|
+
# Uncompressed parameter (1D norms/biases or excluded 2D embeddings/lm_head)
|
|
289
|
+
raw_fp16 = w.detach().half().cpu().numpy().tobytes()
|
|
290
|
+
f.write(struct.pack("<B", 0))
|
|
291
|
+
f.write(struct.pack("<I", len(w.shape)))
|
|
292
|
+
for s in w.shape:
|
|
293
|
+
f.write(struct.pack("<I", s))
|
|
294
|
+
f.write(struct.pack("<I", len(raw_fp16)))
|
|
295
|
+
f.write(raw_fp16)
|
|
296
|
+
|
|
297
|
+
@classmethod
|
|
298
|
+
def load(
|
|
299
|
+
cls,
|
|
300
|
+
filepath: str,
|
|
301
|
+
device: torch.device = torch.device("cpu"),
|
|
302
|
+
quantizer: Optional[Base3TritQuantizer] = None,
|
|
303
|
+
) -> Tuple[Dict[str, Any], Dict[str, torch.Tensor]]:
|
|
304
|
+
"""
|
|
305
|
+
Loads a .tritq checkpoint and reconstructs all weight tensors.
|
|
306
|
+
|
|
307
|
+
If *quantizer* is None, radii are taken from the file header
|
|
308
|
+
(r0, r1, r2 saved at write time). If a custom quantizer is passed,
|
|
309
|
+
its radii must match the serialized band counts.
|
|
310
|
+
|
|
311
|
+
Returns:
|
|
312
|
+
(config_dict, weights_dict) where config_dict is the model config
|
|
313
|
+
and weights_dict maps tensor names to reconstructed torch.Tensors.
|
|
314
|
+
"""
|
|
315
|
+
from .spectral import idct2d
|
|
316
|
+
|
|
317
|
+
with open(filepath, "rb") as f:
|
|
318
|
+
magic = cls._read_exact(f, 8)
|
|
319
|
+
if magic != cls.MAGIC:
|
|
320
|
+
raise ValueError(f"Invalid .tritq file: expected magic {cls.MAGIC!r}, got {magic!r}")
|
|
321
|
+
|
|
322
|
+
(cfg_len,) = struct.unpack("<I", cls._read_exact(f, 4))
|
|
323
|
+
if cfg_len > cls.MAX_HEADER_BYTES:
|
|
324
|
+
raise ValueError(f"Invalid .tritq header: cfg_len {cfg_len} exceeds 1 MB limit")
|
|
325
|
+
cfg_str = cls._read_exact(f, cfg_len).decode("utf-8")
|
|
326
|
+
try:
|
|
327
|
+
header_meta = json.loads(cfg_str)
|
|
328
|
+
except json.JSONDecodeError as e:
|
|
329
|
+
raise ValueError(
|
|
330
|
+
f"Invalid .tritq header: expected JSON metadata, got unparseable content. "
|
|
331
|
+
f"JSON error: {e}"
|
|
332
|
+
) from e
|
|
333
|
+
|
|
334
|
+
if isinstance(header_meta, dict) and "config" in header_meta:
|
|
335
|
+
config_dict = header_meta["config"]
|
|
336
|
+
file_r0 = float(header_meta.get("r0", 0.10))
|
|
337
|
+
file_r1 = float(header_meta.get("r1", 0.25))
|
|
338
|
+
file_r2 = float(header_meta.get("r2", 0.50))
|
|
339
|
+
else:
|
|
340
|
+
config_dict = header_meta
|
|
341
|
+
file_r0, file_r1, file_r2 = 0.10, 0.25, 0.50
|
|
342
|
+
|
|
343
|
+
if quantizer is None:
|
|
344
|
+
quantizer = Base3TritQuantizer(r0=file_r0, r1=file_r1, r2=file_r2)
|
|
345
|
+
|
|
346
|
+
(num_tensors,) = struct.unpack("<I", cls._read_exact(f, 4))
|
|
347
|
+
if num_tensors > cls.MAX_TENSORS:
|
|
348
|
+
raise ValueError(f"Invalid .tritq header: num_tensors {num_tensors} exceeds 10k limit")
|
|
349
|
+
weights: Dict[str, torch.Tensor] = {}
|
|
350
|
+
|
|
351
|
+
for _ in range(num_tensors):
|
|
352
|
+
(name_len,) = struct.unpack("<H", cls._read_exact(f, 2))
|
|
353
|
+
name = cls._read_exact(f, name_len).decode("utf-8")
|
|
354
|
+
(flag,) = struct.unpack("<B", cls._read_exact(f, 1))
|
|
355
|
+
|
|
356
|
+
if flag == 1:
|
|
357
|
+
(M, N) = struct.unpack("<II", cls._read_exact(f, 8))
|
|
358
|
+
if M == 0 or N == 0 or M * N > cls.MAX_MATRIX_ELEMENTS:
|
|
359
|
+
raise ValueError(f"Invalid compressed tensor shape ({M}, {N}) for '{name}'")
|
|
360
|
+
|
|
361
|
+
(b0_len, b0_min, b0_scale) = struct.unpack("<Iff", cls._read_exact(f, 12))
|
|
362
|
+
if b0_len > M * N:
|
|
363
|
+
raise ValueError(f"Invalid Band 0 byte length {b0_len} for '{name}'")
|
|
364
|
+
b0_data = np.frombuffer(cls._read_exact(f, b0_len), dtype=np.uint8).copy()
|
|
365
|
+
|
|
366
|
+
(b1_len, b1_min, b1_scale, b1_count) = struct.unpack("<IffI", cls._read_exact(f, 16))
|
|
367
|
+
if b1_count > M * N or b1_len != (b1_count + 1) // 2:
|
|
368
|
+
raise ValueError(f"Invalid Band 1 header ({b1_len} bytes, {b1_count} count) for '{name}'")
|
|
369
|
+
b1_data = np.frombuffer(cls._read_exact(f, b1_len), dtype=np.uint8).copy()
|
|
370
|
+
|
|
371
|
+
(b2_len, b2_scale, b2_count) = struct.unpack("<IfI", cls._read_exact(f, 12))
|
|
372
|
+
if b2_count > M * N or b2_len != (b2_count + 4) // 5:
|
|
373
|
+
raise ValueError(f"Invalid Band 2 header ({b2_len} bytes, {b2_count} count) for '{name}'")
|
|
374
|
+
b2_data = np.frombuffer(cls._read_exact(f, b2_len), dtype=np.uint8).copy()
|
|
375
|
+
|
|
376
|
+
rho = quantizer.compute_radial_grid(M, N, torch.device("cpu"))
|
|
377
|
+
mask0 = (rho <= quantizer.r0).numpy()
|
|
378
|
+
mask1 = ((rho > quantizer.r0) & (rho <= quantizer.r1)).numpy()
|
|
379
|
+
mask2 = ((rho > quantizer.r1) & (rho <= quantizer.r2)).numpy()
|
|
380
|
+
|
|
381
|
+
if b0_len != int(mask0.sum()) or b1_count != int(mask1.sum()) or b2_count != int(mask2.sum()):
|
|
382
|
+
raise ValueError(
|
|
383
|
+
f"Quantizer radii ({quantizer.r0}, {quantizer.r1}, {quantizer.r2}) do not match "
|
|
384
|
+
f"serialized band counts for '{name}'"
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
packed = {
|
|
388
|
+
"shape": (M, N),
|
|
389
|
+
"band0": {"data": b0_data, "min": b0_min, "scale": b0_scale, "mask": mask0},
|
|
390
|
+
"band1": {"data": b1_data, "min": b1_min, "scale": b1_scale, "mask": mask1, "count": b1_count},
|
|
391
|
+
"band2": {"data": b2_data, "scale": b2_scale, "mask": mask2, "count": b2_count},
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
rec_spec = quantizer.dequantize_matrix(packed, device=device)
|
|
395
|
+
weights[name] = idct2d(rec_spec)
|
|
396
|
+
|
|
397
|
+
elif flag == 0:
|
|
398
|
+
(ndim,) = struct.unpack("<I", cls._read_exact(f, 4))
|
|
399
|
+
if ndim > 8:
|
|
400
|
+
raise ValueError(f"Invalid tensor ndim {ndim} for '{name}'")
|
|
401
|
+
shape = []
|
|
402
|
+
numel = 1
|
|
403
|
+
for _ in range(ndim):
|
|
404
|
+
(s,) = struct.unpack("<I", cls._read_exact(f, 4))
|
|
405
|
+
shape.append(s)
|
|
406
|
+
numel *= s
|
|
407
|
+
if numel > cls.MAX_MATRIX_ELEMENTS:
|
|
408
|
+
raise ValueError(f"Invalid raw tensor size {numel} for '{name}'")
|
|
409
|
+
(raw_len,) = struct.unpack("<I", cls._read_exact(f, 4))
|
|
410
|
+
if raw_len != numel * 2:
|
|
411
|
+
raise ValueError(f"Invalid FP16 byte length {raw_len} for shape {shape} in '{name}'")
|
|
412
|
+
raw_bytes = cls._read_exact(f, raw_len)
|
|
413
|
+
arr = np.frombuffer(raw_bytes, dtype=np.float16).copy()
|
|
414
|
+
weights[name] = torch.from_numpy(arr).float().reshape(shape).to(device)
|
|
415
|
+
else:
|
|
416
|
+
raise ValueError(f"Unknown serialization flag {flag} for '{name}'")
|
|
417
|
+
|
|
418
|
+
return config_dict, weights
|
|
419
|
+
|
dreg/spectral.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Spectral Transform & Block-DCT Tiling Module
|
|
3
|
+
============================================
|
|
4
|
+
Provides fast orthonormal 2D-DCT transforms, basis cache, and fixed-size block tiling
|
|
5
|
+
to decouple inverse transform arithmetic O(B^3) from model width D.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import math
|
|
9
|
+
import threading
|
|
10
|
+
from collections import OrderedDict
|
|
11
|
+
from typing import Tuple
|
|
12
|
+
|
|
13
|
+
import torch
|
|
14
|
+
|
|
15
|
+
_DCT_CACHE: OrderedDict[Tuple[int, torch.device, torch.dtype], torch.Tensor] = OrderedDict()
|
|
16
|
+
_DCT_CACHE_LOCK = threading.Lock()
|
|
17
|
+
_DCT_CACHE_MAXSIZE: int = 32
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def dct_matrix_1d(n: int, device: torch.device = torch.device("cpu"), dtype: torch.dtype = torch.float32) -> torch.Tensor:
|
|
21
|
+
"""
|
|
22
|
+
Constructs the orthonormal DCT-II basis matrix of size (n, n):
|
|
23
|
+
D_{k, i} = sqrt(2/n) * cos(pi * (2i + 1) * k / (2n)), k > 0
|
|
24
|
+
D_{0, i} = sqrt(1/n)
|
|
25
|
+
|
|
26
|
+
Uses vectorized construction and a thread-safe bounded LRU cache (max 32 entries).
|
|
27
|
+
"""
|
|
28
|
+
if n <= 0:
|
|
29
|
+
raise ValueError(f"DCT dimension n must be positive, got {n}")
|
|
30
|
+
|
|
31
|
+
key = (n, device, dtype)
|
|
32
|
+
with _DCT_CACHE_LOCK:
|
|
33
|
+
if key in _DCT_CACHE:
|
|
34
|
+
_DCT_CACHE.move_to_end(key)
|
|
35
|
+
return _DCT_CACHE[key]
|
|
36
|
+
|
|
37
|
+
# Compute basis angles in float64 to avoid float32 phase drift at large n (e.g. n >= 1024)
|
|
38
|
+
k = torch.arange(n, device=device, dtype=torch.float64).unsqueeze(1) # (n, 1)
|
|
39
|
+
i = torch.arange(n, device=device, dtype=torch.float64).unsqueeze(0) # (1, n)
|
|
40
|
+
d = torch.cos(math.pi * (2.0 * i + 1.0) * k / (2.0 * n))
|
|
41
|
+
|
|
42
|
+
# Apply orthonormal scaling: sqrt(1/n) for k=0, sqrt(2/n) for k>0
|
|
43
|
+
d[0, :] *= math.sqrt(1.0 / n)
|
|
44
|
+
d[1:, :] *= math.sqrt(2.0 / n)
|
|
45
|
+
d = d.to(dtype=dtype)
|
|
46
|
+
|
|
47
|
+
with _DCT_CACHE_LOCK:
|
|
48
|
+
if key not in _DCT_CACHE:
|
|
49
|
+
if len(_DCT_CACHE) >= _DCT_CACHE_MAXSIZE:
|
|
50
|
+
_DCT_CACHE.popitem(last=False)
|
|
51
|
+
_DCT_CACHE[key] = d
|
|
52
|
+
return _DCT_CACHE[key]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def dct1d(x: torch.Tensor) -> torch.Tensor:
|
|
56
|
+
"""
|
|
57
|
+
Applies the orthonormal 1D Type-II DCT along the last dimension of ``x``.
|
|
58
|
+
"""
|
|
59
|
+
n = x.shape[-1]
|
|
60
|
+
d_n = dct_matrix_1d(n, device=x.device, dtype=x.dtype)
|
|
61
|
+
return x @ d_n.t()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def idct1d(s: torch.Tensor) -> torch.Tensor:
|
|
65
|
+
"""
|
|
66
|
+
Applies the orthonormal 1D Type-II inverse DCT (DCT-III) along the last dimension of ``s``.
|
|
67
|
+
"""
|
|
68
|
+
n = s.shape[-1]
|
|
69
|
+
d_n = dct_matrix_1d(n, device=s.device, dtype=s.dtype)
|
|
70
|
+
return s @ d_n
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def dct2d(x: torch.Tensor) -> torch.Tensor:
|
|
74
|
+
"""
|
|
75
|
+
Applies the separable orthonormal 2D-DCT-II to a matrix X:
|
|
76
|
+
S = D_out @ X @ D_in^T
|
|
77
|
+
"""
|
|
78
|
+
m, n = x.shape
|
|
79
|
+
d_m = dct_matrix_1d(m, device=x.device, dtype=x.dtype)
|
|
80
|
+
d_n = dct_matrix_1d(n, device=x.device, dtype=x.dtype)
|
|
81
|
+
return d_m @ x @ d_n.t()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def idct2d(s: torch.Tensor) -> torch.Tensor:
|
|
85
|
+
"""
|
|
86
|
+
Applies the separable orthonormal 2D-IDCT-II (inverse DCT) to a spectral matrix S:
|
|
87
|
+
X = D_out^T @ S @ D_in
|
|
88
|
+
"""
|
|
89
|
+
m, n = s.shape
|
|
90
|
+
d_m = dct_matrix_1d(m, device=s.device, dtype=s.dtype)
|
|
91
|
+
d_n = dct_matrix_1d(n, device=s.device, dtype=s.dtype)
|
|
92
|
+
return d_m.t() @ s @ d_n
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def spectral_matvec(x: torch.Tensor, spec_weight: torch.Tensor) -> torch.Tensor:
|
|
96
|
+
"""
|
|
97
|
+
Computes the linear projection ``y = x @ W^T`` directly in the 2D-DCT spectral
|
|
98
|
+
domain without explicitly reconstructing ``W = D_M^T @ S @ D_N``:
|
|
99
|
+
|
|
100
|
+
y = IDCT_1D( DCT_1D(x) @ S^T )
|
|
101
|
+
|
|
102
|
+
For single-token inference (x of shape (..., N), spec_weight S of shape (M, N)),
|
|
103
|
+
this eliminates O(M*N) weight materialization and O(D^3) 2D-IDCT reconstruction.
|
|
104
|
+
"""
|
|
105
|
+
x_spec = dct1d(x)
|
|
106
|
+
y_spec = x_spec @ spec_weight.t()
|
|
107
|
+
return idct1d(y_spec)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class BlockDCTTiler:
|
|
111
|
+
"""
|
|
112
|
+
Splits arbitrary (M, N) weight matrices into uniform B x B blocks
|
|
113
|
+
and applies local 2D-DCT transforms. Guarantees that decoding cost
|
|
114
|
+
is strictly O(B^3) per block, solving the Amdahl scaling limit.
|
|
115
|
+
"""
|
|
116
|
+
def __init__(self, block_size: int = 64, pad_mode: str = "replicate"):
|
|
117
|
+
if block_size <= 0:
|
|
118
|
+
raise ValueError(f"block_size must be positive, got {block_size}")
|
|
119
|
+
if pad_mode not in {"replicate", "constant"}:
|
|
120
|
+
raise ValueError(f"Unsupported pad_mode: {pad_mode}. Expected 'replicate' or 'constant'.")
|
|
121
|
+
self.b = block_size
|
|
122
|
+
self.pad_mode = pad_mode
|
|
123
|
+
|
|
124
|
+
def tile_and_transform(self, weight: torch.Tensor) -> torch.Tensor:
|
|
125
|
+
"""
|
|
126
|
+
Pads weight to multiples of B, splits into (M/B, N/B, B, B) tiles,
|
|
127
|
+
and applies dct2d to each tile. Uses replicate padding by default to
|
|
128
|
+
avoid artificial high-frequency step discontinuities at padded borders.
|
|
129
|
+
"""
|
|
130
|
+
m, n = weight.shape
|
|
131
|
+
pad_m = (self.b - (m % self.b)) % self.b
|
|
132
|
+
pad_n = (self.b - (n % self.b)) % self.b
|
|
133
|
+
|
|
134
|
+
if pad_m > 0 or pad_n > 0:
|
|
135
|
+
if self.pad_mode == "replicate":
|
|
136
|
+
padded = torch.nn.functional.pad(
|
|
137
|
+
weight.unsqueeze(0).unsqueeze(0),
|
|
138
|
+
(0, pad_n, 0, pad_m),
|
|
139
|
+
mode="replicate",
|
|
140
|
+
).squeeze(0).squeeze(0)
|
|
141
|
+
else:
|
|
142
|
+
padded = torch.nn.functional.pad(weight, (0, pad_n, 0, pad_m))
|
|
143
|
+
else:
|
|
144
|
+
padded = weight
|
|
145
|
+
|
|
146
|
+
pm, pn = padded.shape
|
|
147
|
+
tiles = padded.view(pm // self.b, self.b, pn // self.b, self.b).permute(0, 2, 1, 3)
|
|
148
|
+
# Apply 2D DCT to each tile
|
|
149
|
+
d_b = dct_matrix_1d(self.b, device=weight.device, dtype=weight.dtype)
|
|
150
|
+
# Vectorized tile DCT: S = D @ Tile @ D^T
|
|
151
|
+
spec_tiles = torch.einsum("ij, rcjk, lk -> rcil", d_b, tiles, d_b)
|
|
152
|
+
return spec_tiles
|
|
153
|
+
|
|
154
|
+
def untile_and_reconstruct(self, spec_tiles: torch.Tensor, orig_shape: Tuple[int, int]) -> torch.Tensor:
|
|
155
|
+
"""
|
|
156
|
+
Applies inverse 2D-DCT to each tile and reassembles the original (M, N) matrix.
|
|
157
|
+
"""
|
|
158
|
+
d_b = dct_matrix_1d(self.b, device=spec_tiles.device, dtype=spec_tiles.dtype)
|
|
159
|
+
# Vectorized tile IDCT: Tile = D^T @ Spec @ D
|
|
160
|
+
tiles = torch.einsum("ji, rcjk, kl -> rcil", d_b, spec_tiles, d_b)
|
|
161
|
+
pm_tiles, pn_tiles, b, _ = tiles.shape
|
|
162
|
+
reassembled = tiles.permute(0, 2, 1, 3).contiguous().view(pm_tiles * b, pn_tiles * b)
|
|
163
|
+
m, n = orig_shape
|
|
164
|
+
return reassembled[:m, :n]
|