dirichlet-regularization 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dreg/quantization.py ADDED
@@ -0,0 +1,419 @@
1
+ """
2
+ Base-3 Quantum Trit Quantization & .tritq Binary Serialization
3
+ =============================================================
4
+ Implements hierarchical 4-band spectral quantization and base-3 packaging:
5
+ - 5 balanced trits {-1, 0, +1} packed into 1 byte (3^5 = 243 <= 256) -> 1.60 bits/trit.
6
+ - Effective bpp depends on matrix geometry (M×N) and quantizer radii (r0, r1, r2):
7
+ · Default (r0=0.10, r1=0.25, r2=0.50): ~0.47 bpp (~68× on 256×1024)
8
+ · Aggressive (r0=0.10, r1=0.30, r2=0.75): ~0.91 bpp (~35× on 256×1024)
9
+ · Calibrated TinyStories-10M / MNIST (r0=0.15, r1=0.40, r2=1.00): ~1.64 bpp (~19.5×)
10
+ · Mid-Band (r0=0.20, r1=0.55, r2=1.15): ~2.18 bpp (~14.7×)
11
+ See paper Table 5 and §3.3 for the full empirical rate–distortion frontier.
12
+ """
13
+
14
+ import json
15
+ import struct
16
+ from typing import Any, BinaryIO, Dict, Optional, Tuple
17
+
18
+ import numpy as np
19
+ import torch
20
+
21
+
22
+ class Base3TritQuantizer:
23
+ """
24
+ Quantizes 2D-DCT spectral coefficients into 4 hierarchical frequency bands.
25
+
26
+ Default radii (r0=0.10, r1=0.25, r2=0.50) yield ~0.47 bpp (~68×).
27
+ The calibrated TinyStories-10M & MNIST setting (r0=0.15, r1=0.40, r2=1.00)
28
+ yields ~1.64 bpp (~19.5×). See paper Table 5 for the full rate–distortion sweep:
29
+ (0.08, 0.22, 0.50) -> 0.43 bpp, (0.10, 0.30, 0.75) -> 0.91 bpp,
30
+ (0.15, 0.40, 1.00) -> 1.64 bpp, (0.20, 0.55, 1.15) -> 2.18 bpp,
31
+ (0.30, 0.70, 1.30) -> 2.79 bpp, (0.45, 0.90, 1.41) -> 3.77 bpp.
32
+
33
+ - Band 0 (rho <= r0): DC / Basal harmonics -> 8-bit uint (8 bpp)
34
+ - Band 1 (r0 < rho <= r1): Mid frequencies -> 4-bit nibbles (4 bpp)
35
+ - Band 2 (r1 < rho <= r2): High-mid frequencies -> Base-3 Trits (1.6 bpp)
36
+ - Band 3 (rho > r2): High frequencies -> Truncated to zero (0 bpp)
37
+ """
38
+ def __init__(
39
+ self,
40
+ r0: float = 0.10,
41
+ r1: float = 0.25,
42
+ r2: float = 0.50,
43
+ band2_mode: str = "lloyd_max",
44
+ ):
45
+ if not (0.0 <= r0 <= r1 <= r2):
46
+ raise ValueError(
47
+ f"Quantizer radii must satisfy 0 <= r0 <= r1 <= r2, got ({r0}, {r1}, {r2})"
48
+ )
49
+ if band2_mode not in {"lloyd_max", "legacy"}:
50
+ raise ValueError(
51
+ f"Unknown band2_mode: {band2_mode}. Expected 'lloyd_max' or 'legacy'."
52
+ )
53
+ self.r0 = float(r0)
54
+ self.r1 = float(r1)
55
+ self.r2 = float(r2)
56
+ self.band2_mode = band2_mode
57
+ self._lut = self._build_trit_lut()
58
+
59
+ @staticmethod
60
+ def _build_trit_lut() -> np.ndarray:
61
+ """
62
+ Builds a 256 x 5 float32 lookup table for instant O(1) byte-to-trit unpacking:
63
+ byte_val = t0 + t1*3 + t2*9 + t3*27 + t4*81, where t in {0, 1, 2} maps to {-1, 0, +1}
64
+ """
65
+ lut = np.zeros((256, 5), dtype=np.float32)
66
+ for b in range(243):
67
+ val = b
68
+ for i in range(5):
69
+ trit = val % 3
70
+ val //= 3
71
+ lut[b, i] = float(trit - 1) # 0 -> -1.0, 1 -> 0.0, 2 -> +1.0
72
+ return lut
73
+
74
+ def compute_radial_grid(self, m: int, n: int, device: torch.device) -> torch.Tensor:
75
+ """
76
+ Computes normalized Euclidean radial frequency coordinates in [0, sqrt(2)).
77
+ """
78
+ u = torch.arange(m, device=device, dtype=torch.float32).unsqueeze(1) / float(m)
79
+ v = torch.arange(n, device=device, dtype=torch.float32).unsqueeze(0) / float(n)
80
+ return torch.sqrt(u.pow(2) + v.pow(2))
81
+
82
+ def quantize_matrix(self, spec: torch.Tensor) -> Dict[str, Any]:
83
+ """
84
+ Quantizes a 2D spectral tensor into compressed packed buffers.
85
+ """
86
+ if spec.dim() != 2:
87
+ raise ValueError(f"Expected 2D spectral tensor, got shape {tuple(spec.shape)}")
88
+ m, n = spec.shape
89
+ rho = self.compute_radial_grid(m, n, spec.device)
90
+
91
+ mask0 = rho <= self.r0
92
+ mask1 = (rho > self.r0) & (rho <= self.r1)
93
+ mask2 = (rho > self.r1) & (rho <= self.r2)
94
+
95
+ # Band 0: 8-bit
96
+ vals0 = spec[mask0]
97
+ if vals0.numel() > 0:
98
+ min0, max0 = vals0.min().item(), vals0.max().item()
99
+ scale0 = (max0 - min0) / 255.0 if max0 > min0 else 1.0
100
+ q0 = torch.clamp(((vals0 - min0) / (scale0 + 1e-12)).round(), 0, 255).to(torch.uint8)
101
+ else:
102
+ min0, scale0 = 0.0, 1.0
103
+ q0 = torch.empty(0, dtype=torch.uint8, device=spec.device)
104
+
105
+ # Band 1: 4-bit nibbles (2 values per byte)
106
+ vals1 = spec[mask1]
107
+ if vals1.numel() > 0:
108
+ min1, max1 = vals1.min().item(), vals1.max().item()
109
+ scale1 = (max1 - min1) / 15.0 if max1 > min1 else 1.0
110
+ q1_raw = torch.clamp(((vals1 - min1) / (scale1 + 1e-12)).round(), 0, 15).to(torch.uint8)
111
+ if len(q1_raw) % 2 != 0:
112
+ q1_raw = torch.cat([q1_raw, torch.zeros(1, dtype=torch.uint8, device=spec.device)])
113
+ q1_packed = (q1_raw[0::2] << 4) | (q1_raw[1::2] & 0x0F)
114
+ else:
115
+ min1, scale1 = 0.0, 1.0
116
+ q1_packed = torch.empty(0, dtype=torch.uint8, device=spec.device)
117
+
118
+ # Band 2: Base-3 trits (5 trits per byte)
119
+ vals2 = spec[mask2]
120
+ if vals2.numel() > 0:
121
+ if self.band2_mode == "lloyd_max":
122
+ # 1D Lloyd-Max optimal symmetric ternary quantization:
123
+ # strictly minimizes Band-2 MSE and guarantees reconstructed energy <= 100%.
124
+ abs_v = vals2.abs()
125
+ tau = 0.7 * abs_v.mean().item()
126
+ scale2 = 0.0
127
+ for _ in range(3):
128
+ active = abs_v > tau
129
+ if active.any():
130
+ scale2 = abs_v[active].mean().item()
131
+ tau = 0.5 * scale2
132
+ else:
133
+ scale2 = 0.0
134
+ break
135
+ threshold = tau
136
+ else:
137
+ scale2 = vals2.std().item() if vals2.numel() > 1 else (vals2.abs().mean().item() or 1.0)
138
+ threshold = 0.5 * scale2
139
+
140
+ # Map {-1, 0, +1} -> {0, 1, 2}
141
+ trits = torch.ones_like(vals2, dtype=torch.int64) # default 0 (code 1)
142
+ if scale2 > 0.0:
143
+ trits[vals2 > threshold] = 2 # +1
144
+ trits[vals2 < -threshold] = 0 # -1
145
+
146
+ # Pack into uint8 bytes
147
+ rem = (5 - (len(trits) % 5)) % 5
148
+ if rem > 0:
149
+ trits = torch.cat([trits, torch.ones(rem, dtype=torch.int64, device=spec.device)])
150
+
151
+ t0 = trits[0::5]
152
+ t1 = trits[1::5] * 3
153
+ t2 = trits[2::5] * 9
154
+ t3 = trits[3::5] * 27
155
+ t4 = trits[4::5] * 81
156
+ q2_packed = (t0 + t1 + t2 + t3 + t4).to(torch.uint8)
157
+ else:
158
+ scale2 = 1.0
159
+ q2_packed = torch.empty(0, dtype=torch.uint8, device=spec.device)
160
+
161
+ total_bytes = q0.numel() + q1_packed.numel() + q2_packed.numel()
162
+ bpp = (total_bytes * 8.0) / max(spec.numel(), 1)
163
+
164
+ return {
165
+ "shape": (m, n),
166
+ "bpp": bpp,
167
+ "band0": {"data": q0.cpu().numpy(), "min": min0, "scale": scale0, "mask": mask0.cpu().numpy()},
168
+ "band1": {"data": q1_packed.cpu().numpy(), "min": min1, "scale": scale1, "mask": mask1.cpu().numpy(), "count": vals1.numel()},
169
+ "band2": {"data": q2_packed.cpu().numpy(), "scale": scale2, "mask": mask2.cpu().numpy(), "count": vals2.numel()},
170
+ }
171
+
172
+ def dequantize_matrix(self, packed: Dict[str, Any], device: torch.device = torch.device("cpu")) -> torch.Tensor:
173
+ """
174
+ Unpacks and reconstructs the full 2D spectral tensor from compressed buffers.
175
+ """
176
+ m, n = packed["shape"]
177
+ rec = torch.zeros((m, n), dtype=torch.float32, device=device)
178
+
179
+ # Band 0: Unpack 8-bit
180
+ b0 = packed["band0"]
181
+ if len(b0["data"]) > 0:
182
+ q0 = torch.from_numpy(b0["data"]).to(device=device, dtype=torch.float32)
183
+ vals0 = q0 * b0["scale"] + b0["min"]
184
+ mask0 = torch.from_numpy(b0["mask"]).to(device=device)
185
+ rec[mask0] = vals0
186
+
187
+ # Band 1: Unpack 4-bit nibbles
188
+ b1 = packed["band1"]
189
+ if b1["count"] > 0:
190
+ q1_packed = torch.from_numpy(b1["data"]).to(device=device)
191
+ high = (q1_packed >> 4) & 0x0F
192
+ low = q1_packed & 0x0F
193
+ q1_unpacked = torch.stack([high, low], dim=1).view(-1)[:b1["count"]].float()
194
+ vals1 = q1_unpacked * b1["scale"] + b1["min"]
195
+ mask1 = torch.from_numpy(b1["mask"]).to(device=device)
196
+ rec[mask1] = vals1
197
+
198
+ # Band 2: Unpack Base-3 trits via LUT
199
+ b2 = packed["band2"]
200
+ if b2["count"] > 0:
201
+ q2_bytes = b2["data"]
202
+ unpacked_trits = self._lut[q2_bytes].reshape(-1)[:b2["count"]]
203
+ vals2 = torch.from_numpy(unpacked_trits).to(device=device) * b2["scale"]
204
+ mask2 = torch.from_numpy(b2["mask"]).to(device=device)
205
+ rec[mask2] = vals2
206
+
207
+ return rec
208
+
209
+
210
+ class TritQFormat:
211
+ """
212
+ Serializes and deserializes entire Transformer checkpoints in the .tritq binary format.
213
+
214
+ By default, embedding and vocabulary projection tensors (matching ``exclude_substrings``)
215
+ are stored uncompressed in FP16 rather than spectrally quantized, preserving token geometry.
216
+ """
217
+ MAGIC = b"TRITQ_V1"
218
+ DEFAULT_EXCLUDE_SUBSTRINGS: Tuple[str, ...] = ("emb", "embed", "lm_head", "wte", "wpe")
219
+ MAX_HEADER_BYTES = 1_000_000
220
+ MAX_TENSORS = 10_000
221
+ MAX_MATRIX_ELEMENTS = 500_000_000
222
+
223
+ @staticmethod
224
+ def _read_exact(f: BinaryIO, nbytes: int) -> bytes:
225
+ data = f.read(nbytes)
226
+ if len(data) != nbytes:
227
+ raise ValueError(f"Truncated .tritq file: expected {nbytes} bytes, got {len(data)}")
228
+ return data
229
+
230
+ @classmethod
231
+ def _should_compress(cls, name: str, tensor: torch.Tensor, exclude_substrings: Tuple[str, ...]) -> bool:
232
+ if tensor.dim() != 2:
233
+ return False
234
+ lname = name.lower()
235
+ return not any(sub in lname for sub in exclude_substrings)
236
+
237
+ @classmethod
238
+ def save(
239
+ cls,
240
+ filepath: str,
241
+ model_weights: Dict[str, torch.Tensor],
242
+ config_dict: Dict[str, Any],
243
+ quantizer: Optional[Base3TritQuantizer] = None,
244
+ exclude_substrings: Optional[Tuple[str, ...]] = None,
245
+ ):
246
+ if quantizer is None:
247
+ quantizer = Base3TritQuantizer()
248
+ excl = cls.DEFAULT_EXCLUDE_SUBSTRINGS if exclude_substrings is None else tuple(s.lower() for s in exclude_substrings)
249
+
250
+ with open(filepath, "wb") as f:
251
+ f.write(cls.MAGIC)
252
+ header_meta = {
253
+ "config": config_dict,
254
+ "r0": float(quantizer.r0),
255
+ "r1": float(quantizer.r1),
256
+ "r2": float(quantizer.r2),
257
+ "version": 1,
258
+ }
259
+ cfg_bytes = json.dumps(header_meta).encode("utf-8")
260
+ f.write(struct.pack("<I", len(cfg_bytes)))
261
+ f.write(cfg_bytes)
262
+
263
+ f.write(struct.pack("<I", len(model_weights)))
264
+ for name, w in model_weights.items():
265
+ name_bytes = name.encode("utf-8")
266
+ f.write(struct.pack("<H", len(name_bytes)))
267
+ f.write(name_bytes)
268
+ if cls._should_compress(name, w, excl):
269
+ from .spectral import dct2d
270
+ spec = dct2d(w.float())
271
+ packed = quantizer.quantize_matrix(spec)
272
+ # Serialized flag: 1 = compressed spectral
273
+ f.write(struct.pack("<B", 1))
274
+ f.write(struct.pack("<II", packed["shape"][0], packed["shape"][1]))
275
+ # Band 0
276
+ b0_data = packed["band0"]["data"].tobytes()
277
+ f.write(struct.pack("<Iff", len(b0_data), packed["band0"]["min"], packed["band0"]["scale"]))
278
+ f.write(b0_data)
279
+ # Band 1
280
+ b1_data = packed["band1"]["data"].tobytes()
281
+ f.write(struct.pack("<IffI", len(b1_data), packed["band1"]["min"], packed["band1"]["scale"], packed["band1"]["count"]))
282
+ f.write(b1_data)
283
+ # Band 2
284
+ b2_data = packed["band2"]["data"].tobytes()
285
+ f.write(struct.pack("<IfI", len(b2_data), packed["band2"]["scale"], packed["band2"]["count"]))
286
+ f.write(b2_data)
287
+ else:
288
+ # Uncompressed parameter (1D norms/biases or excluded 2D embeddings/lm_head)
289
+ raw_fp16 = w.detach().half().cpu().numpy().tobytes()
290
+ f.write(struct.pack("<B", 0))
291
+ f.write(struct.pack("<I", len(w.shape)))
292
+ for s in w.shape:
293
+ f.write(struct.pack("<I", s))
294
+ f.write(struct.pack("<I", len(raw_fp16)))
295
+ f.write(raw_fp16)
296
+
297
+ @classmethod
298
+ def load(
299
+ cls,
300
+ filepath: str,
301
+ device: torch.device = torch.device("cpu"),
302
+ quantizer: Optional[Base3TritQuantizer] = None,
303
+ ) -> Tuple[Dict[str, Any], Dict[str, torch.Tensor]]:
304
+ """
305
+ Loads a .tritq checkpoint and reconstructs all weight tensors.
306
+
307
+ If *quantizer* is None, radii are taken from the file header
308
+ (r0, r1, r2 saved at write time). If a custom quantizer is passed,
309
+ its radii must match the serialized band counts.
310
+
311
+ Returns:
312
+ (config_dict, weights_dict) where config_dict is the model config
313
+ and weights_dict maps tensor names to reconstructed torch.Tensors.
314
+ """
315
+ from .spectral import idct2d
316
+
317
+ with open(filepath, "rb") as f:
318
+ magic = cls._read_exact(f, 8)
319
+ if magic != cls.MAGIC:
320
+ raise ValueError(f"Invalid .tritq file: expected magic {cls.MAGIC!r}, got {magic!r}")
321
+
322
+ (cfg_len,) = struct.unpack("<I", cls._read_exact(f, 4))
323
+ if cfg_len > cls.MAX_HEADER_BYTES:
324
+ raise ValueError(f"Invalid .tritq header: cfg_len {cfg_len} exceeds 1 MB limit")
325
+ cfg_str = cls._read_exact(f, cfg_len).decode("utf-8")
326
+ try:
327
+ header_meta = json.loads(cfg_str)
328
+ except json.JSONDecodeError as e:
329
+ raise ValueError(
330
+ f"Invalid .tritq header: expected JSON metadata, got unparseable content. "
331
+ f"JSON error: {e}"
332
+ ) from e
333
+
334
+ if isinstance(header_meta, dict) and "config" in header_meta:
335
+ config_dict = header_meta["config"]
336
+ file_r0 = float(header_meta.get("r0", 0.10))
337
+ file_r1 = float(header_meta.get("r1", 0.25))
338
+ file_r2 = float(header_meta.get("r2", 0.50))
339
+ else:
340
+ config_dict = header_meta
341
+ file_r0, file_r1, file_r2 = 0.10, 0.25, 0.50
342
+
343
+ if quantizer is None:
344
+ quantizer = Base3TritQuantizer(r0=file_r0, r1=file_r1, r2=file_r2)
345
+
346
+ (num_tensors,) = struct.unpack("<I", cls._read_exact(f, 4))
347
+ if num_tensors > cls.MAX_TENSORS:
348
+ raise ValueError(f"Invalid .tritq header: num_tensors {num_tensors} exceeds 10k limit")
349
+ weights: Dict[str, torch.Tensor] = {}
350
+
351
+ for _ in range(num_tensors):
352
+ (name_len,) = struct.unpack("<H", cls._read_exact(f, 2))
353
+ name = cls._read_exact(f, name_len).decode("utf-8")
354
+ (flag,) = struct.unpack("<B", cls._read_exact(f, 1))
355
+
356
+ if flag == 1:
357
+ (M, N) = struct.unpack("<II", cls._read_exact(f, 8))
358
+ if M == 0 or N == 0 or M * N > cls.MAX_MATRIX_ELEMENTS:
359
+ raise ValueError(f"Invalid compressed tensor shape ({M}, {N}) for '{name}'")
360
+
361
+ (b0_len, b0_min, b0_scale) = struct.unpack("<Iff", cls._read_exact(f, 12))
362
+ if b0_len > M * N:
363
+ raise ValueError(f"Invalid Band 0 byte length {b0_len} for '{name}'")
364
+ b0_data = np.frombuffer(cls._read_exact(f, b0_len), dtype=np.uint8).copy()
365
+
366
+ (b1_len, b1_min, b1_scale, b1_count) = struct.unpack("<IffI", cls._read_exact(f, 16))
367
+ if b1_count > M * N or b1_len != (b1_count + 1) // 2:
368
+ raise ValueError(f"Invalid Band 1 header ({b1_len} bytes, {b1_count} count) for '{name}'")
369
+ b1_data = np.frombuffer(cls._read_exact(f, b1_len), dtype=np.uint8).copy()
370
+
371
+ (b2_len, b2_scale, b2_count) = struct.unpack("<IfI", cls._read_exact(f, 12))
372
+ if b2_count > M * N or b2_len != (b2_count + 4) // 5:
373
+ raise ValueError(f"Invalid Band 2 header ({b2_len} bytes, {b2_count} count) for '{name}'")
374
+ b2_data = np.frombuffer(cls._read_exact(f, b2_len), dtype=np.uint8).copy()
375
+
376
+ rho = quantizer.compute_radial_grid(M, N, torch.device("cpu"))
377
+ mask0 = (rho <= quantizer.r0).numpy()
378
+ mask1 = ((rho > quantizer.r0) & (rho <= quantizer.r1)).numpy()
379
+ mask2 = ((rho > quantizer.r1) & (rho <= quantizer.r2)).numpy()
380
+
381
+ if b0_len != int(mask0.sum()) or b1_count != int(mask1.sum()) or b2_count != int(mask2.sum()):
382
+ raise ValueError(
383
+ f"Quantizer radii ({quantizer.r0}, {quantizer.r1}, {quantizer.r2}) do not match "
384
+ f"serialized band counts for '{name}'"
385
+ )
386
+
387
+ packed = {
388
+ "shape": (M, N),
389
+ "band0": {"data": b0_data, "min": b0_min, "scale": b0_scale, "mask": mask0},
390
+ "band1": {"data": b1_data, "min": b1_min, "scale": b1_scale, "mask": mask1, "count": b1_count},
391
+ "band2": {"data": b2_data, "scale": b2_scale, "mask": mask2, "count": b2_count},
392
+ }
393
+
394
+ rec_spec = quantizer.dequantize_matrix(packed, device=device)
395
+ weights[name] = idct2d(rec_spec)
396
+
397
+ elif flag == 0:
398
+ (ndim,) = struct.unpack("<I", cls._read_exact(f, 4))
399
+ if ndim > 8:
400
+ raise ValueError(f"Invalid tensor ndim {ndim} for '{name}'")
401
+ shape = []
402
+ numel = 1
403
+ for _ in range(ndim):
404
+ (s,) = struct.unpack("<I", cls._read_exact(f, 4))
405
+ shape.append(s)
406
+ numel *= s
407
+ if numel > cls.MAX_MATRIX_ELEMENTS:
408
+ raise ValueError(f"Invalid raw tensor size {numel} for '{name}'")
409
+ (raw_len,) = struct.unpack("<I", cls._read_exact(f, 4))
410
+ if raw_len != numel * 2:
411
+ raise ValueError(f"Invalid FP16 byte length {raw_len} for shape {shape} in '{name}'")
412
+ raw_bytes = cls._read_exact(f, raw_len)
413
+ arr = np.frombuffer(raw_bytes, dtype=np.float16).copy()
414
+ weights[name] = torch.from_numpy(arr).float().reshape(shape).to(device)
415
+ else:
416
+ raise ValueError(f"Unknown serialization flag {flag} for '{name}'")
417
+
418
+ return config_dict, weights
419
+
dreg/spectral.py ADDED
@@ -0,0 +1,164 @@
1
+ """
2
+ Spectral Transform & Block-DCT Tiling Module
3
+ ============================================
4
+ Provides fast orthonormal 2D-DCT transforms, basis cache, and fixed-size block tiling
5
+ to decouple inverse transform arithmetic O(B^3) from model width D.
6
+ """
7
+
8
+ import math
9
+ import threading
10
+ from collections import OrderedDict
11
+ from typing import Tuple
12
+
13
+ import torch
14
+
15
+ _DCT_CACHE: OrderedDict[Tuple[int, torch.device, torch.dtype], torch.Tensor] = OrderedDict()
16
+ _DCT_CACHE_LOCK = threading.Lock()
17
+ _DCT_CACHE_MAXSIZE: int = 32
18
+
19
+
20
+ def dct_matrix_1d(n: int, device: torch.device = torch.device("cpu"), dtype: torch.dtype = torch.float32) -> torch.Tensor:
21
+ """
22
+ Constructs the orthonormal DCT-II basis matrix of size (n, n):
23
+ D_{k, i} = sqrt(2/n) * cos(pi * (2i + 1) * k / (2n)), k > 0
24
+ D_{0, i} = sqrt(1/n)
25
+
26
+ Uses vectorized construction and a thread-safe bounded LRU cache (max 32 entries).
27
+ """
28
+ if n <= 0:
29
+ raise ValueError(f"DCT dimension n must be positive, got {n}")
30
+
31
+ key = (n, device, dtype)
32
+ with _DCT_CACHE_LOCK:
33
+ if key in _DCT_CACHE:
34
+ _DCT_CACHE.move_to_end(key)
35
+ return _DCT_CACHE[key]
36
+
37
+ # Compute basis angles in float64 to avoid float32 phase drift at large n (e.g. n >= 1024)
38
+ k = torch.arange(n, device=device, dtype=torch.float64).unsqueeze(1) # (n, 1)
39
+ i = torch.arange(n, device=device, dtype=torch.float64).unsqueeze(0) # (1, n)
40
+ d = torch.cos(math.pi * (2.0 * i + 1.0) * k / (2.0 * n))
41
+
42
+ # Apply orthonormal scaling: sqrt(1/n) for k=0, sqrt(2/n) for k>0
43
+ d[0, :] *= math.sqrt(1.0 / n)
44
+ d[1:, :] *= math.sqrt(2.0 / n)
45
+ d = d.to(dtype=dtype)
46
+
47
+ with _DCT_CACHE_LOCK:
48
+ if key not in _DCT_CACHE:
49
+ if len(_DCT_CACHE) >= _DCT_CACHE_MAXSIZE:
50
+ _DCT_CACHE.popitem(last=False)
51
+ _DCT_CACHE[key] = d
52
+ return _DCT_CACHE[key]
53
+
54
+
55
+ def dct1d(x: torch.Tensor) -> torch.Tensor:
56
+ """
57
+ Applies the orthonormal 1D Type-II DCT along the last dimension of ``x``.
58
+ """
59
+ n = x.shape[-1]
60
+ d_n = dct_matrix_1d(n, device=x.device, dtype=x.dtype)
61
+ return x @ d_n.t()
62
+
63
+
64
+ def idct1d(s: torch.Tensor) -> torch.Tensor:
65
+ """
66
+ Applies the orthonormal 1D Type-II inverse DCT (DCT-III) along the last dimension of ``s``.
67
+ """
68
+ n = s.shape[-1]
69
+ d_n = dct_matrix_1d(n, device=s.device, dtype=s.dtype)
70
+ return s @ d_n
71
+
72
+
73
+ def dct2d(x: torch.Tensor) -> torch.Tensor:
74
+ """
75
+ Applies the separable orthonormal 2D-DCT-II to a matrix X:
76
+ S = D_out @ X @ D_in^T
77
+ """
78
+ m, n = x.shape
79
+ d_m = dct_matrix_1d(m, device=x.device, dtype=x.dtype)
80
+ d_n = dct_matrix_1d(n, device=x.device, dtype=x.dtype)
81
+ return d_m @ x @ d_n.t()
82
+
83
+
84
+ def idct2d(s: torch.Tensor) -> torch.Tensor:
85
+ """
86
+ Applies the separable orthonormal 2D-IDCT-II (inverse DCT) to a spectral matrix S:
87
+ X = D_out^T @ S @ D_in
88
+ """
89
+ m, n = s.shape
90
+ d_m = dct_matrix_1d(m, device=s.device, dtype=s.dtype)
91
+ d_n = dct_matrix_1d(n, device=s.device, dtype=s.dtype)
92
+ return d_m.t() @ s @ d_n
93
+
94
+
95
+ def spectral_matvec(x: torch.Tensor, spec_weight: torch.Tensor) -> torch.Tensor:
96
+ """
97
+ Computes the linear projection ``y = x @ W^T`` directly in the 2D-DCT spectral
98
+ domain without explicitly reconstructing ``W = D_M^T @ S @ D_N``:
99
+
100
+ y = IDCT_1D( DCT_1D(x) @ S^T )
101
+
102
+ For single-token inference (x of shape (..., N), spec_weight S of shape (M, N)),
103
+ this eliminates O(M*N) weight materialization and O(D^3) 2D-IDCT reconstruction.
104
+ """
105
+ x_spec = dct1d(x)
106
+ y_spec = x_spec @ spec_weight.t()
107
+ return idct1d(y_spec)
108
+
109
+
110
+ class BlockDCTTiler:
111
+ """
112
+ Splits arbitrary (M, N) weight matrices into uniform B x B blocks
113
+ and applies local 2D-DCT transforms. Guarantees that decoding cost
114
+ is strictly O(B^3) per block, solving the Amdahl scaling limit.
115
+ """
116
+ def __init__(self, block_size: int = 64, pad_mode: str = "replicate"):
117
+ if block_size <= 0:
118
+ raise ValueError(f"block_size must be positive, got {block_size}")
119
+ if pad_mode not in {"replicate", "constant"}:
120
+ raise ValueError(f"Unsupported pad_mode: {pad_mode}. Expected 'replicate' or 'constant'.")
121
+ self.b = block_size
122
+ self.pad_mode = pad_mode
123
+
124
+ def tile_and_transform(self, weight: torch.Tensor) -> torch.Tensor:
125
+ """
126
+ Pads weight to multiples of B, splits into (M/B, N/B, B, B) tiles,
127
+ and applies dct2d to each tile. Uses replicate padding by default to
128
+ avoid artificial high-frequency step discontinuities at padded borders.
129
+ """
130
+ m, n = weight.shape
131
+ pad_m = (self.b - (m % self.b)) % self.b
132
+ pad_n = (self.b - (n % self.b)) % self.b
133
+
134
+ if pad_m > 0 or pad_n > 0:
135
+ if self.pad_mode == "replicate":
136
+ padded = torch.nn.functional.pad(
137
+ weight.unsqueeze(0).unsqueeze(0),
138
+ (0, pad_n, 0, pad_m),
139
+ mode="replicate",
140
+ ).squeeze(0).squeeze(0)
141
+ else:
142
+ padded = torch.nn.functional.pad(weight, (0, pad_n, 0, pad_m))
143
+ else:
144
+ padded = weight
145
+
146
+ pm, pn = padded.shape
147
+ tiles = padded.view(pm // self.b, self.b, pn // self.b, self.b).permute(0, 2, 1, 3)
148
+ # Apply 2D DCT to each tile
149
+ d_b = dct_matrix_1d(self.b, device=weight.device, dtype=weight.dtype)
150
+ # Vectorized tile DCT: S = D @ Tile @ D^T
151
+ spec_tiles = torch.einsum("ij, rcjk, lk -> rcil", d_b, tiles, d_b)
152
+ return spec_tiles
153
+
154
+ def untile_and_reconstruct(self, spec_tiles: torch.Tensor, orig_shape: Tuple[int, int]) -> torch.Tensor:
155
+ """
156
+ Applies inverse 2D-DCT to each tile and reassembles the original (M, N) matrix.
157
+ """
158
+ d_b = dct_matrix_1d(self.b, device=spec_tiles.device, dtype=spec_tiles.dtype)
159
+ # Vectorized tile IDCT: Tile = D^T @ Spec @ D
160
+ tiles = torch.einsum("ji, rcjk, kl -> rcil", d_b, spec_tiles, d_b)
161
+ pm_tiles, pn_tiles, b, _ = tiles.shape
162
+ reassembled = tiles.permute(0, 2, 1, 3).contiguous().view(pm_tiles * b, pn_tiles * b)
163
+ m, n = orig_shape
164
+ return reassembled[:m, :n]