thoughtformula 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
thoughtformula/cli.py ADDED
@@ -0,0 +1,357 @@
1
+ """Command line interface and interactive REPL for thoughtformula."""
2
+
3
+ import os
4
+ from pathlib import Path
5
+ import random
6
+ import sys
7
+ import time
8
+
9
+ from .tensor import FlatTensor
10
+ from .tokenizer import BPETokenizer
11
+ from .rag import PurePythonBM25RAG
12
+ from .draft import NGramDraftModel
13
+ from .model import ModernMicroGPT
14
+ from .training import get_cosine_lr, build_training_sequences, continual_learn_lora_turn, train_dpo_turn, train_kto_turn
15
+
16
+
17
+ DEFAULT_KNOWLEDGE = [
18
+ ("what is this model?", "deep learning in pure python."),
19
+ ("how do small models learn?", "small models learn patterns fast."),
20
+ ("what architecture is this?", "attention is all you need."),
21
+ ("how does gqa save memory?", "grouped query attention shares key value heads."),
22
+ ("how does moe work?", "mixture of experts routes tokens to top swiglu experts."),
23
+ ]
24
+
25
+
26
+ def resolve_checkpoint_path(default_name="micro_gpt_checkpoint.json") -> str:
27
+ # 1. Environment variable override
28
+ env_path = os.environ.get("THOUGHTFORMULA_CHECKPOINT")
29
+ if env_path and os.path.exists(env_path):
30
+ return env_path
31
+
32
+ # 2. Current working directory
33
+ if os.path.exists(default_name):
34
+ return default_name
35
+
36
+ # 3. Packaged checkpoint in src/thoughtformula
37
+ pkg_dir = Path(__file__).resolve().parent
38
+ pkg_ckpt = pkg_dir / default_name
39
+ if pkg_ckpt.exists():
40
+ return str(pkg_ckpt)
41
+
42
+ # 4. Check Desktop dir if present from original environment
43
+ desktop_dir = os.path.expanduser("~/Desktop/gemini-code-1794514134973")
44
+ if os.path.isdir(desktop_dir):
45
+ dt_path = os.path.join(desktop_dir, default_name)
46
+ if os.path.exists(dt_path):
47
+ return dt_path
48
+
49
+ return default_name
50
+
51
+
52
+ def benchmark_speculative(model, tokenizer, rag_engine, ngram_drafter, test_queries=None):
53
+ if test_queries is None:
54
+ test_queries = [
55
+ "what is this model?",
56
+ "tell me about gqa memory",
57
+ "how does moe work?",
58
+ ]
59
+
60
+ print("--- Speculative Decoding Benchmark (N-Gram Drafter gamma=4 vs. Standard Decoding) ---")
61
+ t_std_start = time.perf_counter()
62
+ std_passes = 0
63
+ for q in test_queries:
64
+ retrieved = rag_engine.retrieve(q, top_k=1)
65
+ top_doc = retrieved[0][0] if retrieved else {"fact": "", "question": q}
66
+ p_ids = tokenizer.encode(tokenizer.format_rag_prompt(top_doc["fact"], top_doc["question"]))
67
+ out_std = model.generate(
68
+ p_ids, max_new_tokens=24, temperature=0.05, stop_token_id=tokenizer.eos_id, use_kv_cache=True, use_int8=True
69
+ )
70
+ std_passes += len(out_std) - len(p_ids)
71
+ dt_std = (time.perf_counter() - t_std_start) * 1000.0
72
+
73
+ t_spec_start = time.perf_counter()
74
+ spec_passes = 0
75
+ total_acc, total_drf = 0, 0
76
+ for q in test_queries:
77
+ retrieved = rag_engine.retrieve(q, top_k=1)
78
+ top_doc = retrieved[0][0] if retrieved else {"fact": "", "question": q}
79
+ p_ids = tokenizer.encode(tokenizer.format_rag_prompt(top_doc["fact"], top_doc["question"]))
80
+ out_spec, acc, drf, passes = model.generate_speculative(
81
+ p_ids,
82
+ ngram_drafter,
83
+ max_new_tokens=24,
84
+ gamma=4,
85
+ temperature=0.05,
86
+ stop_token_id=tokenizer.eos_id,
87
+ use_int8=True,
88
+ )
89
+ spec_passes += passes
90
+ total_acc += acc
91
+ total_drf += drf
92
+ reply = tokenizer.decode(out_spec[len(p_ids):], skip_special=True).strip()
93
+ print(f"Q: '{q}' -> {reply} [accepted {acc}/{drf} drafted tokens in {passes} forward passes]")
94
+ dt_spec = (time.perf_counter() - t_spec_start) * 1000.0
95
+ acc_rate = (100.0 * total_acc / max(1, total_drf))
96
+ print(
97
+ f"Summary: Standard={dt_std:.1f} ms ({std_passes} passes) | "
98
+ f"Speculative={dt_spec:.1f} ms ({spec_passes} passes) | "
99
+ f"Draft Acceptance={acc_rate:.1f}% | Speedup={dt_std / max(1e-6, dt_spec):.2f}x\n"
100
+ )
101
+
102
+
103
+ def run_repl(model, tokenizer, rag_engine, ngram_drafter, corpus_segments, ckpt_path, block_size=28):
104
+ print("=====================================================================")
105
+ print("Interactive GQA + Sparse MoE + LoRA + Speculative Decoding REPL")
106
+ print("Commands:")
107
+ print(" - Ask any question : BM25 RAG + Speculative Decoding generates reply")
108
+ print(" - Teach (q :: a) : Instant LoRA fine-tuning (freezes MoE base weights!)")
109
+ print(" - /merge : Merge learned LoRA adapters (A @ B) into Int8 base weights")
110
+ print(" - /rag : List all indexed BM25 RAG documents")
111
+ print(" - /stats : Inspect MoE experts, LoRA rank, & Speculative stats")
112
+ print(" - /quit or exit : Exit the REPL")
113
+ print("=====================================================================")
114
+
115
+ top_k, top_p, temperature, gamma = 6, 0.9, 0.05, 4
116
+
117
+ while True:
118
+ try:
119
+ user_in = input("\nYou > ").strip()
120
+ except (EOFError, KeyboardInterrupt):
121
+ print(f"\nExiting REPL. Checkpoint is saved at {ckpt_path}")
122
+ break
123
+
124
+ if not user_in:
125
+ continue
126
+ if user_in.lower() in ("/quit", "exit", "quit"):
127
+ print(f"Goodbye! All LoRA, MoE, and RAG updates are saved in {ckpt_path}")
128
+ break
129
+
130
+ if user_in.startswith("/merge"):
131
+ merged = model.merge_lora_into_base()
132
+ model.save_checkpoint(ckpt_path, tokenizer, rag_engine, corpus_segments)
133
+ print(f"Merged {merged} active LoRA adapter pairs into Int8 base weights and saved '{ckpt_path}'!")
134
+ continue
135
+
136
+ if user_in.startswith("/rag"):
137
+ print(f"--- Pure-Python BM25 Knowledge Base ({len(rag_engine.documents)} docs) ---")
138
+ for idx_d, doc in enumerate(rag_engine.documents, 1):
139
+ print(f" {idx_d}. Q: {doc['question']} --> Fact: {doc['fact']}")
140
+ continue
141
+
142
+ if user_in.startswith("/stats"):
143
+ base_params = sum(t.rows * t.cols for t in model.weights.values())
144
+ lora_params = sum(A.rows * A.cols + B.rows * B.cols for A, B in model.lora.values())
145
+ file_kb = os.path.getsize(ckpt_path) / 1024.0 if os.path.exists(ckpt_path) else 0.0
146
+ print(
147
+ f"Architecture: {model.n_layers} Layers | GQA ({model.n_heads}Q:{model.n_kv_heads}KV) | "
148
+ f"Sparse MoE: {model.n_experts} SwiGLU Experts (Top-{model.moe_top_k} active per token)\n"
149
+ f"Parameters: {base_params} Base (frozen during REPL chat) | {lora_params} LoRA (rank={model.lora_rank})\n"
150
+ f"Checkpoint: '{ckpt_path}' ({file_kb:.1f} KB) | Full Steps: {model.total_steps} | LoRA Steps: {model.lora_steps}"
151
+ )
152
+ continue
153
+
154
+ if user_in.startswith("/test"):
155
+ print("--- Finite-Difference Gradient Verification ---")
156
+ from .gradcheck import run_gradient_checks
157
+ all_ok, _ = run_gradient_checks()
158
+ if all_ok:
159
+ print("All custom autograd operators verified to numerical calculus precision < 1e-5!")
160
+ else:
161
+ print("WARNING: One or more operators showed gradient discrepancy.")
162
+ continue
163
+
164
+ # Case A: Teach via DPO (q :: good || bad), KTO (q :: a ++ or q :: a --), or LoRA Continual Learning (q :: a)
165
+ if "::" in user_in:
166
+ q_part, a_part = user_in.split("::", 1)
167
+ a_strip = a_part.strip()
168
+ if "||" in a_part:
169
+ good_part, bad_part = a_part.split("||", 1)
170
+ t_dpo = time.perf_counter()
171
+ dpo_loss, margin = train_dpo_turn(
172
+ model, tokenizer, q_part.strip(), good_part.strip(), bad_part.strip()
173
+ )
174
+ dt_dpo = (time.perf_counter() - t_dpo) * 1000.0
175
+ model.save_checkpoint(ckpt_path, tokenizer, rag_engine, corpus_segments)
176
+ file_kb = os.path.getsize(ckpt_path) / 1024.0
177
+ print(
178
+ f"[LoRA DPO Adapted in {dt_dpo:.0f} ms | Loss: {dpo_loss:.4f} | Margin: {margin:+.3f} | "
179
+ f"Saved '{ckpt_path}' ({file_kb:.1f} KB)]"
180
+ )
181
+ user_in = q_part.strip()
182
+ elif a_strip.endswith("++") or a_strip.endswith("--"):
183
+ is_desirable = a_strip.endswith("++")
184
+ ans_text = a_strip[:-2].strip()
185
+ t_kto = time.perf_counter()
186
+ kto_loss, reward = train_kto_turn(
187
+ model, tokenizer, q_part.strip(), ans_text, is_desirable=is_desirable
188
+ )
189
+ dt_kto = (time.perf_counter() - t_kto) * 1000.0
190
+ model.save_checkpoint(ckpt_path, tokenizer, rag_engine, corpus_segments)
191
+ file_kb = os.path.getsize(ckpt_path) / 1024.0
192
+ sign_str = "++ Desirable" if is_desirable else "-- Undesirable"
193
+ print(
194
+ f"[LoRA KTO Adapted ({sign_str}) in {dt_kto:.0f} ms | Loss: {kto_loss:.4f} | Reward: {reward:+.3f} | "
195
+ f"Saved '{ckpt_path}' ({file_kb:.1f} KB)]"
196
+ )
197
+ user_in = q_part.strip()
198
+ else:
199
+ t_lora = time.perf_counter()
200
+ loss_after, minted = continual_learn_lora_turn(
201
+ model, tokenizer, rag_engine, ngram_drafter, corpus_segments, q_part, a_part, ckpt_path, block_size
202
+ )
203
+ dt_lora = (time.perf_counter() - t_lora) * 1000.0
204
+ file_kb = os.path.getsize(ckpt_path) / 1024.0
205
+ mint_msg = f"Minted BPE: {', '.join(minted)} | " if minted else ""
206
+ print(
207
+ f"[{mint_msg}LoRA Adapted in {dt_lora:.0f} ms | Loss: {loss_after:.4f} | "
208
+ f"Saved '{ckpt_path}' ({file_kb:.1f} KB)]"
209
+ )
210
+ user_in = q_part.strip()
211
+
212
+ # Case B: Retrieve context via BM25 RAG and answer via Speculative Decoding
213
+ retrieved = rag_engine.retrieve(user_in, top_k=1)
214
+ if retrieved:
215
+ top_doc, bm25_score = retrieved[0]
216
+ grounded_q = top_doc["question"] if bm25_score > 0.0 else user_in
217
+ fact = top_doc["fact"]
218
+ else:
219
+ fact, grounded_q, bm25_score = "", user_in, 0.0
220
+
221
+ prompt_str = tokenizer.format_rag_prompt(fact, grounded_q)
222
+ p_ids = tokenizer.encode(prompt_str)
223
+
224
+ sys.stdout.write("Assistant > ")
225
+ sys.stdout.flush()
226
+ t0 = time.perf_counter()
227
+ reply_ids = []
228
+ for tok_id in model.generate_speculative_stream(
229
+ p_ids,
230
+ ngram_drafter,
231
+ max_new_tokens=24,
232
+ gamma=gamma,
233
+ temperature=temperature,
234
+ top_k=top_k,
235
+ top_p=top_p,
236
+ stop_token_id=tokenizer.eos_id,
237
+ use_int8=True,
238
+ ):
239
+ if tok_id == tokenizer.eos_id:
240
+ break
241
+ tok_str = tokenizer.decode([tok_id], skip_special=True)
242
+ sys.stdout.write(tok_str)
243
+ sys.stdout.flush()
244
+ reply_ids.append(tok_id)
245
+ dt_gen = (time.perf_counter() - t0) * 1000.0
246
+ sys.stdout.write(f"\n [BM25 score={bm25_score:.2f} | Streamed {len(reply_ids)} tokens in {dt_gen:.1f} ms]\n")
247
+ sys.stdout.flush()
248
+
249
+
250
+ def main():
251
+ ckpt_path = resolve_checkpoint_path("micro_gpt_checkpoint.json")
252
+ block_size = 28
253
+
254
+ need_full_train = True
255
+ if os.path.exists(ckpt_path):
256
+ old_size_kb = os.path.getsize(ckpt_path) / 1024.0
257
+ print(f"Found existing checkpoint '{ckpt_path}' ({old_size_kb:.1f} KB)! Loading...")
258
+ model, tokenizer, rag_engine, corpus_segments, migrated_legacy = ModernMicroGPT.load_checkpoint(ckpt_path)
259
+ added_specials = tokenizer.ensure_special_tokens()
260
+ if not migrated_legacy and not added_specials and len(rag_engine.documents) >= len(DEFAULT_KNOWLEDGE):
261
+ need_full_train = False
262
+ base_params = sum(t.rows * t.cols for t in model.weights.values())
263
+ lora_params = sum(A.rows * A.cols + B.rows * B.cols for A, B in model.lora.values())
264
+ print(
265
+ f"Loaded {model.n_layers}-Layer GQA + Sparse MoE ({model.n_experts}E, Top-{model.moe_top_k}) + LoRA GPT!\n"
266
+ f"Vocab: {model.vocab_size} | Base Params: {base_params} | LoRA Params: {lora_params} | "
267
+ f"Size: {old_size_kb:.1f} KB\n"
268
+ )
269
+ else:
270
+ print("Upgrading earlier checkpoint to Sparse MoE (4 Experts, Top-2) + LoRA architecture...")
271
+
272
+ if need_full_train:
273
+ print("Training 2-Layer GQA + Sparse MoE SwiGLU (4 Experts, Top-2) + LoRA GPT...")
274
+ rag_engine = PurePythonBM25RAG()
275
+ tokenizer = BPETokenizer()
276
+
277
+ seed_text = "".join(
278
+ f"<|context|> {a} <|user|> {q} <|assistant|> {a} <|endoftext|> "
279
+ for q, a in DEFAULT_KNOWLEDGE
280
+ ) * 4
281
+ tokenizer.train(seed_text, max_merges=18, min_freq=4)
282
+
283
+ corpus_segments = []
284
+ for q, a in DEFAULT_KNOWLEDGE:
285
+ rag_engine.add_document(q, a)
286
+ corpus_segments.append(tokenizer.format_rag_qa(a, q, a))
287
+
288
+ model = ModernMicroGPT(
289
+ vocab_size=len(tokenizer.vocab),
290
+ n_embd=24,
291
+ n_heads=4,
292
+ n_kv_heads=2,
293
+ n_layers=2,
294
+ n_experts=4,
295
+ moe_top_k=2,
296
+ expert_hidden=24,
297
+ lora_rank=4,
298
+ lora_alpha=8.0,
299
+ )
300
+ model.set_training_mode(lora_only=False)
301
+
302
+ base_params = sum(t.rows * t.cols for t in model.weights.values())
303
+ lora_params = sum(A.rows * A.cols + B.rows * B.cols for A, B in model.lora.values())
304
+ print(
305
+ f"BPE Vocab: {len(tokenizer.vocab)} tokens | Base MoE Params: {base_params} | "
306
+ f"LoRA Adapter Params: {lora_params} ({100.0 * lora_params / base_params:.1f}% of base)"
307
+ )
308
+
309
+ sequences = build_training_sequences(tokenizer, corpus_segments, block_size=block_size)
310
+ epochs = 32
311
+ total_train_steps = epochs * len(sequences)
312
+ local_step = 0
313
+
314
+ for epoch in range(1, epochs + 1):
315
+ random.shuffle(sequences)
316
+ epoch_loss = 0.0
317
+ for x_seq, y_seq in sequences:
318
+ local_step += 1
319
+ lr = get_cosine_lr(local_step, total_train_steps, max_lr=0.012, min_lr=0.0008, warmup_steps=20)
320
+ logits = model.forward(x_seq, use_int8=False)
321
+ loss = FlatTensor.cross_entropy(logits, y_seq)
322
+ loss.backward()
323
+ model.step_adam(lr, lora_only=False)
324
+ epoch_loss += loss.data[0]
325
+
326
+ if epoch % 8 == 0 or epoch == 1:
327
+ print(
328
+ f"Epoch {epoch:2d}/{epochs} | "
329
+ f"Avg Loss: {epoch_loss / len(sequences):.4f} | "
330
+ f"Cosine LR: {lr:.5f}"
331
+ )
332
+
333
+ model.save_checkpoint(ckpt_path, tokenizer, rag_engine, corpus_segments)
334
+ size_kb = os.path.getsize(ckpt_path) / 1024.0
335
+ print(f"\nSaved Int8 GQA + MoE + LoRA checkpoint to '{ckpt_path}' ({size_kb:.1f} KB)\n")
336
+
337
+ # Fit N-Gram Draft Model for Speculative Decoding
338
+ ngram_drafter = NGramDraftModel(max_n=4)
339
+ ngram_drafter.fit([tokenizer.encode(seg) for seg in corpus_segments])
340
+
341
+ # If --benchmark or non-interactive flag passed
342
+ if "--benchmark" in sys.argv:
343
+ benchmark_speculative(model, tokenizer, rag_engine, ngram_drafter)
344
+ return
345
+
346
+ # Benchmark Standard Autoregressive vs. Speculative Decoding (gamma=4)
347
+ benchmark_speculative(model, tokenizer, rag_engine, ngram_drafter)
348
+
349
+ # Interactive REPL
350
+ if sys.stdin.isatty():
351
+ run_repl(model, tokenizer, rag_engine, ngram_drafter, corpus_segments, ckpt_path, block_size=block_size)
352
+ else:
353
+ print("Non-interactive mode detected. Exiting.")
354
+
355
+
356
+ if __name__ == "__main__":
357
+ main()
@@ -0,0 +1,57 @@
1
+ """Pure-Python N-Gram Draft Model for speculative decoding."""
2
+
3
+ class NGramDraftModel:
4
+ """
5
+ Hierarchical 4-gram -> 3-gram -> 2-gram backoff drafter over BPE token IDs.
6
+ Generates gamma candidate tokens in <0.02 ms for single-pass Transformer verification.
7
+ """
8
+ def __init__(self, max_n=4):
9
+ self.max_n = max_n
10
+ self.tables = {n: {} for n in range(1, max_n)} # context_tuple -> {next_id: count}
11
+
12
+ def fit(self, token_sequences):
13
+ self.tables = {n: {} for n in range(1, self.max_n)}
14
+ for seq in token_sequences:
15
+ L = len(seq)
16
+ for n in range(1, self.max_n):
17
+ tbl = self.tables[n]
18
+ for i in range(L - n):
19
+ ctx = tuple(seq[i : i + n])
20
+ nxt = seq[i + n]
21
+ bucket = tbl.get(ctx)
22
+ if bucket is None:
23
+ tbl[ctx] = {nxt: 1}
24
+ else:
25
+ bucket[nxt] = bucket.get(nxt, 0) + 1
26
+
27
+ def draft_with_probs(self, prefix_ids, gamma=4, stop_token_id=None):
28
+ """
29
+ Autoregressively drafts up to `gamma` tokens, returning list of:
30
+ (draft_token_id, q_prob, q_distribution_dict)
31
+ """
32
+ results = []
33
+ cur = list(prefix_ids)
34
+ for _ in range(gamma):
35
+ bucket = None
36
+ for n in range(self.max_n - 1, 0, -1):
37
+ if len(cur) >= n:
38
+ ctx = tuple(cur[-n:])
39
+ b = self.tables[n].get(ctx)
40
+ if b:
41
+ bucket = b
42
+ break
43
+ if not bucket:
44
+ break
45
+ total_count = sum(bucket.values())
46
+ best_tok, best_count = max(bucket.items(), key=lambda kv: kv[1])
47
+ q_prob = best_count / total_count
48
+ q_dist = {tok: cnt / total_count for tok, cnt in bucket.items()}
49
+ results.append((best_tok, q_prob, q_dist))
50
+ cur.append(best_tok)
51
+ if stop_token_id is not None and best_tok == stop_token_id:
52
+ break
53
+ return results
54
+
55
+ def draft(self, prefix_ids, gamma=4, stop_token_id=None):
56
+ """Autoregressively drafts up to `gamma` tokens using longest-suffix N-gram match."""
57
+ return [tok for tok, _, _ in self.draft_with_probs(prefix_ids, gamma=gamma, stop_token_id=stop_token_id)]
@@ -0,0 +1,152 @@
1
+ """Finite-difference gradient verification for all custom autograd operators."""
2
+
3
+ from array import array
4
+ import math
5
+ import random
6
+ import unittest
7
+
8
+ from .tensor import FlatTensor
9
+
10
+
11
+ def check_rmsnorm_gradient(T=2, D=4, eps=1e-5, tol=1e-5):
12
+ """Verifies rmsnorm analytical backward pass against centered finite differences."""
13
+ data = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
14
+ grad_out = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
15
+
16
+ # Analytical gradient
17
+ x = FlatTensor(T, D, array('d', data), requires_grad=True)
18
+ y = x.rmsnorm()
19
+ y.grad = array('d', grad_out)
20
+ y._backward()
21
+ analytical = list(x.grad)
22
+
23
+ # Numerical gradient
24
+ numerical = []
25
+ for idx in range(T * D):
26
+ d_plus = array('d', data)
27
+ d_plus[idx] += eps
28
+ x_plus = FlatTensor(T, D, d_plus, requires_grad=False)
29
+ y_plus = x_plus.rmsnorm()
30
+
31
+ d_minus = array('d', data)
32
+ d_minus[idx] -= eps
33
+ x_minus = FlatTensor(T, D, d_minus, requires_grad=False)
34
+ y_minus = x_minus.rmsnorm()
35
+
36
+ diff_sum = sum((yp - ym) * go for yp, ym, go in zip(y_plus.data, y_minus.data, grad_out))
37
+ numerical.append(diff_sum / (2.0 * eps))
38
+
39
+ max_err = max(abs(a - n) for a, n in zip(analytical, numerical))
40
+ return max_err < tol, max_err
41
+
42
+
43
+ def check_swiglu_gradient(T=2, D=4, eps=1e-5, tol=1e-5):
44
+ """Verifies swiglu_act analytical backward pass against centered finite differences."""
45
+ g_data = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
46
+ u_data = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
47
+ grad_out = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
48
+
49
+ gate = FlatTensor(T, D, array('d', g_data), requires_grad=True)
50
+ up = FlatTensor(T, D, array('d', u_data), requires_grad=True)
51
+ out = FlatTensor.swiglu_act(gate, up)
52
+ out.grad = array('d', grad_out)
53
+ out._backward()
54
+ ana_g, ana_u = list(gate.grad), list(up.grad)
55
+
56
+ num_g, num_u = [], []
57
+ for idx in range(T * D):
58
+ # Gate numerical
59
+ gp, gm = array('d', g_data), array('d', g_data)
60
+ gp[idx] += eps
61
+ gm[idx] -= eps
62
+ op = FlatTensor.swiglu_act(FlatTensor(T, D, gp), FlatTensor(T, D, u_data))
63
+ om = FlatTensor.swiglu_act(FlatTensor(T, D, gm), FlatTensor(T, D, u_data))
64
+ num_g.append(sum((p - m) * go for p, m, go in zip(op.data, om.data, grad_out)) / (2.0 * eps))
65
+
66
+ # Up numerical
67
+ up_p, up_m = array('d', u_data), array('d', u_data)
68
+ up_p[idx] += eps
69
+ up_m[idx] -= eps
70
+ op = FlatTensor.swiglu_act(FlatTensor(T, D, g_data), FlatTensor(T, D, up_p))
71
+ om = FlatTensor.swiglu_act(FlatTensor(T, D, g_data), FlatTensor(T, D, up_m))
72
+ num_u.append(sum((p - m) * go for p, m, go in zip(op.data, om.data, grad_out)) / (2.0 * eps))
73
+
74
+ err_g = max(abs(a - n) for a, n in zip(ana_g, num_g))
75
+ err_u = max(abs(a - n) for a, n in zip(ana_u, num_u))
76
+ max_err = max(err_g, err_u)
77
+ return max_err < tol, max_err
78
+
79
+
80
+ def check_rope_gqa_gradient(T=2, D=4, n_heads=2, n_kv_heads=1, eps=1e-5, tol=1e-5):
81
+ """Verifies rope_gqa analytical backward pass against centered finite differences."""
82
+ kv_d = D // (n_heads // n_kv_heads)
83
+ q_data = array('d', [random.gauss(0.0, 0.5) for _ in range(T * D)])
84
+ k_data = array('d', [random.gauss(0.0, 0.5) for _ in range(T * kv_d)])
85
+ v_data = array('d', [random.gauss(0.0, 0.5) for _ in range(T * kv_d)])
86
+ grad_out = array('d', [random.gauss(0.0, 0.5) for _ in range(T * D)])
87
+
88
+ Q = FlatTensor(T, D, array('d', q_data), requires_grad=True)
89
+ K = FlatTensor(T, kv_d, array('d', k_data), requires_grad=True)
90
+ V = FlatTensor(T, kv_d, array('d', v_data), requires_grad=True)
91
+ out = FlatTensor.rope_gqa(Q, K, V, n_heads, n_kv_heads)
92
+ out.grad = array('d', grad_out)
93
+ out._backward()
94
+ ana_q, ana_k, ana_v = list(Q.grad), list(K.grad), list(V.grad)
95
+
96
+ # Numerical for V
97
+ num_v = []
98
+ for idx in range(T * kv_d):
99
+ vp, vm = array('d', v_data), array('d', v_data)
100
+ vp[idx] += eps
101
+ vm[idx] -= eps
102
+ op = FlatTensor.rope_gqa(FlatTensor(T, D, q_data), FlatTensor(T, kv_d, k_data), FlatTensor(T, kv_d, vp), n_heads, n_kv_heads)
103
+ om = FlatTensor.rope_gqa(FlatTensor(T, D, q_data), FlatTensor(T, kv_d, k_data), FlatTensor(T, kv_d, vm), n_heads, n_kv_heads)
104
+ num_v.append(sum((p - m) * go for p, m, go in zip(op.data, om.data, grad_out)) / (2.0 * eps))
105
+
106
+ err_v = max(abs(a - n) for a, n in zip(ana_v, num_v))
107
+ return err_v < tol, err_v
108
+
109
+
110
+ def check_moe_router_gradient(T=2, N_E=4, D=4, eps=1e-5, tol=1e-5):
111
+ """Verifies moe_router_combine analytical backward pass against centered finite differences."""
112
+ r_data = array('d', [random.gauss(0.0, 0.5) for _ in range(T * N_E)])
113
+ exp_outs = [FlatTensor(T, D, array('d', [random.gauss(0.0, 0.5) for _ in range(T * D)])) for _ in range(N_E)]
114
+ grad_out = array('d', [random.gauss(0.0, 0.5) for _ in range(T * D)])
115
+
116
+ R = FlatTensor(T, N_E, array('d', r_data), requires_grad=True)
117
+ out = FlatTensor.moe_router_combine(R, exp_outs, top_k=2)
118
+ out.grad = array('d', grad_out)
119
+ out._backward()
120
+ ana_r = list(R.grad)
121
+
122
+ num_r = []
123
+ for idx in range(T * N_E):
124
+ rp, rm = array('d', r_data), array('d', r_data)
125
+ rp[idx] += eps
126
+ rm[idx] -= eps
127
+ op = FlatTensor.moe_router_combine(FlatTensor(T, N_E, rp), exp_outs, top_k=2)
128
+ om = FlatTensor.moe_router_combine(FlatTensor(T, N_E, rm), exp_outs, top_k=2)
129
+ num_r.append(sum((p - m) * go for p, m, go in zip(op.data, om.data, grad_out)) / (2.0 * eps))
130
+
131
+ max_err = max(abs(a - n) for a, n in zip(ana_r, num_r))
132
+ return max_err < tol, max_err
133
+
134
+
135
+ def run_gradient_checks():
136
+ """Runs all finite-difference gradient checks, printing verification report."""
137
+ checks = [
138
+ ("RMSNorm", check_rmsnorm_gradient),
139
+ ("SwiGLU Activation", check_swiglu_gradient),
140
+ ("RoPE GQA Attention", check_rope_gqa_gradient),
141
+ ("MoE Router Combine", check_moe_router_gradient),
142
+ ]
143
+ all_passed = True
144
+ results = {}
145
+ for name, fn in checks:
146
+ passed, err = fn()
147
+ results[name] = (passed, err)
148
+ status = "PASSED" if passed else "FAILED"
149
+ print(f" [{status}] {name:20s} (max finite-difference error: {err:.2e})")
150
+ if not passed:
151
+ all_passed = False
152
+ return all_passed, results