thoughtformula 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- thoughtformula/__init__.py +39 -0
- thoughtformula/batching.py +220 -0
- thoughtformula/cache.py +318 -0
- thoughtformula/cli.py +357 -0
- thoughtformula/draft.py +57 -0
- thoughtformula/gradcheck.py +152 -0
- thoughtformula/medusa.py +138 -0
- thoughtformula/micro_gpt_checkpoint.json +1 -0
- thoughtformula/micro_gpt_v8.py +12 -0
- thoughtformula/model.py +1198 -0
- thoughtformula/rag.py +321 -0
- thoughtformula/tensor.py +1192 -0
- thoughtformula/tokenizer.py +171 -0
- thoughtformula/training.py +246 -0
- thoughtformula-0.3.0.dist-info/METADATA +152 -0
- thoughtformula-0.3.0.dist-info/RECORD +18 -0
- thoughtformula-0.3.0.dist-info/WHEEL +4 -0
- thoughtformula-0.3.0.dist-info/entry_points.txt +3 -0
thoughtformula/cli.py
ADDED
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
"""Command line interface and interactive REPL for thoughtformula."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import random
|
|
6
|
+
import sys
|
|
7
|
+
import time
|
|
8
|
+
|
|
9
|
+
from .tensor import FlatTensor
|
|
10
|
+
from .tokenizer import BPETokenizer
|
|
11
|
+
from .rag import PurePythonBM25RAG
|
|
12
|
+
from .draft import NGramDraftModel
|
|
13
|
+
from .model import ModernMicroGPT
|
|
14
|
+
from .training import get_cosine_lr, build_training_sequences, continual_learn_lora_turn, train_dpo_turn, train_kto_turn
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
DEFAULT_KNOWLEDGE = [
|
|
18
|
+
("what is this model?", "deep learning in pure python."),
|
|
19
|
+
("how do small models learn?", "small models learn patterns fast."),
|
|
20
|
+
("what architecture is this?", "attention is all you need."),
|
|
21
|
+
("how does gqa save memory?", "grouped query attention shares key value heads."),
|
|
22
|
+
("how does moe work?", "mixture of experts routes tokens to top swiglu experts."),
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def resolve_checkpoint_path(default_name="micro_gpt_checkpoint.json") -> str:
|
|
27
|
+
# 1. Environment variable override
|
|
28
|
+
env_path = os.environ.get("THOUGHTFORMULA_CHECKPOINT")
|
|
29
|
+
if env_path and os.path.exists(env_path):
|
|
30
|
+
return env_path
|
|
31
|
+
|
|
32
|
+
# 2. Current working directory
|
|
33
|
+
if os.path.exists(default_name):
|
|
34
|
+
return default_name
|
|
35
|
+
|
|
36
|
+
# 3. Packaged checkpoint in src/thoughtformula
|
|
37
|
+
pkg_dir = Path(__file__).resolve().parent
|
|
38
|
+
pkg_ckpt = pkg_dir / default_name
|
|
39
|
+
if pkg_ckpt.exists():
|
|
40
|
+
return str(pkg_ckpt)
|
|
41
|
+
|
|
42
|
+
# 4. Check Desktop dir if present from original environment
|
|
43
|
+
desktop_dir = os.path.expanduser("~/Desktop/gemini-code-1794514134973")
|
|
44
|
+
if os.path.isdir(desktop_dir):
|
|
45
|
+
dt_path = os.path.join(desktop_dir, default_name)
|
|
46
|
+
if os.path.exists(dt_path):
|
|
47
|
+
return dt_path
|
|
48
|
+
|
|
49
|
+
return default_name
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def benchmark_speculative(model, tokenizer, rag_engine, ngram_drafter, test_queries=None):
|
|
53
|
+
if test_queries is None:
|
|
54
|
+
test_queries = [
|
|
55
|
+
"what is this model?",
|
|
56
|
+
"tell me about gqa memory",
|
|
57
|
+
"how does moe work?",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
print("--- Speculative Decoding Benchmark (N-Gram Drafter gamma=4 vs. Standard Decoding) ---")
|
|
61
|
+
t_std_start = time.perf_counter()
|
|
62
|
+
std_passes = 0
|
|
63
|
+
for q in test_queries:
|
|
64
|
+
retrieved = rag_engine.retrieve(q, top_k=1)
|
|
65
|
+
top_doc = retrieved[0][0] if retrieved else {"fact": "", "question": q}
|
|
66
|
+
p_ids = tokenizer.encode(tokenizer.format_rag_prompt(top_doc["fact"], top_doc["question"]))
|
|
67
|
+
out_std = model.generate(
|
|
68
|
+
p_ids, max_new_tokens=24, temperature=0.05, stop_token_id=tokenizer.eos_id, use_kv_cache=True, use_int8=True
|
|
69
|
+
)
|
|
70
|
+
std_passes += len(out_std) - len(p_ids)
|
|
71
|
+
dt_std = (time.perf_counter() - t_std_start) * 1000.0
|
|
72
|
+
|
|
73
|
+
t_spec_start = time.perf_counter()
|
|
74
|
+
spec_passes = 0
|
|
75
|
+
total_acc, total_drf = 0, 0
|
|
76
|
+
for q in test_queries:
|
|
77
|
+
retrieved = rag_engine.retrieve(q, top_k=1)
|
|
78
|
+
top_doc = retrieved[0][0] if retrieved else {"fact": "", "question": q}
|
|
79
|
+
p_ids = tokenizer.encode(tokenizer.format_rag_prompt(top_doc["fact"], top_doc["question"]))
|
|
80
|
+
out_spec, acc, drf, passes = model.generate_speculative(
|
|
81
|
+
p_ids,
|
|
82
|
+
ngram_drafter,
|
|
83
|
+
max_new_tokens=24,
|
|
84
|
+
gamma=4,
|
|
85
|
+
temperature=0.05,
|
|
86
|
+
stop_token_id=tokenizer.eos_id,
|
|
87
|
+
use_int8=True,
|
|
88
|
+
)
|
|
89
|
+
spec_passes += passes
|
|
90
|
+
total_acc += acc
|
|
91
|
+
total_drf += drf
|
|
92
|
+
reply = tokenizer.decode(out_spec[len(p_ids):], skip_special=True).strip()
|
|
93
|
+
print(f"Q: '{q}' -> {reply} [accepted {acc}/{drf} drafted tokens in {passes} forward passes]")
|
|
94
|
+
dt_spec = (time.perf_counter() - t_spec_start) * 1000.0
|
|
95
|
+
acc_rate = (100.0 * total_acc / max(1, total_drf))
|
|
96
|
+
print(
|
|
97
|
+
f"Summary: Standard={dt_std:.1f} ms ({std_passes} passes) | "
|
|
98
|
+
f"Speculative={dt_spec:.1f} ms ({spec_passes} passes) | "
|
|
99
|
+
f"Draft Acceptance={acc_rate:.1f}% | Speedup={dt_std / max(1e-6, dt_spec):.2f}x\n"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def run_repl(model, tokenizer, rag_engine, ngram_drafter, corpus_segments, ckpt_path, block_size=28):
|
|
104
|
+
print("=====================================================================")
|
|
105
|
+
print("Interactive GQA + Sparse MoE + LoRA + Speculative Decoding REPL")
|
|
106
|
+
print("Commands:")
|
|
107
|
+
print(" - Ask any question : BM25 RAG + Speculative Decoding generates reply")
|
|
108
|
+
print(" - Teach (q :: a) : Instant LoRA fine-tuning (freezes MoE base weights!)")
|
|
109
|
+
print(" - /merge : Merge learned LoRA adapters (A @ B) into Int8 base weights")
|
|
110
|
+
print(" - /rag : List all indexed BM25 RAG documents")
|
|
111
|
+
print(" - /stats : Inspect MoE experts, LoRA rank, & Speculative stats")
|
|
112
|
+
print(" - /quit or exit : Exit the REPL")
|
|
113
|
+
print("=====================================================================")
|
|
114
|
+
|
|
115
|
+
top_k, top_p, temperature, gamma = 6, 0.9, 0.05, 4
|
|
116
|
+
|
|
117
|
+
while True:
|
|
118
|
+
try:
|
|
119
|
+
user_in = input("\nYou > ").strip()
|
|
120
|
+
except (EOFError, KeyboardInterrupt):
|
|
121
|
+
print(f"\nExiting REPL. Checkpoint is saved at {ckpt_path}")
|
|
122
|
+
break
|
|
123
|
+
|
|
124
|
+
if not user_in:
|
|
125
|
+
continue
|
|
126
|
+
if user_in.lower() in ("/quit", "exit", "quit"):
|
|
127
|
+
print(f"Goodbye! All LoRA, MoE, and RAG updates are saved in {ckpt_path}")
|
|
128
|
+
break
|
|
129
|
+
|
|
130
|
+
if user_in.startswith("/merge"):
|
|
131
|
+
merged = model.merge_lora_into_base()
|
|
132
|
+
model.save_checkpoint(ckpt_path, tokenizer, rag_engine, corpus_segments)
|
|
133
|
+
print(f"Merged {merged} active LoRA adapter pairs into Int8 base weights and saved '{ckpt_path}'!")
|
|
134
|
+
continue
|
|
135
|
+
|
|
136
|
+
if user_in.startswith("/rag"):
|
|
137
|
+
print(f"--- Pure-Python BM25 Knowledge Base ({len(rag_engine.documents)} docs) ---")
|
|
138
|
+
for idx_d, doc in enumerate(rag_engine.documents, 1):
|
|
139
|
+
print(f" {idx_d}. Q: {doc['question']} --> Fact: {doc['fact']}")
|
|
140
|
+
continue
|
|
141
|
+
|
|
142
|
+
if user_in.startswith("/stats"):
|
|
143
|
+
base_params = sum(t.rows * t.cols for t in model.weights.values())
|
|
144
|
+
lora_params = sum(A.rows * A.cols + B.rows * B.cols for A, B in model.lora.values())
|
|
145
|
+
file_kb = os.path.getsize(ckpt_path) / 1024.0 if os.path.exists(ckpt_path) else 0.0
|
|
146
|
+
print(
|
|
147
|
+
f"Architecture: {model.n_layers} Layers | GQA ({model.n_heads}Q:{model.n_kv_heads}KV) | "
|
|
148
|
+
f"Sparse MoE: {model.n_experts} SwiGLU Experts (Top-{model.moe_top_k} active per token)\n"
|
|
149
|
+
f"Parameters: {base_params} Base (frozen during REPL chat) | {lora_params} LoRA (rank={model.lora_rank})\n"
|
|
150
|
+
f"Checkpoint: '{ckpt_path}' ({file_kb:.1f} KB) | Full Steps: {model.total_steps} | LoRA Steps: {model.lora_steps}"
|
|
151
|
+
)
|
|
152
|
+
continue
|
|
153
|
+
|
|
154
|
+
if user_in.startswith("/test"):
|
|
155
|
+
print("--- Finite-Difference Gradient Verification ---")
|
|
156
|
+
from .gradcheck import run_gradient_checks
|
|
157
|
+
all_ok, _ = run_gradient_checks()
|
|
158
|
+
if all_ok:
|
|
159
|
+
print("All custom autograd operators verified to numerical calculus precision < 1e-5!")
|
|
160
|
+
else:
|
|
161
|
+
print("WARNING: One or more operators showed gradient discrepancy.")
|
|
162
|
+
continue
|
|
163
|
+
|
|
164
|
+
# Case A: Teach via DPO (q :: good || bad), KTO (q :: a ++ or q :: a --), or LoRA Continual Learning (q :: a)
|
|
165
|
+
if "::" in user_in:
|
|
166
|
+
q_part, a_part = user_in.split("::", 1)
|
|
167
|
+
a_strip = a_part.strip()
|
|
168
|
+
if "||" in a_part:
|
|
169
|
+
good_part, bad_part = a_part.split("||", 1)
|
|
170
|
+
t_dpo = time.perf_counter()
|
|
171
|
+
dpo_loss, margin = train_dpo_turn(
|
|
172
|
+
model, tokenizer, q_part.strip(), good_part.strip(), bad_part.strip()
|
|
173
|
+
)
|
|
174
|
+
dt_dpo = (time.perf_counter() - t_dpo) * 1000.0
|
|
175
|
+
model.save_checkpoint(ckpt_path, tokenizer, rag_engine, corpus_segments)
|
|
176
|
+
file_kb = os.path.getsize(ckpt_path) / 1024.0
|
|
177
|
+
print(
|
|
178
|
+
f"[LoRA DPO Adapted in {dt_dpo:.0f} ms | Loss: {dpo_loss:.4f} | Margin: {margin:+.3f} | "
|
|
179
|
+
f"Saved '{ckpt_path}' ({file_kb:.1f} KB)]"
|
|
180
|
+
)
|
|
181
|
+
user_in = q_part.strip()
|
|
182
|
+
elif a_strip.endswith("++") or a_strip.endswith("--"):
|
|
183
|
+
is_desirable = a_strip.endswith("++")
|
|
184
|
+
ans_text = a_strip[:-2].strip()
|
|
185
|
+
t_kto = time.perf_counter()
|
|
186
|
+
kto_loss, reward = train_kto_turn(
|
|
187
|
+
model, tokenizer, q_part.strip(), ans_text, is_desirable=is_desirable
|
|
188
|
+
)
|
|
189
|
+
dt_kto = (time.perf_counter() - t_kto) * 1000.0
|
|
190
|
+
model.save_checkpoint(ckpt_path, tokenizer, rag_engine, corpus_segments)
|
|
191
|
+
file_kb = os.path.getsize(ckpt_path) / 1024.0
|
|
192
|
+
sign_str = "++ Desirable" if is_desirable else "-- Undesirable"
|
|
193
|
+
print(
|
|
194
|
+
f"[LoRA KTO Adapted ({sign_str}) in {dt_kto:.0f} ms | Loss: {kto_loss:.4f} | Reward: {reward:+.3f} | "
|
|
195
|
+
f"Saved '{ckpt_path}' ({file_kb:.1f} KB)]"
|
|
196
|
+
)
|
|
197
|
+
user_in = q_part.strip()
|
|
198
|
+
else:
|
|
199
|
+
t_lora = time.perf_counter()
|
|
200
|
+
loss_after, minted = continual_learn_lora_turn(
|
|
201
|
+
model, tokenizer, rag_engine, ngram_drafter, corpus_segments, q_part, a_part, ckpt_path, block_size
|
|
202
|
+
)
|
|
203
|
+
dt_lora = (time.perf_counter() - t_lora) * 1000.0
|
|
204
|
+
file_kb = os.path.getsize(ckpt_path) / 1024.0
|
|
205
|
+
mint_msg = f"Minted BPE: {', '.join(minted)} | " if minted else ""
|
|
206
|
+
print(
|
|
207
|
+
f"[{mint_msg}LoRA Adapted in {dt_lora:.0f} ms | Loss: {loss_after:.4f} | "
|
|
208
|
+
f"Saved '{ckpt_path}' ({file_kb:.1f} KB)]"
|
|
209
|
+
)
|
|
210
|
+
user_in = q_part.strip()
|
|
211
|
+
|
|
212
|
+
# Case B: Retrieve context via BM25 RAG and answer via Speculative Decoding
|
|
213
|
+
retrieved = rag_engine.retrieve(user_in, top_k=1)
|
|
214
|
+
if retrieved:
|
|
215
|
+
top_doc, bm25_score = retrieved[0]
|
|
216
|
+
grounded_q = top_doc["question"] if bm25_score > 0.0 else user_in
|
|
217
|
+
fact = top_doc["fact"]
|
|
218
|
+
else:
|
|
219
|
+
fact, grounded_q, bm25_score = "", user_in, 0.0
|
|
220
|
+
|
|
221
|
+
prompt_str = tokenizer.format_rag_prompt(fact, grounded_q)
|
|
222
|
+
p_ids = tokenizer.encode(prompt_str)
|
|
223
|
+
|
|
224
|
+
sys.stdout.write("Assistant > ")
|
|
225
|
+
sys.stdout.flush()
|
|
226
|
+
t0 = time.perf_counter()
|
|
227
|
+
reply_ids = []
|
|
228
|
+
for tok_id in model.generate_speculative_stream(
|
|
229
|
+
p_ids,
|
|
230
|
+
ngram_drafter,
|
|
231
|
+
max_new_tokens=24,
|
|
232
|
+
gamma=gamma,
|
|
233
|
+
temperature=temperature,
|
|
234
|
+
top_k=top_k,
|
|
235
|
+
top_p=top_p,
|
|
236
|
+
stop_token_id=tokenizer.eos_id,
|
|
237
|
+
use_int8=True,
|
|
238
|
+
):
|
|
239
|
+
if tok_id == tokenizer.eos_id:
|
|
240
|
+
break
|
|
241
|
+
tok_str = tokenizer.decode([tok_id], skip_special=True)
|
|
242
|
+
sys.stdout.write(tok_str)
|
|
243
|
+
sys.stdout.flush()
|
|
244
|
+
reply_ids.append(tok_id)
|
|
245
|
+
dt_gen = (time.perf_counter() - t0) * 1000.0
|
|
246
|
+
sys.stdout.write(f"\n [BM25 score={bm25_score:.2f} | Streamed {len(reply_ids)} tokens in {dt_gen:.1f} ms]\n")
|
|
247
|
+
sys.stdout.flush()
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def main():
|
|
251
|
+
ckpt_path = resolve_checkpoint_path("micro_gpt_checkpoint.json")
|
|
252
|
+
block_size = 28
|
|
253
|
+
|
|
254
|
+
need_full_train = True
|
|
255
|
+
if os.path.exists(ckpt_path):
|
|
256
|
+
old_size_kb = os.path.getsize(ckpt_path) / 1024.0
|
|
257
|
+
print(f"Found existing checkpoint '{ckpt_path}' ({old_size_kb:.1f} KB)! Loading...")
|
|
258
|
+
model, tokenizer, rag_engine, corpus_segments, migrated_legacy = ModernMicroGPT.load_checkpoint(ckpt_path)
|
|
259
|
+
added_specials = tokenizer.ensure_special_tokens()
|
|
260
|
+
if not migrated_legacy and not added_specials and len(rag_engine.documents) >= len(DEFAULT_KNOWLEDGE):
|
|
261
|
+
need_full_train = False
|
|
262
|
+
base_params = sum(t.rows * t.cols for t in model.weights.values())
|
|
263
|
+
lora_params = sum(A.rows * A.cols + B.rows * B.cols for A, B in model.lora.values())
|
|
264
|
+
print(
|
|
265
|
+
f"Loaded {model.n_layers}-Layer GQA + Sparse MoE ({model.n_experts}E, Top-{model.moe_top_k}) + LoRA GPT!\n"
|
|
266
|
+
f"Vocab: {model.vocab_size} | Base Params: {base_params} | LoRA Params: {lora_params} | "
|
|
267
|
+
f"Size: {old_size_kb:.1f} KB\n"
|
|
268
|
+
)
|
|
269
|
+
else:
|
|
270
|
+
print("Upgrading earlier checkpoint to Sparse MoE (4 Experts, Top-2) + LoRA architecture...")
|
|
271
|
+
|
|
272
|
+
if need_full_train:
|
|
273
|
+
print("Training 2-Layer GQA + Sparse MoE SwiGLU (4 Experts, Top-2) + LoRA GPT...")
|
|
274
|
+
rag_engine = PurePythonBM25RAG()
|
|
275
|
+
tokenizer = BPETokenizer()
|
|
276
|
+
|
|
277
|
+
seed_text = "".join(
|
|
278
|
+
f"<|context|> {a} <|user|> {q} <|assistant|> {a} <|endoftext|> "
|
|
279
|
+
for q, a in DEFAULT_KNOWLEDGE
|
|
280
|
+
) * 4
|
|
281
|
+
tokenizer.train(seed_text, max_merges=18, min_freq=4)
|
|
282
|
+
|
|
283
|
+
corpus_segments = []
|
|
284
|
+
for q, a in DEFAULT_KNOWLEDGE:
|
|
285
|
+
rag_engine.add_document(q, a)
|
|
286
|
+
corpus_segments.append(tokenizer.format_rag_qa(a, q, a))
|
|
287
|
+
|
|
288
|
+
model = ModernMicroGPT(
|
|
289
|
+
vocab_size=len(tokenizer.vocab),
|
|
290
|
+
n_embd=24,
|
|
291
|
+
n_heads=4,
|
|
292
|
+
n_kv_heads=2,
|
|
293
|
+
n_layers=2,
|
|
294
|
+
n_experts=4,
|
|
295
|
+
moe_top_k=2,
|
|
296
|
+
expert_hidden=24,
|
|
297
|
+
lora_rank=4,
|
|
298
|
+
lora_alpha=8.0,
|
|
299
|
+
)
|
|
300
|
+
model.set_training_mode(lora_only=False)
|
|
301
|
+
|
|
302
|
+
base_params = sum(t.rows * t.cols for t in model.weights.values())
|
|
303
|
+
lora_params = sum(A.rows * A.cols + B.rows * B.cols for A, B in model.lora.values())
|
|
304
|
+
print(
|
|
305
|
+
f"BPE Vocab: {len(tokenizer.vocab)} tokens | Base MoE Params: {base_params} | "
|
|
306
|
+
f"LoRA Adapter Params: {lora_params} ({100.0 * lora_params / base_params:.1f}% of base)"
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
sequences = build_training_sequences(tokenizer, corpus_segments, block_size=block_size)
|
|
310
|
+
epochs = 32
|
|
311
|
+
total_train_steps = epochs * len(sequences)
|
|
312
|
+
local_step = 0
|
|
313
|
+
|
|
314
|
+
for epoch in range(1, epochs + 1):
|
|
315
|
+
random.shuffle(sequences)
|
|
316
|
+
epoch_loss = 0.0
|
|
317
|
+
for x_seq, y_seq in sequences:
|
|
318
|
+
local_step += 1
|
|
319
|
+
lr = get_cosine_lr(local_step, total_train_steps, max_lr=0.012, min_lr=0.0008, warmup_steps=20)
|
|
320
|
+
logits = model.forward(x_seq, use_int8=False)
|
|
321
|
+
loss = FlatTensor.cross_entropy(logits, y_seq)
|
|
322
|
+
loss.backward()
|
|
323
|
+
model.step_adam(lr, lora_only=False)
|
|
324
|
+
epoch_loss += loss.data[0]
|
|
325
|
+
|
|
326
|
+
if epoch % 8 == 0 or epoch == 1:
|
|
327
|
+
print(
|
|
328
|
+
f"Epoch {epoch:2d}/{epochs} | "
|
|
329
|
+
f"Avg Loss: {epoch_loss / len(sequences):.4f} | "
|
|
330
|
+
f"Cosine LR: {lr:.5f}"
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
model.save_checkpoint(ckpt_path, tokenizer, rag_engine, corpus_segments)
|
|
334
|
+
size_kb = os.path.getsize(ckpt_path) / 1024.0
|
|
335
|
+
print(f"\nSaved Int8 GQA + MoE + LoRA checkpoint to '{ckpt_path}' ({size_kb:.1f} KB)\n")
|
|
336
|
+
|
|
337
|
+
# Fit N-Gram Draft Model for Speculative Decoding
|
|
338
|
+
ngram_drafter = NGramDraftModel(max_n=4)
|
|
339
|
+
ngram_drafter.fit([tokenizer.encode(seg) for seg in corpus_segments])
|
|
340
|
+
|
|
341
|
+
# If --benchmark or non-interactive flag passed
|
|
342
|
+
if "--benchmark" in sys.argv:
|
|
343
|
+
benchmark_speculative(model, tokenizer, rag_engine, ngram_drafter)
|
|
344
|
+
return
|
|
345
|
+
|
|
346
|
+
# Benchmark Standard Autoregressive vs. Speculative Decoding (gamma=4)
|
|
347
|
+
benchmark_speculative(model, tokenizer, rag_engine, ngram_drafter)
|
|
348
|
+
|
|
349
|
+
# Interactive REPL
|
|
350
|
+
if sys.stdin.isatty():
|
|
351
|
+
run_repl(model, tokenizer, rag_engine, ngram_drafter, corpus_segments, ckpt_path, block_size=block_size)
|
|
352
|
+
else:
|
|
353
|
+
print("Non-interactive mode detected. Exiting.")
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
if __name__ == "__main__":
|
|
357
|
+
main()
|
thoughtformula/draft.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Pure-Python N-Gram Draft Model for speculative decoding."""
|
|
2
|
+
|
|
3
|
+
class NGramDraftModel:
|
|
4
|
+
"""
|
|
5
|
+
Hierarchical 4-gram -> 3-gram -> 2-gram backoff drafter over BPE token IDs.
|
|
6
|
+
Generates gamma candidate tokens in <0.02 ms for single-pass Transformer verification.
|
|
7
|
+
"""
|
|
8
|
+
def __init__(self, max_n=4):
|
|
9
|
+
self.max_n = max_n
|
|
10
|
+
self.tables = {n: {} for n in range(1, max_n)} # context_tuple -> {next_id: count}
|
|
11
|
+
|
|
12
|
+
def fit(self, token_sequences):
|
|
13
|
+
self.tables = {n: {} for n in range(1, self.max_n)}
|
|
14
|
+
for seq in token_sequences:
|
|
15
|
+
L = len(seq)
|
|
16
|
+
for n in range(1, self.max_n):
|
|
17
|
+
tbl = self.tables[n]
|
|
18
|
+
for i in range(L - n):
|
|
19
|
+
ctx = tuple(seq[i : i + n])
|
|
20
|
+
nxt = seq[i + n]
|
|
21
|
+
bucket = tbl.get(ctx)
|
|
22
|
+
if bucket is None:
|
|
23
|
+
tbl[ctx] = {nxt: 1}
|
|
24
|
+
else:
|
|
25
|
+
bucket[nxt] = bucket.get(nxt, 0) + 1
|
|
26
|
+
|
|
27
|
+
def draft_with_probs(self, prefix_ids, gamma=4, stop_token_id=None):
|
|
28
|
+
"""
|
|
29
|
+
Autoregressively drafts up to `gamma` tokens, returning list of:
|
|
30
|
+
(draft_token_id, q_prob, q_distribution_dict)
|
|
31
|
+
"""
|
|
32
|
+
results = []
|
|
33
|
+
cur = list(prefix_ids)
|
|
34
|
+
for _ in range(gamma):
|
|
35
|
+
bucket = None
|
|
36
|
+
for n in range(self.max_n - 1, 0, -1):
|
|
37
|
+
if len(cur) >= n:
|
|
38
|
+
ctx = tuple(cur[-n:])
|
|
39
|
+
b = self.tables[n].get(ctx)
|
|
40
|
+
if b:
|
|
41
|
+
bucket = b
|
|
42
|
+
break
|
|
43
|
+
if not bucket:
|
|
44
|
+
break
|
|
45
|
+
total_count = sum(bucket.values())
|
|
46
|
+
best_tok, best_count = max(bucket.items(), key=lambda kv: kv[1])
|
|
47
|
+
q_prob = best_count / total_count
|
|
48
|
+
q_dist = {tok: cnt / total_count for tok, cnt in bucket.items()}
|
|
49
|
+
results.append((best_tok, q_prob, q_dist))
|
|
50
|
+
cur.append(best_tok)
|
|
51
|
+
if stop_token_id is not None and best_tok == stop_token_id:
|
|
52
|
+
break
|
|
53
|
+
return results
|
|
54
|
+
|
|
55
|
+
def draft(self, prefix_ids, gamma=4, stop_token_id=None):
|
|
56
|
+
"""Autoregressively drafts up to `gamma` tokens using longest-suffix N-gram match."""
|
|
57
|
+
return [tok for tok, _, _ in self.draft_with_probs(prefix_ids, gamma=gamma, stop_token_id=stop_token_id)]
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""Finite-difference gradient verification for all custom autograd operators."""
|
|
2
|
+
|
|
3
|
+
from array import array
|
|
4
|
+
import math
|
|
5
|
+
import random
|
|
6
|
+
import unittest
|
|
7
|
+
|
|
8
|
+
from .tensor import FlatTensor
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def check_rmsnorm_gradient(T=2, D=4, eps=1e-5, tol=1e-5):
|
|
12
|
+
"""Verifies rmsnorm analytical backward pass against centered finite differences."""
|
|
13
|
+
data = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
|
|
14
|
+
grad_out = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
|
|
15
|
+
|
|
16
|
+
# Analytical gradient
|
|
17
|
+
x = FlatTensor(T, D, array('d', data), requires_grad=True)
|
|
18
|
+
y = x.rmsnorm()
|
|
19
|
+
y.grad = array('d', grad_out)
|
|
20
|
+
y._backward()
|
|
21
|
+
analytical = list(x.grad)
|
|
22
|
+
|
|
23
|
+
# Numerical gradient
|
|
24
|
+
numerical = []
|
|
25
|
+
for idx in range(T * D):
|
|
26
|
+
d_plus = array('d', data)
|
|
27
|
+
d_plus[idx] += eps
|
|
28
|
+
x_plus = FlatTensor(T, D, d_plus, requires_grad=False)
|
|
29
|
+
y_plus = x_plus.rmsnorm()
|
|
30
|
+
|
|
31
|
+
d_minus = array('d', data)
|
|
32
|
+
d_minus[idx] -= eps
|
|
33
|
+
x_minus = FlatTensor(T, D, d_minus, requires_grad=False)
|
|
34
|
+
y_minus = x_minus.rmsnorm()
|
|
35
|
+
|
|
36
|
+
diff_sum = sum((yp - ym) * go for yp, ym, go in zip(y_plus.data, y_minus.data, grad_out))
|
|
37
|
+
numerical.append(diff_sum / (2.0 * eps))
|
|
38
|
+
|
|
39
|
+
max_err = max(abs(a - n) for a, n in zip(analytical, numerical))
|
|
40
|
+
return max_err < tol, max_err
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def check_swiglu_gradient(T=2, D=4, eps=1e-5, tol=1e-5):
|
|
44
|
+
"""Verifies swiglu_act analytical backward pass against centered finite differences."""
|
|
45
|
+
g_data = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
|
|
46
|
+
u_data = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
|
|
47
|
+
grad_out = array('d', [random.gauss(0.0, 1.0) for _ in range(T * D)])
|
|
48
|
+
|
|
49
|
+
gate = FlatTensor(T, D, array('d', g_data), requires_grad=True)
|
|
50
|
+
up = FlatTensor(T, D, array('d', u_data), requires_grad=True)
|
|
51
|
+
out = FlatTensor.swiglu_act(gate, up)
|
|
52
|
+
out.grad = array('d', grad_out)
|
|
53
|
+
out._backward()
|
|
54
|
+
ana_g, ana_u = list(gate.grad), list(up.grad)
|
|
55
|
+
|
|
56
|
+
num_g, num_u = [], []
|
|
57
|
+
for idx in range(T * D):
|
|
58
|
+
# Gate numerical
|
|
59
|
+
gp, gm = array('d', g_data), array('d', g_data)
|
|
60
|
+
gp[idx] += eps
|
|
61
|
+
gm[idx] -= eps
|
|
62
|
+
op = FlatTensor.swiglu_act(FlatTensor(T, D, gp), FlatTensor(T, D, u_data))
|
|
63
|
+
om = FlatTensor.swiglu_act(FlatTensor(T, D, gm), FlatTensor(T, D, u_data))
|
|
64
|
+
num_g.append(sum((p - m) * go for p, m, go in zip(op.data, om.data, grad_out)) / (2.0 * eps))
|
|
65
|
+
|
|
66
|
+
# Up numerical
|
|
67
|
+
up_p, up_m = array('d', u_data), array('d', u_data)
|
|
68
|
+
up_p[idx] += eps
|
|
69
|
+
up_m[idx] -= eps
|
|
70
|
+
op = FlatTensor.swiglu_act(FlatTensor(T, D, g_data), FlatTensor(T, D, up_p))
|
|
71
|
+
om = FlatTensor.swiglu_act(FlatTensor(T, D, g_data), FlatTensor(T, D, up_m))
|
|
72
|
+
num_u.append(sum((p - m) * go for p, m, go in zip(op.data, om.data, grad_out)) / (2.0 * eps))
|
|
73
|
+
|
|
74
|
+
err_g = max(abs(a - n) for a, n in zip(ana_g, num_g))
|
|
75
|
+
err_u = max(abs(a - n) for a, n in zip(ana_u, num_u))
|
|
76
|
+
max_err = max(err_g, err_u)
|
|
77
|
+
return max_err < tol, max_err
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def check_rope_gqa_gradient(T=2, D=4, n_heads=2, n_kv_heads=1, eps=1e-5, tol=1e-5):
|
|
81
|
+
"""Verifies rope_gqa analytical backward pass against centered finite differences."""
|
|
82
|
+
kv_d = D // (n_heads // n_kv_heads)
|
|
83
|
+
q_data = array('d', [random.gauss(0.0, 0.5) for _ in range(T * D)])
|
|
84
|
+
k_data = array('d', [random.gauss(0.0, 0.5) for _ in range(T * kv_d)])
|
|
85
|
+
v_data = array('d', [random.gauss(0.0, 0.5) for _ in range(T * kv_d)])
|
|
86
|
+
grad_out = array('d', [random.gauss(0.0, 0.5) for _ in range(T * D)])
|
|
87
|
+
|
|
88
|
+
Q = FlatTensor(T, D, array('d', q_data), requires_grad=True)
|
|
89
|
+
K = FlatTensor(T, kv_d, array('d', k_data), requires_grad=True)
|
|
90
|
+
V = FlatTensor(T, kv_d, array('d', v_data), requires_grad=True)
|
|
91
|
+
out = FlatTensor.rope_gqa(Q, K, V, n_heads, n_kv_heads)
|
|
92
|
+
out.grad = array('d', grad_out)
|
|
93
|
+
out._backward()
|
|
94
|
+
ana_q, ana_k, ana_v = list(Q.grad), list(K.grad), list(V.grad)
|
|
95
|
+
|
|
96
|
+
# Numerical for V
|
|
97
|
+
num_v = []
|
|
98
|
+
for idx in range(T * kv_d):
|
|
99
|
+
vp, vm = array('d', v_data), array('d', v_data)
|
|
100
|
+
vp[idx] += eps
|
|
101
|
+
vm[idx] -= eps
|
|
102
|
+
op = FlatTensor.rope_gqa(FlatTensor(T, D, q_data), FlatTensor(T, kv_d, k_data), FlatTensor(T, kv_d, vp), n_heads, n_kv_heads)
|
|
103
|
+
om = FlatTensor.rope_gqa(FlatTensor(T, D, q_data), FlatTensor(T, kv_d, k_data), FlatTensor(T, kv_d, vm), n_heads, n_kv_heads)
|
|
104
|
+
num_v.append(sum((p - m) * go for p, m, go in zip(op.data, om.data, grad_out)) / (2.0 * eps))
|
|
105
|
+
|
|
106
|
+
err_v = max(abs(a - n) for a, n in zip(ana_v, num_v))
|
|
107
|
+
return err_v < tol, err_v
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def check_moe_router_gradient(T=2, N_E=4, D=4, eps=1e-5, tol=1e-5):
|
|
111
|
+
"""Verifies moe_router_combine analytical backward pass against centered finite differences."""
|
|
112
|
+
r_data = array('d', [random.gauss(0.0, 0.5) for _ in range(T * N_E)])
|
|
113
|
+
exp_outs = [FlatTensor(T, D, array('d', [random.gauss(0.0, 0.5) for _ in range(T * D)])) for _ in range(N_E)]
|
|
114
|
+
grad_out = array('d', [random.gauss(0.0, 0.5) for _ in range(T * D)])
|
|
115
|
+
|
|
116
|
+
R = FlatTensor(T, N_E, array('d', r_data), requires_grad=True)
|
|
117
|
+
out = FlatTensor.moe_router_combine(R, exp_outs, top_k=2)
|
|
118
|
+
out.grad = array('d', grad_out)
|
|
119
|
+
out._backward()
|
|
120
|
+
ana_r = list(R.grad)
|
|
121
|
+
|
|
122
|
+
num_r = []
|
|
123
|
+
for idx in range(T * N_E):
|
|
124
|
+
rp, rm = array('d', r_data), array('d', r_data)
|
|
125
|
+
rp[idx] += eps
|
|
126
|
+
rm[idx] -= eps
|
|
127
|
+
op = FlatTensor.moe_router_combine(FlatTensor(T, N_E, rp), exp_outs, top_k=2)
|
|
128
|
+
om = FlatTensor.moe_router_combine(FlatTensor(T, N_E, rm), exp_outs, top_k=2)
|
|
129
|
+
num_r.append(sum((p - m) * go for p, m, go in zip(op.data, om.data, grad_out)) / (2.0 * eps))
|
|
130
|
+
|
|
131
|
+
max_err = max(abs(a - n) for a, n in zip(ana_r, num_r))
|
|
132
|
+
return max_err < tol, max_err
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def run_gradient_checks():
|
|
136
|
+
"""Runs all finite-difference gradient checks, printing verification report."""
|
|
137
|
+
checks = [
|
|
138
|
+
("RMSNorm", check_rmsnorm_gradient),
|
|
139
|
+
("SwiGLU Activation", check_swiglu_gradient),
|
|
140
|
+
("RoPE GQA Attention", check_rope_gqa_gradient),
|
|
141
|
+
("MoE Router Combine", check_moe_router_gradient),
|
|
142
|
+
]
|
|
143
|
+
all_passed = True
|
|
144
|
+
results = {}
|
|
145
|
+
for name, fn in checks:
|
|
146
|
+
passed, err = fn()
|
|
147
|
+
results[name] = (passed, err)
|
|
148
|
+
status = "PASSED" if passed else "FAILED"
|
|
149
|
+
print(f" [{status}] {name:20s} (max finite-difference error: {err:.2e})")
|
|
150
|
+
if not passed:
|
|
151
|
+
all_passed = False
|
|
152
|
+
return all_passed, results
|