thoughtformula 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,39 @@
1
+ """Thoughtformula: Pure-Python Transformer with GQA, Sparse MoE SwiGLU, Int8 quantization, LoRA, and Speculative Decoding."""
2
+
3
+ from .tensor import FlatTensor
4
+ from .draft import NGramDraftModel
5
+ from .rag import PurePythonBM25RAG, HybridDenseBM25RAG
6
+ from .tokenizer import BPETokenizer
7
+ from .model import ModernMicroGPT
8
+ from .medusa import MedusaDrafter, MedusaHead
9
+ from .cache import PagedKVCache, PagedKVCacheManager, BlockAllocator
10
+ from .batching import ContinuousBatchScheduler, InferenceRequest
11
+ from .training import get_cosine_lr, build_training_sequences, continual_learn_lora_turn, train_dpo_turn, train_kto_turn
12
+ from .gradcheck import run_gradient_checks
13
+ from .cli import main
14
+
15
+ __version__ = "0.3.0"
16
+
17
+ __all__ = [
18
+ "FlatTensor",
19
+ "NGramDraftModel",
20
+ "PurePythonBM25RAG",
21
+ "HybridDenseBM25RAG",
22
+ "BPETokenizer",
23
+ "ModernMicroGPT",
24
+ "MedusaDrafter",
25
+ "MedusaHead",
26
+ "PagedKVCache",
27
+ "PagedKVCacheManager",
28
+ "BlockAllocator",
29
+ "ContinuousBatchScheduler",
30
+ "InferenceRequest",
31
+ "get_cosine_lr",
32
+ "build_training_sequences",
33
+ "continual_learn_lora_turn",
34
+ "train_dpo_turn",
35
+ "train_kto_turn",
36
+ "run_gradient_checks",
37
+ "main",
38
+ "__version__",
39
+ ]
@@ -0,0 +1,220 @@
1
+ """Pure-Python continuous batching scheduler for concurrent multi-request inference.
2
+
3
+ Implements iteration-level scheduling (Orca / vLLM style):
4
+ - Packs concurrent active decode tokens into unified forward passes.
5
+ - Dynamically admits pending requests as active requests finish.
6
+ - Isolates attention memory per sequence KV-cache while amortizing linear weight projections.
7
+ """
8
+
9
+ from array import array
10
+ import uuid
11
+ from typing import Generator
12
+
13
+ from .model import ModernMicroGPT
14
+
15
+
16
+ class InferenceRequest:
17
+ """Represents a single generation request managed by the continuous batcher."""
18
+
19
+ __slots__ = (
20
+ "request_id",
21
+ "prompt_ids",
22
+ "max_new_tokens",
23
+ "temperature",
24
+ "top_k",
25
+ "top_p",
26
+ "stop_token_id",
27
+ "use_int8",
28
+ "kv_cache",
29
+ "output_tokens",
30
+ "is_finished",
31
+ "prefilled",
32
+ )
33
+
34
+ def __init__(
35
+ self,
36
+ prompt_ids: list[int],
37
+ request_id: str | int | None = None,
38
+ max_new_tokens: int = 24,
39
+ temperature: float = 0.05,
40
+ top_k: int = 6,
41
+ top_p: float = 0.9,
42
+ stop_token_id: int | None = None,
43
+ use_int8: bool = True,
44
+ kv_cache=None,
45
+ ):
46
+ self.request_id = str(uuid.uuid4())[:8] if request_id is None else request_id
47
+ self.prompt_ids = list(prompt_ids)
48
+ self.max_new_tokens = max_new_tokens
49
+ self.temperature = temperature
50
+ self.top_k = top_k
51
+ self.top_p = top_p
52
+ self.stop_token_id = stop_token_id
53
+ self.use_int8 = use_int8
54
+ self.kv_cache = kv_cache
55
+ self.output_tokens: list[int] = []
56
+ self.is_finished = False
57
+ self.prefilled = False
58
+
59
+
60
+ class ContinuousBatchScheduler:
61
+ """
62
+ Synchronous continuous batching scheduler:
63
+ Packs decode tokens from up to max_batch_size active requests into single forward passes,
64
+ admitting new requests on the fly without waiting for previous requests to finish.
65
+ """
66
+
67
+ def __init__(
68
+ self,
69
+ model: ModernMicroGPT,
70
+ max_batch_size: int = 4,
71
+ use_paged_cache: bool = True,
72
+ use_int8: bool = True,
73
+ ):
74
+ self.model = model
75
+ self.max_batch_size = max_batch_size
76
+ self.use_paged_cache = use_paged_cache
77
+ self.use_int8 = use_int8
78
+ self.pending_queue: list[InferenceRequest] = []
79
+ self.active_requests: list[InferenceRequest] = []
80
+ self.completed_requests: dict[str | int, list[int]] = {}
81
+
82
+ def add_request(
83
+ self,
84
+ prompt_ids: list[int],
85
+ request_id: str | int | None = None,
86
+ max_new_tokens: int = 24,
87
+ temperature: float = 0.05,
88
+ top_k: int = 6,
89
+ top_p: float = 0.9,
90
+ stop_token_id: int | None = None,
91
+ ) -> InferenceRequest:
92
+ """Enqueues a new generation request into the pending queue."""
93
+ cache = self.model.init_kv_cache(use_paged=self.use_paged_cache)
94
+ req = InferenceRequest(
95
+ prompt_ids=prompt_ids,
96
+ request_id=request_id,
97
+ max_new_tokens=max_new_tokens,
98
+ temperature=temperature,
99
+ top_k=top_k,
100
+ top_p=top_p,
101
+ stop_token_id=stop_token_id,
102
+ use_int8=self.use_int8,
103
+ kv_cache=cache,
104
+ )
105
+ self.pending_queue.append(req)
106
+ return req
107
+
108
+ def has_work(self) -> bool:
109
+ """Returns True if there are pending or active requests being processed."""
110
+ return bool(self.pending_queue or self.active_requests)
111
+
112
+ def step(self) -> dict[str | int, int]:
113
+ """
114
+ Executes one continuous batching iteration:
115
+ 1. Admits pending requests up to max_batch_size.
116
+ 2. Runs prompt prefill for newly admitted requests.
117
+ 3. Packs active decode requests into a single forward pass.
118
+ 4. Evicts finished requests and returns {request_id: token_id} for this step.
119
+ """
120
+ if not self.has_work():
121
+ return {}
122
+
123
+ step_outputs: dict[str | int, int] = {}
124
+ V = self.model.vocab_size
125
+
126
+ # 1. Admit pending requests into active batch
127
+ while len(self.active_requests) < self.max_batch_size and self.pending_queue:
128
+ admitted = self.pending_queue.pop(0)
129
+ self.active_requests.append(admitted)
130
+
131
+ # 2. Prefill newly admitted requests (1 request at a time to populate its KV cache)
132
+ decode_candidates: list[InferenceRequest] = []
133
+ for req in self.active_requests:
134
+ if not req.prefilled:
135
+ if not req.prompt_ids:
136
+ req.is_finished = True
137
+ continue
138
+ logits = self.model.forward(
139
+ req.prompt_ids,
140
+ start_pos=0,
141
+ kv_caches=req.kv_cache,
142
+ use_int8=req.use_int8,
143
+ )
144
+ last_row = logits.data[(logits.rows - 1) * V : logits.rows * V]
145
+ next_id = self.model.sample_top_k_top_p(
146
+ last_row, req.temperature, req.top_k, req.top_p
147
+ )
148
+ req.output_tokens.append(next_id)
149
+ req.prefilled = True
150
+ step_outputs[req.request_id] = next_id
151
+
152
+ if (req.stop_token_id is not None and next_id == req.stop_token_id) or len(
153
+ req.output_tokens
154
+ ) >= req.max_new_tokens:
155
+ req.is_finished = True
156
+ else:
157
+ decode_candidates.append(req)
158
+
159
+ # 3. Packed continuous decode forward pass for active requests
160
+ if decode_candidates:
161
+ tokens = [req.output_tokens[-1] for req in decode_candidates]
162
+ caches = [req.kv_cache for req in decode_candidates]
163
+ positions = [
164
+ req.kv_cache.seq_len
165
+ if getattr(req.kv_cache, "is_paged", False)
166
+ else req.kv_cache[0]["seq_len"]
167
+ for req in decode_candidates
168
+ ]
169
+
170
+ batched_logits = self.model.forward_batched(
171
+ tokens, caches, positions, use_int8=self.use_int8
172
+ )
173
+
174
+ for idx, req in enumerate(decode_candidates):
175
+ row = batched_logits.data[idx * V : (idx + 1) * V]
176
+ next_id = self.model.sample_top_k_top_p(
177
+ row, req.temperature, req.top_k, req.top_p
178
+ )
179
+ req.output_tokens.append(next_id)
180
+ step_outputs[req.request_id] = next_id
181
+
182
+ if (req.stop_token_id is not None and next_id == req.stop_token_id) or len(
183
+ req.output_tokens
184
+ ) >= req.max_new_tokens:
185
+ req.is_finished = True
186
+
187
+ # 4. Evict finished requests and free their KV-caches
188
+ remaining_active = []
189
+ for req in self.active_requests:
190
+ if req.is_finished:
191
+ self.completed_requests[req.request_id] = list(req.output_tokens)
192
+ if hasattr(req.kv_cache, "release"):
193
+ req.kv_cache.release()
194
+ else:
195
+ remaining_active.append(req)
196
+ self.active_requests = remaining_active
197
+
198
+ return step_outputs
199
+
200
+ def run_all(self) -> dict[str | int, list[int]]:
201
+ """Runs the continuous batcher to completion across all enqueued requests."""
202
+ while self.has_work():
203
+ self.step()
204
+ return self.completed_requests
205
+
206
+ def stream(self) -> Generator[dict[str | int, int], None, None]:
207
+ """Yields step_outputs dict at each decoding iteration until all requests finish."""
208
+ while self.has_work():
209
+ outputs = self.step()
210
+ if outputs:
211
+ yield outputs
212
+
213
+ def stats(self) -> dict:
214
+ """Returns scheduler queues and active batch telemetry."""
215
+ return {
216
+ "pending_count": len(self.pending_queue),
217
+ "active_count": len(self.active_requests),
218
+ "completed_count": len(self.completed_requests),
219
+ "max_batch_size": self.max_batch_size,
220
+ }
@@ -0,0 +1,318 @@
1
+ """Paged KV-cache memory manager for non-contiguous multi-turn allocation.
2
+
3
+ Implements vLLM-style paged attention memory management:
4
+ - PhysicalBlock: Fixed-size page holding quantized key/value states and scales.
5
+ - BlockAllocator: Global pool for physical block allocation and reference counting.
6
+ - PagedLayerCache: Logical page table mapping sequence positions to physical blocks.
7
+ - PagedKVCache: Multi-layer sequence container supporting O(1) prefix forks and copy-on-write.
8
+ - PagedKVCacheManager: High-level coordinator for multi-turn request allocation.
9
+ """
10
+
11
+ from array import array
12
+ import math
13
+
14
+
15
+ class PhysicalBlock:
16
+ """Fixed-size physical block (page) storing quantized KV token slots."""
17
+
18
+ __slots__ = (
19
+ "block_id",
20
+ "page_size",
21
+ "kv_dim",
22
+ "ref_count",
23
+ "num_tokens",
24
+ "k_int8",
25
+ "v_int8",
26
+ "k_scale",
27
+ "v_scale",
28
+ )
29
+
30
+ def __init__(self, block_id: int, page_size: int, kv_dim: int):
31
+ self.block_id = block_id
32
+ self.page_size = page_size
33
+ self.kv_dim = kv_dim
34
+ self.ref_count = 1
35
+ self.num_tokens = 0
36
+ self.k_int8 = array('b', [0] * (page_size * kv_dim))
37
+ self.v_int8 = array('b', [0] * (page_size * kv_dim))
38
+ self.k_scale = array('d', [0.0] * page_size)
39
+ self.v_scale = array('d', [0.0] * page_size)
40
+
41
+ def is_full(self) -> bool:
42
+ return self.num_tokens >= self.page_size
43
+
44
+ def append_slot(self, k_quant, v_quant, sk: float, sv: float):
45
+ """Appends a single token's KV quant data and scale factors into the block."""
46
+ idx = self.num_tokens
47
+ base = idx * self.kv_dim
48
+ for d in range(self.kv_dim):
49
+ self.k_int8[base + d] = k_quant[d]
50
+ self.v_int8[base + d] = v_quant[d]
51
+ self.k_scale[idx] = sk
52
+ self.v_scale[idx] = sv
53
+ self.num_tokens += 1
54
+
55
+ def copy(self, new_block_id: int) -> "PhysicalBlock":
56
+ """Creates a deep copy of this physical block with ref_count=1 (Copy-On-Write)."""
57
+ copied = PhysicalBlock(new_block_id, self.page_size, self.kv_dim)
58
+ copied.num_tokens = self.num_tokens
59
+ copied.k_int8 = array('b', self.k_int8)
60
+ copied.v_int8 = array('b', self.v_int8)
61
+ copied.k_scale = array('d', self.k_scale)
62
+ copied.v_scale = array('d', self.v_scale)
63
+ return copied
64
+
65
+ def reset(self):
66
+ """Resets block state to be reused in the allocator pool."""
67
+ self.ref_count = 1
68
+ self.num_tokens = 0
69
+
70
+
71
+ class BlockAllocator:
72
+ """Manages a pool of physical blocks with reference counting and reclamation."""
73
+
74
+ def __init__(self, page_size: int = 16, kv_dim: int = 24, initial_blocks: int = 64):
75
+ self.page_size = page_size
76
+ self.kv_dim = kv_dim
77
+ self.blocks: dict[int, PhysicalBlock] = {}
78
+ self.free_pool: list[int] = []
79
+ self._next_id = 0
80
+
81
+ for _ in range(initial_blocks):
82
+ self._grow()
83
+
84
+ def _grow(self) -> int:
85
+ bid = self._next_id
86
+ self._next_id += 1
87
+ block = PhysicalBlock(bid, self.page_size, self.kv_dim)
88
+ self.blocks[bid] = block
89
+ self.free_pool.append(bid)
90
+ return bid
91
+
92
+ def allocate(self) -> PhysicalBlock:
93
+ """Allocates a free block, growing the pool if empty."""
94
+ if not self.free_pool:
95
+ self._grow()
96
+ bid = self.free_pool.pop()
97
+ block = self.blocks[bid]
98
+ block.reset()
99
+ return block
100
+
101
+ def retain(self, block_id: int):
102
+ """Increments block reference count for shared prefix pages."""
103
+ self.blocks[block_id].ref_count += 1
104
+
105
+ def release(self, block_id: int):
106
+ """Decrements reference count and returns block to free pool if ref_count == 0."""
107
+ block = self.blocks[block_id]
108
+ block.ref_count -= 1
109
+ if block.ref_count <= 0:
110
+ block.ref_count = 0
111
+ self.free_pool.append(block_id)
112
+
113
+ @property
114
+ def free_blocks_count(self) -> int:
115
+ return len(self.free_pool)
116
+
117
+ @property
118
+ def total_blocks_count(self) -> int:
119
+ return self._next_id
120
+
121
+ @property
122
+ def active_blocks_count(self) -> int:
123
+ return self._next_id - len(self.free_pool)
124
+
125
+
126
+ class PagedLayerCache:
127
+ """Logical page table for one transformer layer of an active sequence."""
128
+
129
+ __slots__ = (
130
+ "layer_idx",
131
+ "page_size",
132
+ "kv_dim",
133
+ "allocator",
134
+ "page_table",
135
+ "seq_len",
136
+ "is_paged",
137
+ )
138
+
139
+ def __init__(self, layer_idx: int, page_size: int, kv_dim: int, allocator: BlockAllocator):
140
+ self.layer_idx = layer_idx
141
+ self.page_size = page_size
142
+ self.kv_dim = kv_dim
143
+ self.allocator = allocator
144
+ self.page_table: list[int] = []
145
+ self.seq_len = 0
146
+ self.is_paged = True
147
+
148
+ def get_block(self, page_idx: int) -> PhysicalBlock:
149
+ return self.allocator.blocks[self.page_table[page_idx]]
150
+
151
+ def append_token(self, k_slice, v_slice):
152
+ """Quantizes token KV to Int8 and appends to the current tail page with CoW support."""
153
+ kv_d = self.kv_dim
154
+
155
+ # Compute int8 dynamic scales
156
+ max_k = 0.0
157
+ max_v = 0.0
158
+ for i in range(kv_d):
159
+ ak = abs(k_slice[i])
160
+ av = abs(v_slice[i])
161
+ if ak > max_k:
162
+ max_k = ak
163
+ if av > max_v:
164
+ max_v = av
165
+
166
+ sk = max_k / 127.0 if max_k > 1e-12 else 1.0
167
+ sv = max_v / 127.0 if max_v > 1e-12 else 1.0
168
+ inv_sk = 127.0 / max_k if max_k > 1e-12 else 0.0
169
+ inv_sv = 127.0 / max_v if max_v > 1e-12 else 0.0
170
+
171
+ k_quant = [max(-127, min(127, int(round(k_slice[i] * inv_sk)))) for i in range(kv_d)]
172
+ v_quant = [max(-127, min(127, int(round(v_slice[i] * inv_sv)))) for i in range(kv_d)]
173
+
174
+ # Find or allocate active tail block
175
+ if not self.page_table or self.get_block(len(self.page_table) - 1).is_full():
176
+ new_block = self.allocator.allocate()
177
+ self.page_table.append(new_block.block_id)
178
+ tail_block = new_block
179
+ else:
180
+ tail_block = self.get_block(len(self.page_table) - 1)
181
+ # Copy-on-Write: If block is shared with another sequence, clone it
182
+ if tail_block.ref_count > 1:
183
+ cloned = tail_block.copy(self.allocator.allocate().block_id)
184
+ self.allocator.release(tail_block.block_id)
185
+ self.page_table[-1] = cloned.block_id
186
+ tail_block = cloned
187
+
188
+ tail_block.append_slot(k_quant, v_quant, sk, sv)
189
+ self.seq_len += 1
190
+
191
+ def truncate(self, target_len: int):
192
+ """Truncates layer sequence to target_len and reclaims unused trailing blocks."""
193
+ if self.seq_len <= target_len:
194
+ return
195
+
196
+ needed_pages = (target_len + self.page_size - 1) // self.page_size if target_len > 0 else 0
197
+ while len(self.page_table) > needed_pages:
198
+ bid = self.page_table.pop()
199
+ self.allocator.release(bid)
200
+
201
+ if needed_pages > 0:
202
+ tail = self.get_block(needed_pages - 1)
203
+ # Copy on write if shared
204
+ if tail.ref_count > 1:
205
+ cloned = tail.copy(self.allocator.allocate().block_id)
206
+ self.allocator.release(tail.block_id)
207
+ self.page_table[-1] = cloned.block_id
208
+ tail = cloned
209
+ rem = target_len % self.page_size
210
+ tail.num_tokens = rem if rem > 0 else self.page_size
211
+
212
+ self.seq_len = target_len
213
+
214
+ def fork(self) -> "PagedLayerCache":
215
+ """Forks this layer cache, sharing all physical blocks via reference counting."""
216
+ forked = PagedLayerCache(self.layer_idx, self.page_size, self.kv_dim, self.allocator)
217
+ forked.page_table = list(self.page_table)
218
+ forked.seq_len = self.seq_len
219
+ for bid in self.page_table:
220
+ self.allocator.retain(bid)
221
+ return forked
222
+
223
+ def release(self):
224
+ """Releases all blocks allocated by this layer cache."""
225
+ for bid in self.page_table:
226
+ self.allocator.release(bid)
227
+ self.page_table.clear()
228
+ self.seq_len = 0
229
+
230
+
231
+ class PagedKVCache:
232
+ """Full multi-layer paged KV-cache container for a single sequence."""
233
+
234
+ __slots__ = ("n_layers", "page_size", "kv_dim", "allocator", "layers", "is_paged")
235
+
236
+ def __init__(self, n_layers: int, page_size: int, kv_dim: int, allocator: BlockAllocator):
237
+ self.n_layers = n_layers
238
+ self.page_size = page_size
239
+ self.kv_dim = kv_dim
240
+ self.allocator = allocator
241
+ self.layers = [
242
+ PagedLayerCache(l, page_size, kv_dim, allocator) for l in range(n_layers)
243
+ ]
244
+ self.is_paged = True
245
+
246
+ @property
247
+ def seq_len(self) -> int:
248
+ return self.layers[0].seq_len if self.layers else 0
249
+
250
+ def __getitem__(self, idx: int) -> PagedLayerCache:
251
+ return self.layers[idx]
252
+
253
+ def __len__(self) -> int:
254
+ return self.n_layers
255
+
256
+ def fork(self) -> "PagedKVCache":
257
+ """O(1) prefix sharing: forks the entire multi-layer cache without copying buffers."""
258
+ forked = PagedKVCache(self.n_layers, self.page_size, self.kv_dim, self.allocator)
259
+ forked.layers = [layer.fork() for layer in self.layers]
260
+ return forked
261
+
262
+ def truncate(self, target_len: int):
263
+ """Truncates all layer caches to target_len."""
264
+ for layer in self.layers:
265
+ layer.truncate(target_len)
266
+
267
+ def release(self):
268
+ """Releases all physical memory held by this cache."""
269
+ for layer in self.layers:
270
+ layer.release()
271
+
272
+
273
+ class PagedKVCacheManager:
274
+ """High-level coordinator managing block allocation and sequence caches."""
275
+
276
+ def __init__(self, n_layers: int, kv_dim: int, page_size: int = 16, initial_blocks: int = 64):
277
+ self.n_layers = n_layers
278
+ self.kv_dim = kv_dim
279
+ self.page_size = page_size
280
+ self.allocator = BlockAllocator(page_size=page_size, kv_dim=kv_dim, initial_blocks=initial_blocks)
281
+ self.active_caches: list[PagedKVCache] = []
282
+
283
+ def create_cache(self) -> PagedKVCache:
284
+ cache = PagedKVCache(self.n_layers, self.page_size, self.kv_dim, self.allocator)
285
+ self.active_caches.append(cache)
286
+ return cache
287
+
288
+ def fork_cache(self, cache: PagedKVCache) -> PagedKVCache:
289
+ forked = cache.fork()
290
+ self.active_caches.append(forked)
291
+ return forked
292
+
293
+ def free_cache(self, cache: PagedKVCache):
294
+ if cache in self.active_caches:
295
+ self.active_caches.remove(cache)
296
+ cache.release()
297
+
298
+ def stats(self) -> dict:
299
+ """Returns memory manager statistics including prefix sharing deduplication."""
300
+ total_tokens = sum(c.seq_len for c in self.active_caches)
301
+ unpaged_blocks_needed = sum(
302
+ math.ceil(c.seq_len / self.page_size) * self.n_layers for c in self.active_caches
303
+ )
304
+ active_blocks = self.allocator.active_blocks_count
305
+ saved_blocks = max(0, unpaged_blocks_needed - active_blocks)
306
+ sharing_ratio = (
307
+ unpaged_blocks_needed / active_blocks if active_blocks > 0 else 1.0
308
+ )
309
+ return {
310
+ "page_size": self.page_size,
311
+ "total_sequences": len(self.active_caches),
312
+ "total_tokens": total_tokens,
313
+ "total_blocks": self.allocator.total_blocks_count,
314
+ "free_blocks": self.allocator.free_blocks_count,
315
+ "active_blocks": active_blocks,
316
+ "saved_blocks_via_sharing": saved_blocks,
317
+ "sharing_ratio": sharing_ratio,
318
+ }