thoughtformula 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- thoughtformula/__init__.py +39 -0
- thoughtformula/batching.py +220 -0
- thoughtformula/cache.py +318 -0
- thoughtformula/cli.py +357 -0
- thoughtformula/draft.py +57 -0
- thoughtformula/gradcheck.py +152 -0
- thoughtformula/medusa.py +138 -0
- thoughtformula/micro_gpt_checkpoint.json +1 -0
- thoughtformula/micro_gpt_v8.py +12 -0
- thoughtformula/model.py +1198 -0
- thoughtformula/rag.py +321 -0
- thoughtformula/tensor.py +1192 -0
- thoughtformula/tokenizer.py +171 -0
- thoughtformula/training.py +246 -0
- thoughtformula-0.3.0.dist-info/METADATA +152 -0
- thoughtformula-0.3.0.dist-info/RECORD +18 -0
- thoughtformula-0.3.0.dist-info/WHEEL +4 -0
- thoughtformula-0.3.0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Thoughtformula: Pure-Python Transformer with GQA, Sparse MoE SwiGLU, Int8 quantization, LoRA, and Speculative Decoding."""
|
|
2
|
+
|
|
3
|
+
from .tensor import FlatTensor
|
|
4
|
+
from .draft import NGramDraftModel
|
|
5
|
+
from .rag import PurePythonBM25RAG, HybridDenseBM25RAG
|
|
6
|
+
from .tokenizer import BPETokenizer
|
|
7
|
+
from .model import ModernMicroGPT
|
|
8
|
+
from .medusa import MedusaDrafter, MedusaHead
|
|
9
|
+
from .cache import PagedKVCache, PagedKVCacheManager, BlockAllocator
|
|
10
|
+
from .batching import ContinuousBatchScheduler, InferenceRequest
|
|
11
|
+
from .training import get_cosine_lr, build_training_sequences, continual_learn_lora_turn, train_dpo_turn, train_kto_turn
|
|
12
|
+
from .gradcheck import run_gradient_checks
|
|
13
|
+
from .cli import main
|
|
14
|
+
|
|
15
|
+
__version__ = "0.3.0"
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"FlatTensor",
|
|
19
|
+
"NGramDraftModel",
|
|
20
|
+
"PurePythonBM25RAG",
|
|
21
|
+
"HybridDenseBM25RAG",
|
|
22
|
+
"BPETokenizer",
|
|
23
|
+
"ModernMicroGPT",
|
|
24
|
+
"MedusaDrafter",
|
|
25
|
+
"MedusaHead",
|
|
26
|
+
"PagedKVCache",
|
|
27
|
+
"PagedKVCacheManager",
|
|
28
|
+
"BlockAllocator",
|
|
29
|
+
"ContinuousBatchScheduler",
|
|
30
|
+
"InferenceRequest",
|
|
31
|
+
"get_cosine_lr",
|
|
32
|
+
"build_training_sequences",
|
|
33
|
+
"continual_learn_lora_turn",
|
|
34
|
+
"train_dpo_turn",
|
|
35
|
+
"train_kto_turn",
|
|
36
|
+
"run_gradient_checks",
|
|
37
|
+
"main",
|
|
38
|
+
"__version__",
|
|
39
|
+
]
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Pure-Python continuous batching scheduler for concurrent multi-request inference.
|
|
2
|
+
|
|
3
|
+
Implements iteration-level scheduling (Orca / vLLM style):
|
|
4
|
+
- Packs concurrent active decode tokens into unified forward passes.
|
|
5
|
+
- Dynamically admits pending requests as active requests finish.
|
|
6
|
+
- Isolates attention memory per sequence KV-cache while amortizing linear weight projections.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from array import array
|
|
10
|
+
import uuid
|
|
11
|
+
from typing import Generator
|
|
12
|
+
|
|
13
|
+
from .model import ModernMicroGPT
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class InferenceRequest:
|
|
17
|
+
"""Represents a single generation request managed by the continuous batcher."""
|
|
18
|
+
|
|
19
|
+
__slots__ = (
|
|
20
|
+
"request_id",
|
|
21
|
+
"prompt_ids",
|
|
22
|
+
"max_new_tokens",
|
|
23
|
+
"temperature",
|
|
24
|
+
"top_k",
|
|
25
|
+
"top_p",
|
|
26
|
+
"stop_token_id",
|
|
27
|
+
"use_int8",
|
|
28
|
+
"kv_cache",
|
|
29
|
+
"output_tokens",
|
|
30
|
+
"is_finished",
|
|
31
|
+
"prefilled",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
prompt_ids: list[int],
|
|
37
|
+
request_id: str | int | None = None,
|
|
38
|
+
max_new_tokens: int = 24,
|
|
39
|
+
temperature: float = 0.05,
|
|
40
|
+
top_k: int = 6,
|
|
41
|
+
top_p: float = 0.9,
|
|
42
|
+
stop_token_id: int | None = None,
|
|
43
|
+
use_int8: bool = True,
|
|
44
|
+
kv_cache=None,
|
|
45
|
+
):
|
|
46
|
+
self.request_id = str(uuid.uuid4())[:8] if request_id is None else request_id
|
|
47
|
+
self.prompt_ids = list(prompt_ids)
|
|
48
|
+
self.max_new_tokens = max_new_tokens
|
|
49
|
+
self.temperature = temperature
|
|
50
|
+
self.top_k = top_k
|
|
51
|
+
self.top_p = top_p
|
|
52
|
+
self.stop_token_id = stop_token_id
|
|
53
|
+
self.use_int8 = use_int8
|
|
54
|
+
self.kv_cache = kv_cache
|
|
55
|
+
self.output_tokens: list[int] = []
|
|
56
|
+
self.is_finished = False
|
|
57
|
+
self.prefilled = False
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ContinuousBatchScheduler:
|
|
61
|
+
"""
|
|
62
|
+
Synchronous continuous batching scheduler:
|
|
63
|
+
Packs decode tokens from up to max_batch_size active requests into single forward passes,
|
|
64
|
+
admitting new requests on the fly without waiting for previous requests to finish.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def __init__(
|
|
68
|
+
self,
|
|
69
|
+
model: ModernMicroGPT,
|
|
70
|
+
max_batch_size: int = 4,
|
|
71
|
+
use_paged_cache: bool = True,
|
|
72
|
+
use_int8: bool = True,
|
|
73
|
+
):
|
|
74
|
+
self.model = model
|
|
75
|
+
self.max_batch_size = max_batch_size
|
|
76
|
+
self.use_paged_cache = use_paged_cache
|
|
77
|
+
self.use_int8 = use_int8
|
|
78
|
+
self.pending_queue: list[InferenceRequest] = []
|
|
79
|
+
self.active_requests: list[InferenceRequest] = []
|
|
80
|
+
self.completed_requests: dict[str | int, list[int]] = {}
|
|
81
|
+
|
|
82
|
+
def add_request(
|
|
83
|
+
self,
|
|
84
|
+
prompt_ids: list[int],
|
|
85
|
+
request_id: str | int | None = None,
|
|
86
|
+
max_new_tokens: int = 24,
|
|
87
|
+
temperature: float = 0.05,
|
|
88
|
+
top_k: int = 6,
|
|
89
|
+
top_p: float = 0.9,
|
|
90
|
+
stop_token_id: int | None = None,
|
|
91
|
+
) -> InferenceRequest:
|
|
92
|
+
"""Enqueues a new generation request into the pending queue."""
|
|
93
|
+
cache = self.model.init_kv_cache(use_paged=self.use_paged_cache)
|
|
94
|
+
req = InferenceRequest(
|
|
95
|
+
prompt_ids=prompt_ids,
|
|
96
|
+
request_id=request_id,
|
|
97
|
+
max_new_tokens=max_new_tokens,
|
|
98
|
+
temperature=temperature,
|
|
99
|
+
top_k=top_k,
|
|
100
|
+
top_p=top_p,
|
|
101
|
+
stop_token_id=stop_token_id,
|
|
102
|
+
use_int8=self.use_int8,
|
|
103
|
+
kv_cache=cache,
|
|
104
|
+
)
|
|
105
|
+
self.pending_queue.append(req)
|
|
106
|
+
return req
|
|
107
|
+
|
|
108
|
+
def has_work(self) -> bool:
|
|
109
|
+
"""Returns True if there are pending or active requests being processed."""
|
|
110
|
+
return bool(self.pending_queue or self.active_requests)
|
|
111
|
+
|
|
112
|
+
def step(self) -> dict[str | int, int]:
|
|
113
|
+
"""
|
|
114
|
+
Executes one continuous batching iteration:
|
|
115
|
+
1. Admits pending requests up to max_batch_size.
|
|
116
|
+
2. Runs prompt prefill for newly admitted requests.
|
|
117
|
+
3. Packs active decode requests into a single forward pass.
|
|
118
|
+
4. Evicts finished requests and returns {request_id: token_id} for this step.
|
|
119
|
+
"""
|
|
120
|
+
if not self.has_work():
|
|
121
|
+
return {}
|
|
122
|
+
|
|
123
|
+
step_outputs: dict[str | int, int] = {}
|
|
124
|
+
V = self.model.vocab_size
|
|
125
|
+
|
|
126
|
+
# 1. Admit pending requests into active batch
|
|
127
|
+
while len(self.active_requests) < self.max_batch_size and self.pending_queue:
|
|
128
|
+
admitted = self.pending_queue.pop(0)
|
|
129
|
+
self.active_requests.append(admitted)
|
|
130
|
+
|
|
131
|
+
# 2. Prefill newly admitted requests (1 request at a time to populate its KV cache)
|
|
132
|
+
decode_candidates: list[InferenceRequest] = []
|
|
133
|
+
for req in self.active_requests:
|
|
134
|
+
if not req.prefilled:
|
|
135
|
+
if not req.prompt_ids:
|
|
136
|
+
req.is_finished = True
|
|
137
|
+
continue
|
|
138
|
+
logits = self.model.forward(
|
|
139
|
+
req.prompt_ids,
|
|
140
|
+
start_pos=0,
|
|
141
|
+
kv_caches=req.kv_cache,
|
|
142
|
+
use_int8=req.use_int8,
|
|
143
|
+
)
|
|
144
|
+
last_row = logits.data[(logits.rows - 1) * V : logits.rows * V]
|
|
145
|
+
next_id = self.model.sample_top_k_top_p(
|
|
146
|
+
last_row, req.temperature, req.top_k, req.top_p
|
|
147
|
+
)
|
|
148
|
+
req.output_tokens.append(next_id)
|
|
149
|
+
req.prefilled = True
|
|
150
|
+
step_outputs[req.request_id] = next_id
|
|
151
|
+
|
|
152
|
+
if (req.stop_token_id is not None and next_id == req.stop_token_id) or len(
|
|
153
|
+
req.output_tokens
|
|
154
|
+
) >= req.max_new_tokens:
|
|
155
|
+
req.is_finished = True
|
|
156
|
+
else:
|
|
157
|
+
decode_candidates.append(req)
|
|
158
|
+
|
|
159
|
+
# 3. Packed continuous decode forward pass for active requests
|
|
160
|
+
if decode_candidates:
|
|
161
|
+
tokens = [req.output_tokens[-1] for req in decode_candidates]
|
|
162
|
+
caches = [req.kv_cache for req in decode_candidates]
|
|
163
|
+
positions = [
|
|
164
|
+
req.kv_cache.seq_len
|
|
165
|
+
if getattr(req.kv_cache, "is_paged", False)
|
|
166
|
+
else req.kv_cache[0]["seq_len"]
|
|
167
|
+
for req in decode_candidates
|
|
168
|
+
]
|
|
169
|
+
|
|
170
|
+
batched_logits = self.model.forward_batched(
|
|
171
|
+
tokens, caches, positions, use_int8=self.use_int8
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
for idx, req in enumerate(decode_candidates):
|
|
175
|
+
row = batched_logits.data[idx * V : (idx + 1) * V]
|
|
176
|
+
next_id = self.model.sample_top_k_top_p(
|
|
177
|
+
row, req.temperature, req.top_k, req.top_p
|
|
178
|
+
)
|
|
179
|
+
req.output_tokens.append(next_id)
|
|
180
|
+
step_outputs[req.request_id] = next_id
|
|
181
|
+
|
|
182
|
+
if (req.stop_token_id is not None and next_id == req.stop_token_id) or len(
|
|
183
|
+
req.output_tokens
|
|
184
|
+
) >= req.max_new_tokens:
|
|
185
|
+
req.is_finished = True
|
|
186
|
+
|
|
187
|
+
# 4. Evict finished requests and free their KV-caches
|
|
188
|
+
remaining_active = []
|
|
189
|
+
for req in self.active_requests:
|
|
190
|
+
if req.is_finished:
|
|
191
|
+
self.completed_requests[req.request_id] = list(req.output_tokens)
|
|
192
|
+
if hasattr(req.kv_cache, "release"):
|
|
193
|
+
req.kv_cache.release()
|
|
194
|
+
else:
|
|
195
|
+
remaining_active.append(req)
|
|
196
|
+
self.active_requests = remaining_active
|
|
197
|
+
|
|
198
|
+
return step_outputs
|
|
199
|
+
|
|
200
|
+
def run_all(self) -> dict[str | int, list[int]]:
|
|
201
|
+
"""Runs the continuous batcher to completion across all enqueued requests."""
|
|
202
|
+
while self.has_work():
|
|
203
|
+
self.step()
|
|
204
|
+
return self.completed_requests
|
|
205
|
+
|
|
206
|
+
def stream(self) -> Generator[dict[str | int, int], None, None]:
|
|
207
|
+
"""Yields step_outputs dict at each decoding iteration until all requests finish."""
|
|
208
|
+
while self.has_work():
|
|
209
|
+
outputs = self.step()
|
|
210
|
+
if outputs:
|
|
211
|
+
yield outputs
|
|
212
|
+
|
|
213
|
+
def stats(self) -> dict:
|
|
214
|
+
"""Returns scheduler queues and active batch telemetry."""
|
|
215
|
+
return {
|
|
216
|
+
"pending_count": len(self.pending_queue),
|
|
217
|
+
"active_count": len(self.active_requests),
|
|
218
|
+
"completed_count": len(self.completed_requests),
|
|
219
|
+
"max_batch_size": self.max_batch_size,
|
|
220
|
+
}
|
thoughtformula/cache.py
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
"""Paged KV-cache memory manager for non-contiguous multi-turn allocation.
|
|
2
|
+
|
|
3
|
+
Implements vLLM-style paged attention memory management:
|
|
4
|
+
- PhysicalBlock: Fixed-size page holding quantized key/value states and scales.
|
|
5
|
+
- BlockAllocator: Global pool for physical block allocation and reference counting.
|
|
6
|
+
- PagedLayerCache: Logical page table mapping sequence positions to physical blocks.
|
|
7
|
+
- PagedKVCache: Multi-layer sequence container supporting O(1) prefix forks and copy-on-write.
|
|
8
|
+
- PagedKVCacheManager: High-level coordinator for multi-turn request allocation.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from array import array
|
|
12
|
+
import math
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class PhysicalBlock:
|
|
16
|
+
"""Fixed-size physical block (page) storing quantized KV token slots."""
|
|
17
|
+
|
|
18
|
+
__slots__ = (
|
|
19
|
+
"block_id",
|
|
20
|
+
"page_size",
|
|
21
|
+
"kv_dim",
|
|
22
|
+
"ref_count",
|
|
23
|
+
"num_tokens",
|
|
24
|
+
"k_int8",
|
|
25
|
+
"v_int8",
|
|
26
|
+
"k_scale",
|
|
27
|
+
"v_scale",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
def __init__(self, block_id: int, page_size: int, kv_dim: int):
|
|
31
|
+
self.block_id = block_id
|
|
32
|
+
self.page_size = page_size
|
|
33
|
+
self.kv_dim = kv_dim
|
|
34
|
+
self.ref_count = 1
|
|
35
|
+
self.num_tokens = 0
|
|
36
|
+
self.k_int8 = array('b', [0] * (page_size * kv_dim))
|
|
37
|
+
self.v_int8 = array('b', [0] * (page_size * kv_dim))
|
|
38
|
+
self.k_scale = array('d', [0.0] * page_size)
|
|
39
|
+
self.v_scale = array('d', [0.0] * page_size)
|
|
40
|
+
|
|
41
|
+
def is_full(self) -> bool:
|
|
42
|
+
return self.num_tokens >= self.page_size
|
|
43
|
+
|
|
44
|
+
def append_slot(self, k_quant, v_quant, sk: float, sv: float):
|
|
45
|
+
"""Appends a single token's KV quant data and scale factors into the block."""
|
|
46
|
+
idx = self.num_tokens
|
|
47
|
+
base = idx * self.kv_dim
|
|
48
|
+
for d in range(self.kv_dim):
|
|
49
|
+
self.k_int8[base + d] = k_quant[d]
|
|
50
|
+
self.v_int8[base + d] = v_quant[d]
|
|
51
|
+
self.k_scale[idx] = sk
|
|
52
|
+
self.v_scale[idx] = sv
|
|
53
|
+
self.num_tokens += 1
|
|
54
|
+
|
|
55
|
+
def copy(self, new_block_id: int) -> "PhysicalBlock":
|
|
56
|
+
"""Creates a deep copy of this physical block with ref_count=1 (Copy-On-Write)."""
|
|
57
|
+
copied = PhysicalBlock(new_block_id, self.page_size, self.kv_dim)
|
|
58
|
+
copied.num_tokens = self.num_tokens
|
|
59
|
+
copied.k_int8 = array('b', self.k_int8)
|
|
60
|
+
copied.v_int8 = array('b', self.v_int8)
|
|
61
|
+
copied.k_scale = array('d', self.k_scale)
|
|
62
|
+
copied.v_scale = array('d', self.v_scale)
|
|
63
|
+
return copied
|
|
64
|
+
|
|
65
|
+
def reset(self):
|
|
66
|
+
"""Resets block state to be reused in the allocator pool."""
|
|
67
|
+
self.ref_count = 1
|
|
68
|
+
self.num_tokens = 0
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class BlockAllocator:
|
|
72
|
+
"""Manages a pool of physical blocks with reference counting and reclamation."""
|
|
73
|
+
|
|
74
|
+
def __init__(self, page_size: int = 16, kv_dim: int = 24, initial_blocks: int = 64):
|
|
75
|
+
self.page_size = page_size
|
|
76
|
+
self.kv_dim = kv_dim
|
|
77
|
+
self.blocks: dict[int, PhysicalBlock] = {}
|
|
78
|
+
self.free_pool: list[int] = []
|
|
79
|
+
self._next_id = 0
|
|
80
|
+
|
|
81
|
+
for _ in range(initial_blocks):
|
|
82
|
+
self._grow()
|
|
83
|
+
|
|
84
|
+
def _grow(self) -> int:
|
|
85
|
+
bid = self._next_id
|
|
86
|
+
self._next_id += 1
|
|
87
|
+
block = PhysicalBlock(bid, self.page_size, self.kv_dim)
|
|
88
|
+
self.blocks[bid] = block
|
|
89
|
+
self.free_pool.append(bid)
|
|
90
|
+
return bid
|
|
91
|
+
|
|
92
|
+
def allocate(self) -> PhysicalBlock:
|
|
93
|
+
"""Allocates a free block, growing the pool if empty."""
|
|
94
|
+
if not self.free_pool:
|
|
95
|
+
self._grow()
|
|
96
|
+
bid = self.free_pool.pop()
|
|
97
|
+
block = self.blocks[bid]
|
|
98
|
+
block.reset()
|
|
99
|
+
return block
|
|
100
|
+
|
|
101
|
+
def retain(self, block_id: int):
|
|
102
|
+
"""Increments block reference count for shared prefix pages."""
|
|
103
|
+
self.blocks[block_id].ref_count += 1
|
|
104
|
+
|
|
105
|
+
def release(self, block_id: int):
|
|
106
|
+
"""Decrements reference count and returns block to free pool if ref_count == 0."""
|
|
107
|
+
block = self.blocks[block_id]
|
|
108
|
+
block.ref_count -= 1
|
|
109
|
+
if block.ref_count <= 0:
|
|
110
|
+
block.ref_count = 0
|
|
111
|
+
self.free_pool.append(block_id)
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def free_blocks_count(self) -> int:
|
|
115
|
+
return len(self.free_pool)
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def total_blocks_count(self) -> int:
|
|
119
|
+
return self._next_id
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def active_blocks_count(self) -> int:
|
|
123
|
+
return self._next_id - len(self.free_pool)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class PagedLayerCache:
|
|
127
|
+
"""Logical page table for one transformer layer of an active sequence."""
|
|
128
|
+
|
|
129
|
+
__slots__ = (
|
|
130
|
+
"layer_idx",
|
|
131
|
+
"page_size",
|
|
132
|
+
"kv_dim",
|
|
133
|
+
"allocator",
|
|
134
|
+
"page_table",
|
|
135
|
+
"seq_len",
|
|
136
|
+
"is_paged",
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def __init__(self, layer_idx: int, page_size: int, kv_dim: int, allocator: BlockAllocator):
|
|
140
|
+
self.layer_idx = layer_idx
|
|
141
|
+
self.page_size = page_size
|
|
142
|
+
self.kv_dim = kv_dim
|
|
143
|
+
self.allocator = allocator
|
|
144
|
+
self.page_table: list[int] = []
|
|
145
|
+
self.seq_len = 0
|
|
146
|
+
self.is_paged = True
|
|
147
|
+
|
|
148
|
+
def get_block(self, page_idx: int) -> PhysicalBlock:
|
|
149
|
+
return self.allocator.blocks[self.page_table[page_idx]]
|
|
150
|
+
|
|
151
|
+
def append_token(self, k_slice, v_slice):
|
|
152
|
+
"""Quantizes token KV to Int8 and appends to the current tail page with CoW support."""
|
|
153
|
+
kv_d = self.kv_dim
|
|
154
|
+
|
|
155
|
+
# Compute int8 dynamic scales
|
|
156
|
+
max_k = 0.0
|
|
157
|
+
max_v = 0.0
|
|
158
|
+
for i in range(kv_d):
|
|
159
|
+
ak = abs(k_slice[i])
|
|
160
|
+
av = abs(v_slice[i])
|
|
161
|
+
if ak > max_k:
|
|
162
|
+
max_k = ak
|
|
163
|
+
if av > max_v:
|
|
164
|
+
max_v = av
|
|
165
|
+
|
|
166
|
+
sk = max_k / 127.0 if max_k > 1e-12 else 1.0
|
|
167
|
+
sv = max_v / 127.0 if max_v > 1e-12 else 1.0
|
|
168
|
+
inv_sk = 127.0 / max_k if max_k > 1e-12 else 0.0
|
|
169
|
+
inv_sv = 127.0 / max_v if max_v > 1e-12 else 0.0
|
|
170
|
+
|
|
171
|
+
k_quant = [max(-127, min(127, int(round(k_slice[i] * inv_sk)))) for i in range(kv_d)]
|
|
172
|
+
v_quant = [max(-127, min(127, int(round(v_slice[i] * inv_sv)))) for i in range(kv_d)]
|
|
173
|
+
|
|
174
|
+
# Find or allocate active tail block
|
|
175
|
+
if not self.page_table or self.get_block(len(self.page_table) - 1).is_full():
|
|
176
|
+
new_block = self.allocator.allocate()
|
|
177
|
+
self.page_table.append(new_block.block_id)
|
|
178
|
+
tail_block = new_block
|
|
179
|
+
else:
|
|
180
|
+
tail_block = self.get_block(len(self.page_table) - 1)
|
|
181
|
+
# Copy-on-Write: If block is shared with another sequence, clone it
|
|
182
|
+
if tail_block.ref_count > 1:
|
|
183
|
+
cloned = tail_block.copy(self.allocator.allocate().block_id)
|
|
184
|
+
self.allocator.release(tail_block.block_id)
|
|
185
|
+
self.page_table[-1] = cloned.block_id
|
|
186
|
+
tail_block = cloned
|
|
187
|
+
|
|
188
|
+
tail_block.append_slot(k_quant, v_quant, sk, sv)
|
|
189
|
+
self.seq_len += 1
|
|
190
|
+
|
|
191
|
+
def truncate(self, target_len: int):
|
|
192
|
+
"""Truncates layer sequence to target_len and reclaims unused trailing blocks."""
|
|
193
|
+
if self.seq_len <= target_len:
|
|
194
|
+
return
|
|
195
|
+
|
|
196
|
+
needed_pages = (target_len + self.page_size - 1) // self.page_size if target_len > 0 else 0
|
|
197
|
+
while len(self.page_table) > needed_pages:
|
|
198
|
+
bid = self.page_table.pop()
|
|
199
|
+
self.allocator.release(bid)
|
|
200
|
+
|
|
201
|
+
if needed_pages > 0:
|
|
202
|
+
tail = self.get_block(needed_pages - 1)
|
|
203
|
+
# Copy on write if shared
|
|
204
|
+
if tail.ref_count > 1:
|
|
205
|
+
cloned = tail.copy(self.allocator.allocate().block_id)
|
|
206
|
+
self.allocator.release(tail.block_id)
|
|
207
|
+
self.page_table[-1] = cloned.block_id
|
|
208
|
+
tail = cloned
|
|
209
|
+
rem = target_len % self.page_size
|
|
210
|
+
tail.num_tokens = rem if rem > 0 else self.page_size
|
|
211
|
+
|
|
212
|
+
self.seq_len = target_len
|
|
213
|
+
|
|
214
|
+
def fork(self) -> "PagedLayerCache":
|
|
215
|
+
"""Forks this layer cache, sharing all physical blocks via reference counting."""
|
|
216
|
+
forked = PagedLayerCache(self.layer_idx, self.page_size, self.kv_dim, self.allocator)
|
|
217
|
+
forked.page_table = list(self.page_table)
|
|
218
|
+
forked.seq_len = self.seq_len
|
|
219
|
+
for bid in self.page_table:
|
|
220
|
+
self.allocator.retain(bid)
|
|
221
|
+
return forked
|
|
222
|
+
|
|
223
|
+
def release(self):
|
|
224
|
+
"""Releases all blocks allocated by this layer cache."""
|
|
225
|
+
for bid in self.page_table:
|
|
226
|
+
self.allocator.release(bid)
|
|
227
|
+
self.page_table.clear()
|
|
228
|
+
self.seq_len = 0
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class PagedKVCache:
|
|
232
|
+
"""Full multi-layer paged KV-cache container for a single sequence."""
|
|
233
|
+
|
|
234
|
+
__slots__ = ("n_layers", "page_size", "kv_dim", "allocator", "layers", "is_paged")
|
|
235
|
+
|
|
236
|
+
def __init__(self, n_layers: int, page_size: int, kv_dim: int, allocator: BlockAllocator):
|
|
237
|
+
self.n_layers = n_layers
|
|
238
|
+
self.page_size = page_size
|
|
239
|
+
self.kv_dim = kv_dim
|
|
240
|
+
self.allocator = allocator
|
|
241
|
+
self.layers = [
|
|
242
|
+
PagedLayerCache(l, page_size, kv_dim, allocator) for l in range(n_layers)
|
|
243
|
+
]
|
|
244
|
+
self.is_paged = True
|
|
245
|
+
|
|
246
|
+
@property
|
|
247
|
+
def seq_len(self) -> int:
|
|
248
|
+
return self.layers[0].seq_len if self.layers else 0
|
|
249
|
+
|
|
250
|
+
def __getitem__(self, idx: int) -> PagedLayerCache:
|
|
251
|
+
return self.layers[idx]
|
|
252
|
+
|
|
253
|
+
def __len__(self) -> int:
|
|
254
|
+
return self.n_layers
|
|
255
|
+
|
|
256
|
+
def fork(self) -> "PagedKVCache":
|
|
257
|
+
"""O(1) prefix sharing: forks the entire multi-layer cache without copying buffers."""
|
|
258
|
+
forked = PagedKVCache(self.n_layers, self.page_size, self.kv_dim, self.allocator)
|
|
259
|
+
forked.layers = [layer.fork() for layer in self.layers]
|
|
260
|
+
return forked
|
|
261
|
+
|
|
262
|
+
def truncate(self, target_len: int):
|
|
263
|
+
"""Truncates all layer caches to target_len."""
|
|
264
|
+
for layer in self.layers:
|
|
265
|
+
layer.truncate(target_len)
|
|
266
|
+
|
|
267
|
+
def release(self):
|
|
268
|
+
"""Releases all physical memory held by this cache."""
|
|
269
|
+
for layer in self.layers:
|
|
270
|
+
layer.release()
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
class PagedKVCacheManager:
|
|
274
|
+
"""High-level coordinator managing block allocation and sequence caches."""
|
|
275
|
+
|
|
276
|
+
def __init__(self, n_layers: int, kv_dim: int, page_size: int = 16, initial_blocks: int = 64):
|
|
277
|
+
self.n_layers = n_layers
|
|
278
|
+
self.kv_dim = kv_dim
|
|
279
|
+
self.page_size = page_size
|
|
280
|
+
self.allocator = BlockAllocator(page_size=page_size, kv_dim=kv_dim, initial_blocks=initial_blocks)
|
|
281
|
+
self.active_caches: list[PagedKVCache] = []
|
|
282
|
+
|
|
283
|
+
def create_cache(self) -> PagedKVCache:
|
|
284
|
+
cache = PagedKVCache(self.n_layers, self.page_size, self.kv_dim, self.allocator)
|
|
285
|
+
self.active_caches.append(cache)
|
|
286
|
+
return cache
|
|
287
|
+
|
|
288
|
+
def fork_cache(self, cache: PagedKVCache) -> PagedKVCache:
|
|
289
|
+
forked = cache.fork()
|
|
290
|
+
self.active_caches.append(forked)
|
|
291
|
+
return forked
|
|
292
|
+
|
|
293
|
+
def free_cache(self, cache: PagedKVCache):
|
|
294
|
+
if cache in self.active_caches:
|
|
295
|
+
self.active_caches.remove(cache)
|
|
296
|
+
cache.release()
|
|
297
|
+
|
|
298
|
+
def stats(self) -> dict:
|
|
299
|
+
"""Returns memory manager statistics including prefix sharing deduplication."""
|
|
300
|
+
total_tokens = sum(c.seq_len for c in self.active_caches)
|
|
301
|
+
unpaged_blocks_needed = sum(
|
|
302
|
+
math.ceil(c.seq_len / self.page_size) * self.n_layers for c in self.active_caches
|
|
303
|
+
)
|
|
304
|
+
active_blocks = self.allocator.active_blocks_count
|
|
305
|
+
saved_blocks = max(0, unpaged_blocks_needed - active_blocks)
|
|
306
|
+
sharing_ratio = (
|
|
307
|
+
unpaged_blocks_needed / active_blocks if active_blocks > 0 else 1.0
|
|
308
|
+
)
|
|
309
|
+
return {
|
|
310
|
+
"page_size": self.page_size,
|
|
311
|
+
"total_sequences": len(self.active_caches),
|
|
312
|
+
"total_tokens": total_tokens,
|
|
313
|
+
"total_blocks": self.allocator.total_blocks_count,
|
|
314
|
+
"free_blocks": self.allocator.free_blocks_count,
|
|
315
|
+
"active_blocks": active_blocks,
|
|
316
|
+
"saved_blocks_via_sharing": saved_blocks,
|
|
317
|
+
"sharing_ratio": sharing_ratio,
|
|
318
|
+
}
|