dag-kv 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dag_kv/__init__.py +49 -0
- dag_kv/cli.py +204 -0
- dag_kv/core/__init__.py +45 -0
- dag_kv/core/block_manager.py +312 -0
- dag_kv/core/lse_probe.py +147 -0
- dag_kv/core/quantizer.py +89 -0
- dag_kv/core/rope_adjoint.py +159 -0
- dag_kv/core/sink_canonicalizer.py +156 -0
- dag_kv/devices/__init__.py +13 -0
- dag_kv/devices/device_manager.py +268 -0
- dag_kv/devices/shard_config.py +199 -0
- dag_kv/integrations/__init__.py +11 -0
- dag_kv/integrations/huggingface.py +145 -0
- dag_kv/integrations/vllm.py +90 -0
- dag_kv/kernels/__init__.py +35 -0
- dag_kv/kernels/dispatch.py +152 -0
- dag_kv/kernels/pytorch_fallbacks.py +282 -0
- dag_kv/kernels/triton_adjoint_paged_attn.py +268 -0
- dag_kv/kernels/triton_sparse_stitch.py +259 -0
- dag_kv/models/__init__.py +5 -0
- dag_kv/models/qwen2_dag_patch.py +571 -0
- dag_kv/server.py +175 -0
- dag_kv/worker/__init__.py +10 -0
- dag_kv/worker/dual_gpu_pipeline.py +287 -0
- dag_kv/worker/universal_pipeline.py +232 -0
- dag_kv-0.2.0.dist-info/METADATA +351 -0
- dag_kv-0.2.0.dist-info/RECORD +31 -0
- dag_kv-0.2.0.dist-info/WHEEL +5 -0
- dag_kv-0.2.0.dist-info/entry_points.txt +2 -0
- dag_kv-0.2.0.dist-info/licenses/LICENSE +17 -0
- dag_kv-0.2.0.dist-info/top_level.txt +1 -0
dag_kv/__init__.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""DAG-KV: Position-Decoupled Graph KV Cache with Query-Side Adjoint RoPE & LSE-Bounded Sparse Bridge Repair."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.2.0"
|
|
4
|
+
|
|
5
|
+
from .core.block_manager import BlockManager, OffsetBlockTable, LogicalBlock, PhysicalBlock
|
|
6
|
+
from .core.lse_probe import LSEProbe, compute_lse_perturbation, select_bridge_tokens
|
|
7
|
+
from .core.quantizer import KVQuantizer
|
|
8
|
+
from .core.rope_adjoint import apply_adjoint_rope, apply_rope_half, compute_rope_angles
|
|
9
|
+
from .core.sink_canonicalizer import CanonicalChunk, SinkCanonicalizer
|
|
10
|
+
from .devices.device_manager import DeviceInfo, DeviceManager, get_device_manager
|
|
11
|
+
from .devices.shard_config import QuantizationMode, ShardConfig, TopologyMode
|
|
12
|
+
from .integrations.huggingface import DAGKVModelWrapper, patch_hf_model
|
|
13
|
+
from .integrations.vllm import DAGKVvLLMBlockManagerPlugin
|
|
14
|
+
from .kernels.dispatch import adjoint_paged_attention, is_triton_available, sparse_bridge_attention
|
|
15
|
+
from .models.qwen2_dag_patch import Qwen2DAGKVEngine
|
|
16
|
+
from .worker.dual_gpu_pipeline import DualGPUPipeline
|
|
17
|
+
from .worker.universal_pipeline import UniversalPipeline
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"__version__",
|
|
21
|
+
"BlockManager",
|
|
22
|
+
"OffsetBlockTable",
|
|
23
|
+
"LogicalBlock",
|
|
24
|
+
"PhysicalBlock",
|
|
25
|
+
"LSEProbe",
|
|
26
|
+
"compute_lse_perturbation",
|
|
27
|
+
"select_bridge_tokens",
|
|
28
|
+
"KVQuantizer",
|
|
29
|
+
"apply_adjoint_rope",
|
|
30
|
+
"apply_rope_half",
|
|
31
|
+
"compute_rope_angles",
|
|
32
|
+
"CanonicalChunk",
|
|
33
|
+
"SinkCanonicalizer",
|
|
34
|
+
"DeviceInfo",
|
|
35
|
+
"DeviceManager",
|
|
36
|
+
"get_device_manager",
|
|
37
|
+
"QuantizationMode",
|
|
38
|
+
"ShardConfig",
|
|
39
|
+
"TopologyMode",
|
|
40
|
+
"DAGKVModelWrapper",
|
|
41
|
+
"patch_hf_model",
|
|
42
|
+
"DAGKVvLLMBlockManagerPlugin",
|
|
43
|
+
"adjoint_paged_attention",
|
|
44
|
+
"sparse_bridge_attention",
|
|
45
|
+
"is_triton_available",
|
|
46
|
+
"Qwen2DAGKVEngine",
|
|
47
|
+
"DualGPUPipeline",
|
|
48
|
+
"UniversalPipeline",
|
|
49
|
+
]
|
dag_kv/cli.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""DAG-KV Command Line Interface (CLI).
|
|
2
|
+
|
|
3
|
+
Commands:
|
|
4
|
+
- `dag-kv info`: Inspects hardware topology, VRAM, and recommended configuration.
|
|
5
|
+
- `dag-kv serve`: Launches OpenAI-compatible /v1/chat/completions server.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
import threading
|
|
13
|
+
import time
|
|
14
|
+
import uuid
|
|
15
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
16
|
+
import torch
|
|
17
|
+
|
|
18
|
+
from dag_kv.devices.device_manager import get_device_manager
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def cmd_info(args):
|
|
22
|
+
"""Outputs hardware probe results and optimal topology recommendations."""
|
|
23
|
+
mgr = get_device_manager()
|
|
24
|
+
print("=" * 60)
|
|
25
|
+
print(" DAG-KV Hardware Abstraction Layer (HAL) Report")
|
|
26
|
+
print("=" * 60)
|
|
27
|
+
print(f"CUDA Available: {torch.cuda.is_available()}")
|
|
28
|
+
print(f"CUDA Device Count: {mgr.cuda_device_count}")
|
|
29
|
+
print(f"Apple Silicon MPS: {mgr.has_mps}")
|
|
30
|
+
print(f"Triton JIT Available: {mgr._has_triton}")
|
|
31
|
+
print(f"Primary Compute Dev: {mgr.primary_device}")
|
|
32
|
+
print("-" * 60)
|
|
33
|
+
print("Discovered Devices:")
|
|
34
|
+
for idx, d in enumerate(mgr.devices):
|
|
35
|
+
cc_str = f"sm_{d.compute_capability[0]}{d.compute_capability[1]}" if d.compute_capability else "N/A"
|
|
36
|
+
print(f" [{idx}] {d.device_type.upper():<4} | {d.name:<32} | Total: {d.total_memory_gb:.2f} GB | Free: {d.free_memory_gb:.2f} GB | CC: {cc_str}")
|
|
37
|
+
|
|
38
|
+
print("-" * 60)
|
|
39
|
+
auto_config = mgr.auto_detect_topology()
|
|
40
|
+
print("Recommended Topology:")
|
|
41
|
+
print(f" Mode: {auto_config.topology.value}")
|
|
42
|
+
print(f" Compute Dev: {auto_config.device}")
|
|
43
|
+
print(f" KV Device: {auto_config.kv_device}")
|
|
44
|
+
print(f" Dtype: {auto_config.dtype}")
|
|
45
|
+
print(f" Block Size: {auto_config.block_size}")
|
|
46
|
+
print("=" * 60)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def cmd_serve(args):
|
|
50
|
+
"""Launches the universal OpenAI-compatible HTTP server."""
|
|
51
|
+
from dag_kv.integrations.huggingface import DAGKVModelWrapper
|
|
52
|
+
|
|
53
|
+
print(f"Starting DAG-KV HTTP Server on {args.host}:{args.port}")
|
|
54
|
+
print(f"Model: {args.model} | Device: {args.device} | Quantization: {args.quantization}")
|
|
55
|
+
|
|
56
|
+
class UniversalService:
|
|
57
|
+
def __init__(self, model_name: str, device: str, quantization: str, max_blocks: int, block_size: int):
|
|
58
|
+
self.model = DAGKVModelWrapper.from_pretrained(
|
|
59
|
+
model_name_or_path=model_name,
|
|
60
|
+
device=device,
|
|
61
|
+
quantization=quantization,
|
|
62
|
+
max_num_blocks=max_blocks,
|
|
63
|
+
block_size=block_size,
|
|
64
|
+
)
|
|
65
|
+
self.chunk_cache = {}
|
|
66
|
+
self.lock = threading.RLock()
|
|
67
|
+
self.total_requests = 0
|
|
68
|
+
self.total_file_hits = 0
|
|
69
|
+
|
|
70
|
+
def _parse_files_and_query(self, messages):
|
|
71
|
+
full_text = "\n".join(m.get("content", "") for m in messages)
|
|
72
|
+
if "### FILE:" not in full_text:
|
|
73
|
+
return [], full_text
|
|
74
|
+
parts = re.split(r"(?=### FILE: )", full_text)
|
|
75
|
+
file_segments = [p.strip() + "\n\n" for p in parts if p.startswith("### FILE: ")]
|
|
76
|
+
non_file = [p for p in parts if not p.startswith("### FILE: ")]
|
|
77
|
+
query_str = "\n".join(non_file).strip()
|
|
78
|
+
if file_segments and "\nQuestion:" in file_segments[-1]:
|
|
79
|
+
last_file, q_tail = file_segments[-1].rsplit("\nQuestion:", 1)
|
|
80
|
+
file_segments[-1] = last_file.strip() + "\n\n"
|
|
81
|
+
query_str = "\nQuestion:" + q_tail
|
|
82
|
+
return file_segments, query_str
|
|
83
|
+
|
|
84
|
+
def chat_completion(self, payload):
|
|
85
|
+
t0 = time.perf_counter()
|
|
86
|
+
self.total_requests += 1
|
|
87
|
+
messages = payload.get("messages", [])
|
|
88
|
+
max_tokens = int(payload.get("max_tokens", 32))
|
|
89
|
+
file_segments, query_str = self._parse_files_and_query(messages)
|
|
90
|
+
|
|
91
|
+
active_chunks = []
|
|
92
|
+
hits = 0
|
|
93
|
+
with self.lock:
|
|
94
|
+
for seg in file_segments:
|
|
95
|
+
h_key = hashlib.sha256(seg.encode("utf-8")).hexdigest()[:16]
|
|
96
|
+
if h_key in self.chunk_cache:
|
|
97
|
+
active_chunks.append(self.chunk_cache[h_key])
|
|
98
|
+
hits += 1
|
|
99
|
+
self.total_file_hits += 1
|
|
100
|
+
else:
|
|
101
|
+
chunk = self.model.prefill_chunk(seg)
|
|
102
|
+
self.chunk_cache[h_key] = chunk
|
|
103
|
+
active_chunks.append(chunk)
|
|
104
|
+
|
|
105
|
+
if not active_chunks:
|
|
106
|
+
chunk = self.model.prefill_chunk(query_str or "Hello")
|
|
107
|
+
active_chunks = [chunk]
|
|
108
|
+
query_str = ""
|
|
109
|
+
|
|
110
|
+
answer = self.model.generate(
|
|
111
|
+
chunks=active_chunks,
|
|
112
|
+
prompt_suffix=query_str,
|
|
113
|
+
max_new_tokens=max_tokens,
|
|
114
|
+
).strip()
|
|
115
|
+
total_ms = round((time.perf_counter() - t0) * 1000.0, 1)
|
|
116
|
+
|
|
117
|
+
return {
|
|
118
|
+
"id": f"chatcmpl-{uuid.uuid4().hex[:10]}",
|
|
119
|
+
"object": "chat.completion",
|
|
120
|
+
"created": int(time.time()),
|
|
121
|
+
"model": args.model,
|
|
122
|
+
"choices": [{"index": 0, "message": {"role": "assistant", "content": answer}, "finish_reason": "stop"}],
|
|
123
|
+
"usage": {"prompt_tokens": len(query_str.split()), "completion_tokens": len(answer.split()), "total_tokens": len(query_str.split()) + len(answer.split())},
|
|
124
|
+
"dag_kv_telemetry": {
|
|
125
|
+
"files_attached": len(file_segments),
|
|
126
|
+
"canonical_file_hits": hits,
|
|
127
|
+
"total_latency_ms": total_ms,
|
|
128
|
+
},
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
service = UniversalService(
|
|
132
|
+
model_name=args.model,
|
|
133
|
+
device=args.device,
|
|
134
|
+
quantization=args.quantization,
|
|
135
|
+
max_blocks=args.max_blocks,
|
|
136
|
+
block_size=args.block_size,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
class RequestHandler(BaseHTTPRequestHandler):
|
|
140
|
+
def do_POST(self):
|
|
141
|
+
if self.path == "/v1/chat/completions":
|
|
142
|
+
length = int(self.headers.get("Content-Length", 0))
|
|
143
|
+
body = self.rfile.read(length)
|
|
144
|
+
try:
|
|
145
|
+
payload = json.loads(body.decode("utf-8"))
|
|
146
|
+
resp = service.chat_completion(payload)
|
|
147
|
+
self.send_response(200)
|
|
148
|
+
self.send_header("Content-Type", "application/json")
|
|
149
|
+
self.end_headers()
|
|
150
|
+
self.wfile.write(json.dumps(resp).encode("utf-8"))
|
|
151
|
+
except Exception as e:
|
|
152
|
+
self.send_response(500)
|
|
153
|
+
self.end_headers()
|
|
154
|
+
self.wfile.write(str(e).encode("utf-8"))
|
|
155
|
+
else:
|
|
156
|
+
self.send_response(404)
|
|
157
|
+
self.end_headers()
|
|
158
|
+
|
|
159
|
+
def do_GET(self):
|
|
160
|
+
if self.path in ("/health", "/v1/health"):
|
|
161
|
+
self.send_response(200)
|
|
162
|
+
self.send_header("Content-Type", "application/json")
|
|
163
|
+
self.end_headers()
|
|
164
|
+
self.wfile.write(json.dumps({"status": "ok", "engine": "dag-kv"}).encode("utf-8"))
|
|
165
|
+
else:
|
|
166
|
+
self.send_response(404)
|
|
167
|
+
self.end_headers()
|
|
168
|
+
|
|
169
|
+
server = ThreadingHTTPServer((args.host, args.port), RequestHandler)
|
|
170
|
+
print(f"DAG-KV Server listening on http://{args.host}:{args.port}")
|
|
171
|
+
try:
|
|
172
|
+
server.serve_forever()
|
|
173
|
+
except KeyboardInterrupt:
|
|
174
|
+
print("\nShutting down server...")
|
|
175
|
+
server.shutdown()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def main():
|
|
179
|
+
parser = argparse.ArgumentParser(prog="dag-kv", description="DAG-KV Position-Decoupled KV Cache Inference Engine")
|
|
180
|
+
subparsers = parser.add_subparsers(dest="command", help="Available commands")
|
|
181
|
+
|
|
182
|
+
subparsers.add_parser("info", help="Display hardware topology and device discovery")
|
|
183
|
+
|
|
184
|
+
p_serve = subparsers.add_parser("serve", help="Run OpenAI-compatible HTTP inference server")
|
|
185
|
+
p_serve.add_argument("--host", type=str, default="0.0.0.0", help="Host interface to bind")
|
|
186
|
+
p_serve.add_argument("--port", type=int, default=8000, help="Port to bind")
|
|
187
|
+
p_serve.add_argument("--model", type=str, default="Qwen/Qwen2.5-3B-Instruct", help="HuggingFace model ID")
|
|
188
|
+
p_serve.add_argument("--device", type=str, default="auto", help="Compute device ('auto', 'cuda:0', 'mps', 'cpu')")
|
|
189
|
+
p_serve.add_argument("--quantization", type=str, default="none", choices=["none", "int8", "int4"], help="KV cache quantization")
|
|
190
|
+
p_serve.add_argument("--max-blocks", type=int, default=2048, help="Maximum physical blocks in KV pool")
|
|
191
|
+
p_serve.add_argument("--block-size", type=int, default=16, help="Tokens per physical block (8, 16, 32, 64)")
|
|
192
|
+
|
|
193
|
+
args = parser.parse_args()
|
|
194
|
+
|
|
195
|
+
if args.command == "info" or args.command is None:
|
|
196
|
+
cmd_info(args)
|
|
197
|
+
elif args.command == "serve":
|
|
198
|
+
cmd_serve(args)
|
|
199
|
+
else:
|
|
200
|
+
parser.print_help()
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
if __name__ == "__main__":
|
|
204
|
+
main()
|
dag_kv/core/__init__.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Core mathematical and structural components of DAG-KV."""
|
|
2
|
+
|
|
3
|
+
from .rope_adjoint import (
|
|
4
|
+
apply_rope_half,
|
|
5
|
+
apply_adjoint_rope,
|
|
6
|
+
precompute_freqs_cis,
|
|
7
|
+
compute_rope_angles,
|
|
8
|
+
verify_adjoint_identity,
|
|
9
|
+
)
|
|
10
|
+
from .sink_canonicalizer import (
|
|
11
|
+
CanonicalChunk,
|
|
12
|
+
SinkCanonicalizer,
|
|
13
|
+
)
|
|
14
|
+
from .lse_probe import (
|
|
15
|
+
LSEProbe,
|
|
16
|
+
compute_lse_perturbation,
|
|
17
|
+
select_bridge_tokens,
|
|
18
|
+
)
|
|
19
|
+
from .block_manager import (
|
|
20
|
+
BlockManager,
|
|
21
|
+
OffsetBlockTable,
|
|
22
|
+
LogicalBlock,
|
|
23
|
+
PhysicalBlock,
|
|
24
|
+
)
|
|
25
|
+
from .quantizer import (
|
|
26
|
+
KVQuantizer,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"apply_rope_half",
|
|
31
|
+
"apply_adjoint_rope",
|
|
32
|
+
"precompute_freqs_cis",
|
|
33
|
+
"compute_rope_angles",
|
|
34
|
+
"verify_adjoint_identity",
|
|
35
|
+
"CanonicalChunk",
|
|
36
|
+
"SinkCanonicalizer",
|
|
37
|
+
"LSEProbe",
|
|
38
|
+
"compute_lse_perturbation",
|
|
39
|
+
"select_bridge_tokens",
|
|
40
|
+
"BlockManager",
|
|
41
|
+
"OffsetBlockTable",
|
|
42
|
+
"LogicalBlock",
|
|
43
|
+
"PhysicalBlock",
|
|
44
|
+
"KVQuantizer",
|
|
45
|
+
]
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""Paged Block Manager, Content-SHA256 Registry & Adaptive Memory Pools.
|
|
2
|
+
|
|
3
|
+
Manages physical KV cache memory across hardware platforms:
|
|
4
|
+
1. Physical memory pool allocated upfront across CUDA, Apple Silicon MPS, or Host RAM (CPU).
|
|
5
|
+
2. Content-addressable SHA256 storage for canonical chunks.
|
|
6
|
+
3. Logical-to-Physical OffsetBlockTable mapping with per-block RoPE Delta.
|
|
7
|
+
4. Copy-on-Write (CoW) Shadow Blocks for request-private Bridge Token updates.
|
|
8
|
+
5. Dynamic Block Sizes (8, 16, 32, 64) and Runtime Quantization (FP16, INT8, INT4).
|
|
9
|
+
6. Pinned Host Memory for CPU-Offloaded KV Pooling.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import Dict, List, Optional, Set, Tuple, Union
|
|
14
|
+
import torch
|
|
15
|
+
|
|
16
|
+
from .quantizer import KVQuantizer
|
|
17
|
+
from .sink_canonicalizer import CanonicalChunk
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class LogicalBlock:
|
|
22
|
+
"""Logical block descriptor in a sequence's block table."""
|
|
23
|
+
logical_idx: int
|
|
24
|
+
physical_idx: int
|
|
25
|
+
delta: int # Delta = global_start - canonical_pos_start (e.g. G - 4)
|
|
26
|
+
is_cow_shadow: bool = False
|
|
27
|
+
valid_tokens: int = 16
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class PhysicalBlock:
|
|
32
|
+
"""Metadata for a physical block in the pool."""
|
|
33
|
+
block_id: int
|
|
34
|
+
ref_count: int = 0
|
|
35
|
+
is_canonical: bool = False
|
|
36
|
+
chunk_id: Optional[str] = None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class OffsetBlockTable:
|
|
40
|
+
"""Per-sequence block table maintaining physical mappings and RoPE offsets."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, seq_id: str, block_size: int = 16):
|
|
43
|
+
self.seq_id = seq_id
|
|
44
|
+
self.block_size = block_size
|
|
45
|
+
self.blocks: List[LogicalBlock] = []
|
|
46
|
+
|
|
47
|
+
def append_block(
|
|
48
|
+
self,
|
|
49
|
+
physical_idx: int,
|
|
50
|
+
delta: int,
|
|
51
|
+
is_cow_shadow: bool = False,
|
|
52
|
+
valid_tokens: Optional[int] = None,
|
|
53
|
+
):
|
|
54
|
+
logical_idx = len(self.blocks)
|
|
55
|
+
v_tokens = self.block_size if valid_tokens is None else valid_tokens
|
|
56
|
+
self.blocks.append(
|
|
57
|
+
LogicalBlock(
|
|
58
|
+
logical_idx=logical_idx,
|
|
59
|
+
physical_idx=physical_idx,
|
|
60
|
+
delta=delta,
|
|
61
|
+
is_cow_shadow=is_cow_shadow,
|
|
62
|
+
valid_tokens=v_tokens,
|
|
63
|
+
)
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def get_physical_indices(self, device: Optional[torch.device] = None) -> torch.Tensor:
|
|
67
|
+
"""Returns 1D tensor of physical block IDs."""
|
|
68
|
+
return torch.tensor([b.physical_idx for b in self.blocks], dtype=torch.int32, device=device)
|
|
69
|
+
|
|
70
|
+
def get_deltas(self, device: Optional[torch.device] = None) -> torch.Tensor:
|
|
71
|
+
"""Returns 1D tensor of integer RoPE offsets Delta."""
|
|
72
|
+
return torch.tensor([b.delta for b in self.blocks], dtype=torch.int32, device=device)
|
|
73
|
+
|
|
74
|
+
def get_valid_tokens(self, device: Optional[torch.device] = None) -> torch.Tensor:
|
|
75
|
+
"""Returns 1D tensor of valid token counts per logical block."""
|
|
76
|
+
return torch.tensor([b.valid_tokens for b in self.blocks], dtype=torch.int32, device=device)
|
|
77
|
+
|
|
78
|
+
def get_block_starts(self, device: Optional[torch.device] = None) -> torch.Tensor:
|
|
79
|
+
"""Returns 1D tensor of cumulative starting global positions for each logical block."""
|
|
80
|
+
starts = []
|
|
81
|
+
cum = 0
|
|
82
|
+
for b in self.blocks:
|
|
83
|
+
starts.append(cum)
|
|
84
|
+
cum += b.valid_tokens
|
|
85
|
+
return torch.tensor(starts, dtype=torch.int32, device=device)
|
|
86
|
+
|
|
87
|
+
def get_chunk_groups(self) -> List[Tuple[int, List[LogicalBlock]]]:
|
|
88
|
+
"""Groups contiguous blocks with identical delta (representing a chunk or decode tail)."""
|
|
89
|
+
if not self.blocks:
|
|
90
|
+
return []
|
|
91
|
+
groups = []
|
|
92
|
+
cur_delta = self.blocks[0].delta
|
|
93
|
+
cur_blocks = [self.blocks[0]]
|
|
94
|
+
for b in self.blocks[1:]:
|
|
95
|
+
if b.delta == cur_delta:
|
|
96
|
+
cur_blocks.append(b)
|
|
97
|
+
else:
|
|
98
|
+
groups.append((cur_delta, cur_blocks))
|
|
99
|
+
cur_delta = b.delta
|
|
100
|
+
cur_blocks = [b]
|
|
101
|
+
groups.append((cur_delta, cur_blocks))
|
|
102
|
+
return groups
|
|
103
|
+
|
|
104
|
+
def __len__(self) -> int:
|
|
105
|
+
return len(self.blocks)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class BlockManager:
|
|
109
|
+
"""Central Paged KV Cache Memory Manager supporting Adjoint Offsets and CoW."""
|
|
110
|
+
|
|
111
|
+
def __init__(
|
|
112
|
+
self,
|
|
113
|
+
num_layers: int,
|
|
114
|
+
num_kv_heads: int,
|
|
115
|
+
head_dim: int,
|
|
116
|
+
max_num_blocks: int = 4096,
|
|
117
|
+
block_size: int = 16,
|
|
118
|
+
device: Union[str, torch.device] = "cuda:0",
|
|
119
|
+
dtype: Optional[torch.dtype] = None,
|
|
120
|
+
quantization: str = "none",
|
|
121
|
+
pin_memory: bool = False,
|
|
122
|
+
):
|
|
123
|
+
self.num_layers = num_layers
|
|
124
|
+
self.num_kv_heads = num_kv_heads
|
|
125
|
+
self.head_dim = head_dim
|
|
126
|
+
self.max_num_blocks = max_num_blocks
|
|
127
|
+
self.block_size = block_size
|
|
128
|
+
self.device = torch.device(device)
|
|
129
|
+
self.quantization = quantization.lower()
|
|
130
|
+
self.pin_memory = pin_memory
|
|
131
|
+
|
|
132
|
+
# Determine optimal computation and storage dtype
|
|
133
|
+
if dtype is not None:
|
|
134
|
+
self.dtype = dtype
|
|
135
|
+
else:
|
|
136
|
+
if self.device.type == "cuda":
|
|
137
|
+
self.dtype = torch.float16
|
|
138
|
+
elif self.device.type == "mps":
|
|
139
|
+
self.dtype = torch.float32
|
|
140
|
+
else:
|
|
141
|
+
self.dtype = torch.float32
|
|
142
|
+
|
|
143
|
+
self.storage_dtype = torch.int8 if self.quantization == "int8" else self.dtype
|
|
144
|
+
|
|
145
|
+
# Allocate physical memory pool
|
|
146
|
+
# Shape: (max_num_blocks, num_layers, num_kv_heads, block_size, head_dim)
|
|
147
|
+
pool_shape = (max_num_blocks, num_layers, num_kv_heads, block_size, head_dim)
|
|
148
|
+
|
|
149
|
+
if self.device.type == "cpu" and self.pin_memory and torch.cuda.is_available():
|
|
150
|
+
self.k_pool = torch.zeros(pool_shape, dtype=self.storage_dtype, device="cpu").pin_memory()
|
|
151
|
+
self.v_pool = torch.zeros(pool_shape, dtype=self.storage_dtype, device="cpu").pin_memory()
|
|
152
|
+
else:
|
|
153
|
+
self.k_pool = torch.zeros(pool_shape, dtype=self.storage_dtype, device=self.device)
|
|
154
|
+
self.v_pool = torch.zeros(pool_shape, dtype=self.storage_dtype, device=self.device)
|
|
155
|
+
|
|
156
|
+
# Quantization scale pools for INT8
|
|
157
|
+
if self.quantization == "int8":
|
|
158
|
+
scale_shape = (max_num_blocks, num_layers, num_kv_heads, block_size, 1)
|
|
159
|
+
scale_dtype = torch.float32 if self.device.type == "cpu" else torch.float16
|
|
160
|
+
self.k_scale_pool = torch.ones(scale_shape, dtype=scale_dtype, device=self.device)
|
|
161
|
+
self.v_scale_pool = torch.ones(scale_shape, dtype=scale_dtype, device=self.device)
|
|
162
|
+
else:
|
|
163
|
+
self.k_scale_pool = None
|
|
164
|
+
self.v_scale_pool = None
|
|
165
|
+
|
|
166
|
+
# Free list & metadata tracking
|
|
167
|
+
self.free_blocks: List[int] = list(range(max_num_blocks))
|
|
168
|
+
self.physical_metadata: Dict[int, PhysicalBlock] = {
|
|
169
|
+
i: PhysicalBlock(block_id=i) for i in range(max_num_blocks)
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
# Canonical Content Registry: chunk_id (SHA256) -> list of physical block IDs
|
|
173
|
+
self.canonical_registry: Dict[str, List[int]] = {}
|
|
174
|
+
self.canonical_lengths: Dict[str, int] = {}
|
|
175
|
+
|
|
176
|
+
def allocate_block(self, is_canonical: bool = False, chunk_id: Optional[str] = None) -> int:
|
|
177
|
+
"""Allocates a single physical block."""
|
|
178
|
+
if not self.free_blocks:
|
|
179
|
+
raise MemoryError(f"DAG-KV BlockManager out of memory: {self.max_num_blocks} blocks exhausted")
|
|
180
|
+
block_id = self.free_blocks.pop(0)
|
|
181
|
+
meta = self.physical_metadata[block_id]
|
|
182
|
+
meta.ref_count = 1
|
|
183
|
+
meta.is_canonical = is_canonical
|
|
184
|
+
meta.chunk_id = chunk_id
|
|
185
|
+
return block_id
|
|
186
|
+
|
|
187
|
+
def free_block(self, block_id: int):
|
|
188
|
+
"""Decrements ref_count and returns to free list if ref_count reaches 0."""
|
|
189
|
+
meta = self.physical_metadata[block_id]
|
|
190
|
+
meta.ref_count -= 1
|
|
191
|
+
if meta.ref_count <= 0:
|
|
192
|
+
meta.ref_count = 0
|
|
193
|
+
meta.is_canonical = False
|
|
194
|
+
meta.chunk_id = None
|
|
195
|
+
self.free_blocks.append(block_id)
|
|
196
|
+
|
|
197
|
+
def register_canonical_chunk(self, chunk: CanonicalChunk) -> List[int]:
|
|
198
|
+
"""Loads a CanonicalChunk into read-only physical pool blocks.
|
|
199
|
+
|
|
200
|
+
Deduplicates via SHA256: if already cached, increments ref count.
|
|
201
|
+
"""
|
|
202
|
+
if chunk.chunk_id in self.canonical_registry:
|
|
203
|
+
block_ids = self.canonical_registry[chunk.chunk_id]
|
|
204
|
+
for b_id in block_ids:
|
|
205
|
+
self.physical_metadata[b_id].ref_count += 1
|
|
206
|
+
return block_ids
|
|
207
|
+
|
|
208
|
+
length = chunk.length
|
|
209
|
+
num_blocks = (length + self.block_size - 1) // self.block_size
|
|
210
|
+
allocated_blocks: List[int] = []
|
|
211
|
+
|
|
212
|
+
k_cache = chunk.k_cache.to(self.device, dtype=self.dtype)
|
|
213
|
+
v_cache = chunk.v_cache.to(self.device, dtype=self.dtype)
|
|
214
|
+
|
|
215
|
+
# Handle INT8 quantization during chunk registration
|
|
216
|
+
if self.quantization == "int8":
|
|
217
|
+
k_storage, k_scale = KVQuantizer.quantize_int8(k_cache, dim=-1)
|
|
218
|
+
v_storage, v_scale = KVQuantizer.quantize_int8(v_cache, dim=-1)
|
|
219
|
+
else:
|
|
220
|
+
k_storage = k_cache
|
|
221
|
+
v_storage = v_cache
|
|
222
|
+
k_scale = None
|
|
223
|
+
v_scale = None
|
|
224
|
+
|
|
225
|
+
for b_idx in range(num_blocks):
|
|
226
|
+
start = b_idx * self.block_size
|
|
227
|
+
end = min(start + self.block_size, length)
|
|
228
|
+
block_len = end - start
|
|
229
|
+
|
|
230
|
+
phys_id = self.allocate_block(is_canonical=True, chunk_id=chunk.chunk_id)
|
|
231
|
+
allocated_blocks.append(phys_id)
|
|
232
|
+
|
|
233
|
+
# Copy slice into physical block pool:
|
|
234
|
+
# k_cache is (num_layers, num_kv_heads, L_c, head_dim)
|
|
235
|
+
self.k_pool[phys_id, :, :, :block_len, :] = k_storage[:, :, start:end, :]
|
|
236
|
+
self.v_pool[phys_id, :, :, :block_len, :] = v_storage[:, :, start:end, :]
|
|
237
|
+
|
|
238
|
+
if self.quantization == "int8" and k_scale is not None and v_scale is not None:
|
|
239
|
+
self.k_scale_pool[phys_id, :, :, :block_len, :] = k_scale[:, :, start:end, :]
|
|
240
|
+
self.v_scale_pool[phys_id, :, :, :block_len, :] = v_scale[:, :, start:end, :]
|
|
241
|
+
|
|
242
|
+
self.canonical_registry[chunk.chunk_id] = allocated_blocks
|
|
243
|
+
self.canonical_lengths[chunk.chunk_id] = length
|
|
244
|
+
return allocated_blocks
|
|
245
|
+
|
|
246
|
+
def assemble_dag_sequence(
|
|
247
|
+
self,
|
|
248
|
+
seq_id: str,
|
|
249
|
+
chunk_sequence: List[Tuple[CanonicalChunk, int]], # List of (chunk, global_start_pos)
|
|
250
|
+
) -> OffsetBlockTable:
|
|
251
|
+
"""Constructs an OffsetBlockTable for a stitched sequence of chunks.
|
|
252
|
+
|
|
253
|
+
For each chunk c starting at global position G_c:
|
|
254
|
+
Its tokens sit at canonical positions p in [4, ..., L_c + 3].
|
|
255
|
+
Therefore, delta = G_c - 4.
|
|
256
|
+
"""
|
|
257
|
+
table = OffsetBlockTable(seq_id=seq_id, block_size=self.block_size)
|
|
258
|
+
|
|
259
|
+
for chunk, global_start in chunk_sequence:
|
|
260
|
+
delta = global_start - 4 # Canonical sink offset
|
|
261
|
+
phys_blocks = self.register_canonical_chunk(chunk)
|
|
262
|
+
|
|
263
|
+
remaining = chunk.length
|
|
264
|
+
for p_id in phys_blocks:
|
|
265
|
+
v_tokens = min(remaining, self.block_size)
|
|
266
|
+
table.append_block(
|
|
267
|
+
physical_idx=p_id,
|
|
268
|
+
delta=delta,
|
|
269
|
+
is_cow_shadow=False,
|
|
270
|
+
valid_tokens=v_tokens,
|
|
271
|
+
)
|
|
272
|
+
remaining -= v_tokens
|
|
273
|
+
|
|
274
|
+
return table
|
|
275
|
+
|
|
276
|
+
def create_cow_shadow_block(self, table: OffsetBlockTable, logical_idx: int) -> int:
|
|
277
|
+
"""Creates a private Copy-on-Write shadow block for a bridge repair.
|
|
278
|
+
|
|
279
|
+
Replaces the logical table entry with the new shadow block, leaving the
|
|
280
|
+
original canonical physical block intact and unmodified in memory.
|
|
281
|
+
"""
|
|
282
|
+
log_block = table.blocks[logical_idx]
|
|
283
|
+
old_phys_id = log_block.physical_idx
|
|
284
|
+
|
|
285
|
+
# If already a private shadow block, return directly
|
|
286
|
+
if log_block.is_cow_shadow:
|
|
287
|
+
return old_phys_id
|
|
288
|
+
|
|
289
|
+
# Allocate new shadow block
|
|
290
|
+
new_phys_id = self.allocate_block(is_canonical=False, chunk_id=None)
|
|
291
|
+
|
|
292
|
+
# Copy existing contents from canonical block to shadow block
|
|
293
|
+
self.k_pool[new_phys_id].copy_(self.k_pool[old_phys_id])
|
|
294
|
+
self.v_pool[new_phys_id].copy_(self.v_pool[old_phys_id])
|
|
295
|
+
|
|
296
|
+
if self.quantization == "int8" and self.k_scale_pool is not None:
|
|
297
|
+
self.k_scale_pool[new_phys_id].copy_(self.k_scale_pool[old_phys_id])
|
|
298
|
+
self.v_scale_pool[new_phys_id].copy_(self.v_scale_pool[old_phys_id])
|
|
299
|
+
|
|
300
|
+
# Release refcount on old canonical block
|
|
301
|
+
self.free_block(old_phys_id)
|
|
302
|
+
|
|
303
|
+
# Update table entry
|
|
304
|
+
log_block.physical_idx = new_phys_id
|
|
305
|
+
log_block.is_cow_shadow = True
|
|
306
|
+
return new_phys_id
|
|
307
|
+
|
|
308
|
+
def free_sequence(self, table: OffsetBlockTable):
|
|
309
|
+
"""Releases all logical block references held by a completed sequence table."""
|
|
310
|
+
for log_block in table.blocks:
|
|
311
|
+
self.free_block(log_block.physical_idx)
|
|
312
|
+
table.blocks.clear()
|