pico-vllm 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pico_vllm-0.1.0/.github/workflows/ci.yml +33 -0
- pico_vllm-0.1.0/.gitignore +30 -0
- pico_vllm-0.1.0/PKG-INFO +258 -0
- pico_vllm-0.1.0/README.md +228 -0
- pico_vllm-0.1.0/ROADMAP.md +213 -0
- pico_vllm-0.1.0/docs/concepts/decode-attention-and-cuda-graph.md +153 -0
- pico_vllm-0.1.0/docs/design-vs-nano-vllm.md +45 -0
- pico_vllm-0.1.0/docs/experiments/m6-triton-kv-store.md +152 -0
- pico_vllm-0.1.0/docs/m6-optimizations.md +74 -0
- pico_vllm-0.1.0/example.py +51 -0
- pico_vllm-0.1.0/pyproject.toml +80 -0
- pico_vllm-0.1.0/src/pico_vllm/__init__.py +5 -0
- pico_vllm-0.1.0/src/pico_vllm/benchmark.py +232 -0
- pico_vllm-0.1.0/src/pico_vllm/cli.py +29 -0
- pico_vllm-0.1.0/src/pico_vllm/config.py +80 -0
- pico_vllm-0.1.0/src/pico_vllm/core/__init__.py +1 -0
- pico_vllm-0.1.0/src/pico_vllm/core/block_manager.py +196 -0
- pico_vllm-0.1.0/src/pico_vllm/core/paged_kv_cache.py +90 -0
- pico_vllm-0.1.0/src/pico_vllm/core/scheduler.py +139 -0
- pico_vllm-0.1.0/src/pico_vllm/engine/__init__.py +1 -0
- pico_vllm-0.1.0/src/pico_vllm/engine/llm_engine.py +139 -0
- pico_vllm-0.1.0/src/pico_vllm/kernels/__init__.py +1 -0
- pico_vllm-0.1.0/src/pico_vllm/kernels/kv_store.py +79 -0
- pico_vllm-0.1.0/src/pico_vllm/layers/__init__.py +1 -0
- pico_vllm-0.1.0/src/pico_vllm/layers/activation.py +13 -0
- pico_vllm-0.1.0/src/pico_vllm/layers/attention.py +80 -0
- pico_vllm-0.1.0/src/pico_vllm/layers/layernorm.py +19 -0
- pico_vllm-0.1.0/src/pico_vllm/layers/linear.py +47 -0
- pico_vllm-0.1.0/src/pico_vllm/layers/rotary_embedding.py +30 -0
- pico_vllm-0.1.0/src/pico_vllm/llm.py +54 -0
- pico_vllm-0.1.0/src/pico_vllm/models/__init__.py +1 -0
- pico_vllm-0.1.0/src/pico_vllm/models/qwen3.py +140 -0
- pico_vllm-0.1.0/src/pico_vllm/sampler.py +31 -0
- pico_vllm-0.1.0/src/pico_vllm/sampling_params.py +27 -0
- pico_vllm-0.1.0/src/pico_vllm/sequence.py +81 -0
- pico_vllm-0.1.0/src/pico_vllm/utils/__init__.py +1 -0
- pico_vllm-0.1.0/src/pico_vllm/utils/context.py +78 -0
- pico_vllm-0.1.0/src/pico_vllm/utils/loader.py +83 -0
- pico_vllm-0.1.0/src/pico_vllm/worker/__init__.py +1 -0
- pico_vllm-0.1.0/src/pico_vllm/worker/model_runner.py +85 -0
- pico_vllm-0.1.0/tests/conftest.py +13 -0
- pico_vllm-0.1.0/tests/cuda/test_triton_store.py +87 -0
- pico_vllm-0.1.0/tests/integration/test_benchmark.py +46 -0
- pico_vllm-0.1.0/tests/integration/test_llm_engine.py +235 -0
- pico_vllm-0.1.0/tests/integration/test_runner_fake_model.py +189 -0
- pico_vllm-0.1.0/tests/test_cli.py +29 -0
- pico_vllm-0.1.0/tests/test_package.py +7 -0
- pico_vllm-0.1.0/tests/unit/test_attention.py +108 -0
- pico_vllm-0.1.0/tests/unit/test_block_manager.py +450 -0
- pico_vllm-0.1.0/tests/unit/test_device_dtype.py +100 -0
- pico_vllm-0.1.0/tests/unit/test_paged_kv_cache.py +137 -0
- pico_vllm-0.1.0/tests/unit/test_qwen3_parity.py +96 -0
- pico_vllm-0.1.0/tests/unit/test_qwen3_shapes.py +157 -0
- pico_vllm-0.1.0/tests/unit/test_rms_norm.py +69 -0
- pico_vllm-0.1.0/tests/unit/test_rope.py +99 -0
- pico_vllm-0.1.0/tests/unit/test_sampler.py +81 -0
- pico_vllm-0.1.0/tests/unit/test_sampling_params.py +27 -0
- pico_vllm-0.1.0/tests/unit/test_scheduler.py +385 -0
- pico_vllm-0.1.0/tests/unit/test_sequence.py +56 -0
- pico_vllm-0.1.0/uv.lock +3510 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.10", "3.12"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- uses: astral-sh/setup-uv@v6
|
|
19
|
+
with:
|
|
20
|
+
python-version: ${{ matrix.python-version }}
|
|
21
|
+
|
|
22
|
+
# CPU-only torch: tests never need CUDA, and the default Linux wheels pull
|
|
23
|
+
# several GB of NVIDIA libraries.
|
|
24
|
+
- name: Install dependencies
|
|
25
|
+
run: |
|
|
26
|
+
uv venv
|
|
27
|
+
uv pip install --torch-backend cpu -e ".[dev,model]"
|
|
28
|
+
|
|
29
|
+
- name: Lint
|
|
30
|
+
run: uv run --no-sync ruff check src tests
|
|
31
|
+
|
|
32
|
+
- name: Test
|
|
33
|
+
run: uv run --no-sync pytest -q
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Python bytecode and test artifacts
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
.pytest_cache/
|
|
5
|
+
.mypy_cache/
|
|
6
|
+
.ruff_cache/
|
|
7
|
+
.coverage
|
|
8
|
+
coverage.xml
|
|
9
|
+
htmlcov/
|
|
10
|
+
|
|
11
|
+
# Virtual environments and build artifacts
|
|
12
|
+
.venv/
|
|
13
|
+
venv/
|
|
14
|
+
dist/
|
|
15
|
+
build/
|
|
16
|
+
*.egg-info/
|
|
17
|
+
|
|
18
|
+
# IDE and OS files
|
|
19
|
+
.idea/
|
|
20
|
+
.vscode/
|
|
21
|
+
.DS_Store
|
|
22
|
+
|
|
23
|
+
# Local model artifacts (never commit model weights)
|
|
24
|
+
checkpoints/
|
|
25
|
+
weights/
|
|
26
|
+
.models/
|
|
27
|
+
*.safetensors
|
|
28
|
+
*.bin
|
|
29
|
+
*.pt
|
|
30
|
+
*.pth
|
pico_vllm-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pico-vllm
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A small, test-driven vLLM-inspired inference engine for Qwen3 models
|
|
5
|
+
Author: pico-vLLM contributors
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Requires-Dist: numpy>=2.2.6
|
|
13
|
+
Requires-Dist: torch>=2.2
|
|
14
|
+
Requires-Dist: xxhash>=3.5
|
|
15
|
+
Provides-Extra: api
|
|
16
|
+
Requires-Dist: fastapi>=0.115; extra == 'api'
|
|
17
|
+
Requires-Dist: uvicorn[standard]>=0.30; extra == 'api'
|
|
18
|
+
Provides-Extra: cuda
|
|
19
|
+
Requires-Dist: triton>=3.0; (sys_platform == 'linux') and extra == 'cuda'
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
22
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
24
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
25
|
+
Provides-Extra: model
|
|
26
|
+
Requires-Dist: safetensors>=0.4; extra == 'model'
|
|
27
|
+
Requires-Dist: tokenizers>=0.20; extra == 'model'
|
|
28
|
+
Requires-Dist: transformers>=4.51; extra == 'model'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# pico-vLLM
|
|
32
|
+
|
|
33
|
+
`pico-vLLM` is a single-GPU, single-process inference engine for learning the core ideas behind vLLM. The project deliberately focuses on the execution path that matters most: request lifecycle, continuous batching, KV-cache allocation, paged memory, and testable text generation.
|
|
34
|
+
|
|
35
|
+
The immediate target is a compact core replica for a dense **Qwen3 decoder-only** checkpoint, loaded from a Hugging Face-compatible local directory. Qwen3 provides the important real inference features—RMSNorm, RoPE, grouped-query attention (GQA), Q/K normalization, and SwiGLU—without multimodal or MoE complexity.
|
|
36
|
+
|
|
37
|
+
The milestone-by-milestone plan is in [ROADMAP.md](ROADMAP.md).
|
|
38
|
+
|
|
39
|
+
## Recommended reading
|
|
40
|
+
|
|
41
|
+
- [AIInfraGuide — AI Infra 全栈知识库](https://caomaolufei.github.io/AIInfraGuide/): a Chinese guide to the AI infrastructure stack, covering CUDA kernel optimization, distributed training, inference optimization, and performance analysis. Useful background for the paged KV cache, FlashAttention, and CUDA Graph work in this project.
|
|
42
|
+
|
|
43
|
+
## Scope
|
|
44
|
+
|
|
45
|
+
The first track will support:
|
|
46
|
+
|
|
47
|
+
- decoder-only Qwen-style autoregressive generation;
|
|
48
|
+
- continuous batching across several requests;
|
|
49
|
+
- paged KV-cache allocation, sharing, release, and copy-on-write;
|
|
50
|
+
- greedy, temperature, top-k, top-p, EOS, and `max_tokens` controls;
|
|
51
|
+
- deterministic CPU tests with a fake model runner;
|
|
52
|
+
- a single-GPU Qwen3 runner and an offline Python API.
|
|
53
|
+
|
|
54
|
+
It intentionally defers distributed execution, quantization, LoRA, multimodal models, mixture-of-experts, speculative decoding, HTTP serving, custom CUDA/Triton kernels, and production authentication. They belong to the later hardening track, not the initial core replica.
|
|
55
|
+
|
|
56
|
+
## Repository layout
|
|
57
|
+
|
|
58
|
+
```text
|
|
59
|
+
.
|
|
60
|
+
├── pyproject.toml
|
|
61
|
+
├── README.md
|
|
62
|
+
├── ROADMAP.md
|
|
63
|
+
├── src/pico_vllm/
|
|
64
|
+
│ ├── config.py
|
|
65
|
+
│ ├── sampling_params.py
|
|
66
|
+
│ ├── sequence.py
|
|
67
|
+
│ ├── core/ # scheduler, blocks, KV cache
|
|
68
|
+
│ ├── engine/ # LLMEngine and output events
|
|
69
|
+
│ ├── worker/ # model runners
|
|
70
|
+
│ ├── model_executor/ # Qwen layers and weight loader
|
|
71
|
+
│ └── entrypoints/ # CLI and HTTP API
|
|
72
|
+
└── tests/
|
|
73
|
+
├── unit/
|
|
74
|
+
├── integration/
|
|
75
|
+
└── conftest.py
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Installation
|
|
79
|
+
|
|
80
|
+
Install [uv](https://docs.astral.sh/uv/getting-started/installation/) once. From the repository root, create the project environment and install the development tools:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
uv sync --extra dev
|
|
84
|
+
uv run pytest -q
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
`uv sync` creates `.venv` and installs the project in editable mode. `uv run` uses that environment even if your shell still shows Conda `(base)`, avoiding unrelated global pytest plugins.
|
|
88
|
+
|
|
89
|
+
Install the Qwen loading dependencies at M3. The web dependencies are only needed at M9:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
uv sync --extra dev --extra model
|
|
93
|
+
uv sync --extra dev --extra model --extra api
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
`pyproject.toml` deliberately keeps `transformers`, FastAPI, and Uvicorn optional. The scheduler/cache unit tests must remain fast and independent of network access, model downloads, and CUDA.
|
|
97
|
+
|
|
98
|
+
## Qwen3 model boundary
|
|
99
|
+
|
|
100
|
+
Start with a small dense Qwen3 model, preferably `Qwen/Qwen3-0.6B`. Parse `config.json` into a local `QwenConfig`; do not make scheduler and cache code depend on Hugging Face configuration objects.
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
@dataclass(frozen=True)
|
|
104
|
+
class Qwen3Config:
|
|
105
|
+
vocab_size: int
|
|
106
|
+
hidden_size: int
|
|
107
|
+
intermediate_size: int
|
|
108
|
+
num_hidden_layers: int
|
|
109
|
+
num_attention_heads: int
|
|
110
|
+
num_key_value_heads: int
|
|
111
|
+
head_dim: int
|
|
112
|
+
rms_norm_eps: float
|
|
113
|
+
rope_theta: float
|
|
114
|
+
max_position_embeddings: int
|
|
115
|
+
tie_word_embeddings: bool
|
|
116
|
+
attention_bias: bool
|
|
117
|
+
use_qk_norm: bool
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
The forward path is:
|
|
121
|
+
|
|
122
|
+
```text
|
|
123
|
+
token ids → embedding
|
|
124
|
+
→ repeated [RMSNorm → GQA attention with RoPE → residual
|
|
125
|
+
RMSNorm → SwiGLU MLP → residual]
|
|
126
|
+
→ final RMSNorm → LM head → logits
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Important implementation details:
|
|
130
|
+
|
|
131
|
+
- GQA means `num_key_value_heads` may be smaller than `num_attention_heads`.
|
|
132
|
+
- RoPE positions must be identical in prefill and decode paths.
|
|
133
|
+
- Qwen3 needs its configuration read exactly: Q/K normalization, projection bias, and RoPE settings must come from the selected checkpoint rather than from hard-coded assumptions.
|
|
134
|
+
- The initial implementation supports the checkpoint's normal RoPE settings. Long-context RoPE scaling is a later extension.
|
|
135
|
+
- Unit tests use a fake runner only. Real-model parity tests receive a local checkpoint through `PICO_VLLM_QWEN_MODEL=/absolute/path` and skip if absent.
|
|
136
|
+
|
|
137
|
+
## Component boundaries
|
|
138
|
+
|
|
139
|
+
| Component | Owns | Must not own |
|
|
140
|
+
| --- | --- | --- |
|
|
141
|
+
| `SamplingParams` | validation of generation settings | scheduling |
|
|
142
|
+
| `Sequence` | one request's tokens, state, output | physical blocks |
|
|
143
|
+
| `Scheduler` | waiting/running selection under budgets | model execution |
|
|
144
|
+
| `BlockSpaceManager` | block allocation, sharing, refcounts | request priority |
|
|
145
|
+
| `ModelRunner` | Qwen3 prefill/decode and logits | request lifecycle |
|
|
146
|
+
| `LLMEngine` | orchestration and output events | HTTP protocol |
|
|
147
|
+
|
|
148
|
+
Use the state machine below and allow only the scheduler/engine to transition states:
|
|
149
|
+
|
|
150
|
+
```text
|
|
151
|
+
WAITING → RUNNING → FINISHED_STOPPED
|
|
152
|
+
└→ FINISHED_LENGTH_CAPPED
|
|
153
|
+
└→ FINISHED_ABORTED
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
## How the components fit together
|
|
157
|
+
|
|
158
|
+
Two independent halves meet only through `Sequence.block_table`, a plain list of block ids:
|
|
159
|
+
|
|
160
|
+
- the **bookkeeping half** (CPU, Python objects) decides *which* block ids a sequence owns;
|
|
161
|
+
- the **storage half** (CPU or GPU tensors) holds the actual K/V values at those ids.
|
|
162
|
+
|
|
163
|
+
`BlockManager` never touches a tensor, and `PagedKVCache` never decides who owns a block. In particular, `BlockManager` does **not** call `PagedKVCache`.
|
|
164
|
+
|
|
165
|
+
```text
|
|
166
|
+
LLMEngine (M5) — calls both halves once per step
|
|
167
|
+
/ \
|
|
168
|
+
bookkeeping (CPU) storage + compute (device)
|
|
169
|
+
───────────────── ──────────────────────────
|
|
170
|
+
Scheduler (M2) ModelRunner (M4)
|
|
171
|
+
│ owns │ owns │ owns
|
|
172
|
+
▼ ▼ ▼
|
|
173
|
+
BlockManager (M2) Qwen3ForCausalLM PagedKVCache (M4)
|
|
174
|
+
· free queue, ref_count, hash (M3) k_cache / v_cache
|
|
175
|
+
· writes block ids into ─┐ │ each layer [L, N, B, H, D]
|
|
176
|
+
│ ▼ ▲
|
|
177
|
+
▼ Attention (M4) ────────┘
|
|
178
|
+
Sequence.block_table reads Context: write()/read()
|
|
179
|
+
e.g. [3, 8] slot_mapping,
|
|
180
|
+
│ block_tables, ...
|
|
181
|
+
│ ▲
|
|
182
|
+
└──► ModelRunner.prepare_inputs builds Context
|
|
183
|
+
(set_context / get_context, utils/context.py)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Who calls whom:
|
|
187
|
+
|
|
188
|
+
| Caller | Callee | Why |
|
|
189
|
+
| --- | --- | --- |
|
|
190
|
+
| `LLMEngine.step` | `Scheduler.schedule` | pick the seqs for this step; blocks are reserved here |
|
|
191
|
+
| `Scheduler` | `BlockManager.can_allocate / allocate / may_append / deallocate / hash_blocks` | manage block ids, write `seq.block_table` |
|
|
192
|
+
| `LLMEngine.step` | `ModelRunner.run(seqs, is_prefill)` | run the model on the scheduled batch |
|
|
193
|
+
| `ModelRunner.prepare_inputs` | reads `seq.block_table`, `num_cached_tokens`, ... | turn block ids into `slot_mapping` and a `Context` |
|
|
194
|
+
| `ModelRunner.run` | `set_context` → `model(...)` → `reset_context` | expose the step's metadata to every layer |
|
|
195
|
+
| `Attention.forward` | `get_context().kv_cache.write / read` | store this step's K/V and read the full context |
|
|
196
|
+
| `LLMEngine.step` | `Scheduler.postprocess` | append sampled tokens, hash full blocks, finish/free seqs |
|
|
197
|
+
|
|
198
|
+
One step, end to end:
|
|
199
|
+
|
|
200
|
+
```text
|
|
201
|
+
1. Scheduler.schedule() BlockManager gives seq A blocks [3, 8] (ids only)
|
|
202
|
+
2. ModelRunner.prepare_inputs pos 0..4 of A → slots 12,13,14,15,32 (ids → slots)
|
|
203
|
+
3. Attention, every layer kv_cache.write(slots, k, v) (values land in k_cache)
|
|
204
|
+
kv_cache.read(block_table=[3, 8], len=5)
|
|
205
|
+
4. Scheduler.postprocess append token; finished? → deallocate (ids return to free queue;
|
|
206
|
+
k_cache is left untouched)
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
`PagedKVCache` is allocated once at startup with `num_blocks` equal to the `BlockManager`'s `num_blocks`; the two must agree, since block id `i` in one is index `i` in the other.
|
|
210
|
+
|
|
211
|
+
## Paged KV cache
|
|
212
|
+
|
|
213
|
+
For `block_size = 4`, ten tokens occupy three logical blocks:
|
|
214
|
+
|
|
215
|
+
```text
|
|
216
|
+
logical token positions: 0 1 2 3 | 4 5 6 7 | 8 9
|
|
217
|
+
logical block index: 0 | 1 | 2
|
|
218
|
+
block table: [12, 7, 19]
|
|
219
|
+
physical block id: 12 7 19
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
`Sequence.block_table` maps logical blocks to physical blocks. Prompt-sharing sequences may reference the same blocks. If a sequence appends to a shared partial tail block, it first receives a copied block and changes only its own table: copy-on-write. This behavior needs focused tests; an end-to-end generation assertion is not enough.
|
|
223
|
+
|
|
224
|
+
Core invariants:
|
|
225
|
+
|
|
226
|
+
- A physical block contains exactly `block_size` token positions.
|
|
227
|
+
- `num_computed_tokens <= len(sequence.token_ids)`.
|
|
228
|
+
- A block returns to the free queue only when its reference count becomes zero.
|
|
229
|
+
- A scheduler step does not exceed `max_num_batched_tokens`.
|
|
230
|
+
|
|
231
|
+
## Test and commit contract
|
|
232
|
+
|
|
233
|
+
For every milestone:
|
|
234
|
+
|
|
235
|
+
1. Write a failing, focused test.
|
|
236
|
+
2. Implement the smallest correct behavior.
|
|
237
|
+
3. Run the milestone tests and then all tests.
|
|
238
|
+
4. Make one Git commit only after all tests pass.
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
uv run pytest tests/unit -q
|
|
242
|
+
uv run pytest tests/integration -q
|
|
243
|
+
uv run pytest -q
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
Use a seeded `torch.Generator` whenever a test samples randomly. Unit tests must never need CUDA, an internet connection, or downloaded weights. Real Qwen parity tests have both `integration` and `model` markers; CUDA-only tests additionally have `cuda`.
|
|
247
|
+
|
|
248
|
+
## Delivery plan and estimated size
|
|
249
|
+
|
|
250
|
+
The roadmap deliberately starts with a compact Qwen3 core replica, then expands only after it works. It excludes model weights, dependencies, generated reports, and deployment infrastructure.
|
|
251
|
+
|
|
252
|
+
| Scope | Production | Tests | Total |
|
|
253
|
+
| --- | ---: | ---: | ---: |
|
|
254
|
+
| Track A M0–M5: eager Qwen3 core | 1,500–2,500 LOC | 600–1,200 LOC | 2,100–3,700 LOC |
|
|
255
|
+
| Track A M6: optional CUDA fast path | +300–800 LOC | +150–350 LOC | +450–1,150 LOC |
|
|
256
|
+
| Track B: hardening, API, metrics, scale-out | +1,500–3,000 LOC | +1,500–2,500 LOC | +3,000–5,500 LOC |
|
|
257
|
+
|
|
258
|
+
The immediate goal is Track A only. It is deliberately close to the compact reference-project scale, but still includes enough focused tests to make scheduler and block-manager bugs visible.
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# pico-vLLM
|
|
2
|
+
|
|
3
|
+
`pico-vLLM` is a single-GPU, single-process inference engine for learning the core ideas behind vLLM. The project deliberately focuses on the execution path that matters most: request lifecycle, continuous batching, KV-cache allocation, paged memory, and testable text generation.
|
|
4
|
+
|
|
5
|
+
The immediate target is a compact core replica for a dense **Qwen3 decoder-only** checkpoint, loaded from a Hugging Face-compatible local directory. Qwen3 provides the important real inference features—RMSNorm, RoPE, grouped-query attention (GQA), Q/K normalization, and SwiGLU—without multimodal or MoE complexity.
|
|
6
|
+
|
|
7
|
+
The milestone-by-milestone plan is in [ROADMAP.md](ROADMAP.md).
|
|
8
|
+
|
|
9
|
+
## Recommended reading
|
|
10
|
+
|
|
11
|
+
- [AIInfraGuide — AI Infra 全栈知识库](https://caomaolufei.github.io/AIInfraGuide/): a Chinese guide to the AI infrastructure stack, covering CUDA kernel optimization, distributed training, inference optimization, and performance analysis. Useful background for the paged KV cache, FlashAttention, and CUDA Graph work in this project.
|
|
12
|
+
|
|
13
|
+
## Scope
|
|
14
|
+
|
|
15
|
+
The first track will support:
|
|
16
|
+
|
|
17
|
+
- decoder-only Qwen-style autoregressive generation;
|
|
18
|
+
- continuous batching across several requests;
|
|
19
|
+
- paged KV-cache allocation, sharing, release, and copy-on-write;
|
|
20
|
+
- greedy, temperature, top-k, top-p, EOS, and `max_tokens` controls;
|
|
21
|
+
- deterministic CPU tests with a fake model runner;
|
|
22
|
+
- a single-GPU Qwen3 runner and an offline Python API.
|
|
23
|
+
|
|
24
|
+
It intentionally defers distributed execution, quantization, LoRA, multimodal models, mixture-of-experts, speculative decoding, HTTP serving, custom CUDA/Triton kernels, and production authentication. They belong to the later hardening track, not the initial core replica.
|
|
25
|
+
|
|
26
|
+
## Repository layout
|
|
27
|
+
|
|
28
|
+
```text
|
|
29
|
+
.
|
|
30
|
+
├── pyproject.toml
|
|
31
|
+
├── README.md
|
|
32
|
+
├── ROADMAP.md
|
|
33
|
+
├── src/pico_vllm/
|
|
34
|
+
│ ├── config.py
|
|
35
|
+
│ ├── sampling_params.py
|
|
36
|
+
│ ├── sequence.py
|
|
37
|
+
│ ├── core/ # scheduler, blocks, KV cache
|
|
38
|
+
│ ├── engine/ # LLMEngine and output events
|
|
39
|
+
│ ├── worker/ # model runners
|
|
40
|
+
│ ├── model_executor/ # Qwen layers and weight loader
|
|
41
|
+
│ └── entrypoints/ # CLI and HTTP API
|
|
42
|
+
└── tests/
|
|
43
|
+
├── unit/
|
|
44
|
+
├── integration/
|
|
45
|
+
└── conftest.py
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Installation
|
|
49
|
+
|
|
50
|
+
Install [uv](https://docs.astral.sh/uv/getting-started/installation/) once. From the repository root, create the project environment and install the development tools:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
uv sync --extra dev
|
|
54
|
+
uv run pytest -q
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
`uv sync` creates `.venv` and installs the project in editable mode. `uv run` uses that environment even if your shell still shows Conda `(base)`, avoiding unrelated global pytest plugins.
|
|
58
|
+
|
|
59
|
+
Install the Qwen loading dependencies at M3. The web dependencies are only needed at M9:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
uv sync --extra dev --extra model
|
|
63
|
+
uv sync --extra dev --extra model --extra api
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`pyproject.toml` deliberately keeps `transformers`, FastAPI, and Uvicorn optional. The scheduler/cache unit tests must remain fast and independent of network access, model downloads, and CUDA.
|
|
67
|
+
|
|
68
|
+
## Qwen3 model boundary
|
|
69
|
+
|
|
70
|
+
Start with a small dense Qwen3 model, preferably `Qwen/Qwen3-0.6B`. Parse `config.json` into a local `QwenConfig`; do not make scheduler and cache code depend on Hugging Face configuration objects.
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class Qwen3Config:
|
|
75
|
+
vocab_size: int
|
|
76
|
+
hidden_size: int
|
|
77
|
+
intermediate_size: int
|
|
78
|
+
num_hidden_layers: int
|
|
79
|
+
num_attention_heads: int
|
|
80
|
+
num_key_value_heads: int
|
|
81
|
+
head_dim: int
|
|
82
|
+
rms_norm_eps: float
|
|
83
|
+
rope_theta: float
|
|
84
|
+
max_position_embeddings: int
|
|
85
|
+
tie_word_embeddings: bool
|
|
86
|
+
attention_bias: bool
|
|
87
|
+
use_qk_norm: bool
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The forward path is:
|
|
91
|
+
|
|
92
|
+
```text
|
|
93
|
+
token ids → embedding
|
|
94
|
+
→ repeated [RMSNorm → GQA attention with RoPE → residual
|
|
95
|
+
RMSNorm → SwiGLU MLP → residual]
|
|
96
|
+
→ final RMSNorm → LM head → logits
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Important implementation details:
|
|
100
|
+
|
|
101
|
+
- GQA means `num_key_value_heads` may be smaller than `num_attention_heads`.
|
|
102
|
+
- RoPE positions must be identical in prefill and decode paths.
|
|
103
|
+
- Qwen3 needs its configuration read exactly: Q/K normalization, projection bias, and RoPE settings must come from the selected checkpoint rather than from hard-coded assumptions.
|
|
104
|
+
- The initial implementation supports the checkpoint's normal RoPE settings. Long-context RoPE scaling is a later extension.
|
|
105
|
+
- Unit tests use a fake runner only. Real-model parity tests receive a local checkpoint through `PICO_VLLM_QWEN_MODEL=/absolute/path` and skip if absent.
|
|
106
|
+
|
|
107
|
+
## Component boundaries
|
|
108
|
+
|
|
109
|
+
| Component | Owns | Must not own |
|
|
110
|
+
| --- | --- | --- |
|
|
111
|
+
| `SamplingParams` | validation of generation settings | scheduling |
|
|
112
|
+
| `Sequence` | one request's tokens, state, output | physical blocks |
|
|
113
|
+
| `Scheduler` | waiting/running selection under budgets | model execution |
|
|
114
|
+
| `BlockSpaceManager` | block allocation, sharing, refcounts | request priority |
|
|
115
|
+
| `ModelRunner` | Qwen3 prefill/decode and logits | request lifecycle |
|
|
116
|
+
| `LLMEngine` | orchestration and output events | HTTP protocol |
|
|
117
|
+
|
|
118
|
+
Use the state machine below and allow only the scheduler/engine to transition states:
|
|
119
|
+
|
|
120
|
+
```text
|
|
121
|
+
WAITING → RUNNING → FINISHED_STOPPED
|
|
122
|
+
└→ FINISHED_LENGTH_CAPPED
|
|
123
|
+
└→ FINISHED_ABORTED
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## How the components fit together
|
|
127
|
+
|
|
128
|
+
Two independent halves meet only through `Sequence.block_table`, a plain list of block ids:
|
|
129
|
+
|
|
130
|
+
- the **bookkeeping half** (CPU, Python objects) decides *which* block ids a sequence owns;
|
|
131
|
+
- the **storage half** (CPU or GPU tensors) holds the actual K/V values at those ids.
|
|
132
|
+
|
|
133
|
+
`BlockManager` never touches a tensor, and `PagedKVCache` never decides who owns a block. In particular, `BlockManager` does **not** call `PagedKVCache`.
|
|
134
|
+
|
|
135
|
+
```text
|
|
136
|
+
LLMEngine (M5) — calls both halves once per step
|
|
137
|
+
/ \
|
|
138
|
+
bookkeeping (CPU) storage + compute (device)
|
|
139
|
+
───────────────── ──────────────────────────
|
|
140
|
+
Scheduler (M2) ModelRunner (M4)
|
|
141
|
+
│ owns │ owns │ owns
|
|
142
|
+
▼ ▼ ▼
|
|
143
|
+
BlockManager (M2) Qwen3ForCausalLM PagedKVCache (M4)
|
|
144
|
+
· free queue, ref_count, hash (M3) k_cache / v_cache
|
|
145
|
+
· writes block ids into ─┐ │ each layer [L, N, B, H, D]
|
|
146
|
+
│ ▼ ▲
|
|
147
|
+
▼ Attention (M4) ────────┘
|
|
148
|
+
Sequence.block_table reads Context: write()/read()
|
|
149
|
+
e.g. [3, 8] slot_mapping,
|
|
150
|
+
│ block_tables, ...
|
|
151
|
+
│ ▲
|
|
152
|
+
└──► ModelRunner.prepare_inputs builds Context
|
|
153
|
+
(set_context / get_context, utils/context.py)
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Who calls whom:
|
|
157
|
+
|
|
158
|
+
| Caller | Callee | Why |
|
|
159
|
+
| --- | --- | --- |
|
|
160
|
+
| `LLMEngine.step` | `Scheduler.schedule` | pick the seqs for this step; blocks are reserved here |
|
|
161
|
+
| `Scheduler` | `BlockManager.can_allocate / allocate / may_append / deallocate / hash_blocks` | manage block ids, write `seq.block_table` |
|
|
162
|
+
| `LLMEngine.step` | `ModelRunner.run(seqs, is_prefill)` | run the model on the scheduled batch |
|
|
163
|
+
| `ModelRunner.prepare_inputs` | reads `seq.block_table`, `num_cached_tokens`, ... | turn block ids into `slot_mapping` and a `Context` |
|
|
164
|
+
| `ModelRunner.run` | `set_context` → `model(...)` → `reset_context` | expose the step's metadata to every layer |
|
|
165
|
+
| `Attention.forward` | `get_context().kv_cache.write / read` | store this step's K/V and read the full context |
|
|
166
|
+
| `LLMEngine.step` | `Scheduler.postprocess` | append sampled tokens, hash full blocks, finish/free seqs |
|
|
167
|
+
|
|
168
|
+
One step, end to end:
|
|
169
|
+
|
|
170
|
+
```text
|
|
171
|
+
1. Scheduler.schedule() BlockManager gives seq A blocks [3, 8] (ids only)
|
|
172
|
+
2. ModelRunner.prepare_inputs pos 0..4 of A → slots 12,13,14,15,32 (ids → slots)
|
|
173
|
+
3. Attention, every layer kv_cache.write(slots, k, v) (values land in k_cache)
|
|
174
|
+
kv_cache.read(block_table=[3, 8], len=5)
|
|
175
|
+
4. Scheduler.postprocess append token; finished? → deallocate (ids return to free queue;
|
|
176
|
+
k_cache is left untouched)
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
`PagedKVCache` is allocated once at startup with `num_blocks` equal to the `BlockManager`'s `num_blocks`; the two must agree, since block id `i` in one is index `i` in the other.
|
|
180
|
+
|
|
181
|
+
## Paged KV cache
|
|
182
|
+
|
|
183
|
+
For `block_size = 4`, ten tokens occupy three logical blocks:
|
|
184
|
+
|
|
185
|
+
```text
|
|
186
|
+
logical token positions: 0 1 2 3 | 4 5 6 7 | 8 9
|
|
187
|
+
logical block index: 0 | 1 | 2
|
|
188
|
+
block table: [12, 7, 19]
|
|
189
|
+
physical block id: 12 7 19
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
`Sequence.block_table` maps logical blocks to physical blocks. Prompt-sharing sequences may reference the same blocks. If a sequence appends to a shared partial tail block, it first receives a copied block and changes only its own table: copy-on-write. This behavior needs focused tests; an end-to-end generation assertion is not enough.
|
|
193
|
+
|
|
194
|
+
Core invariants:
|
|
195
|
+
|
|
196
|
+
- A physical block contains exactly `block_size` token positions.
|
|
197
|
+
- `num_computed_tokens <= len(sequence.token_ids)`.
|
|
198
|
+
- A block returns to the free queue only when its reference count becomes zero.
|
|
199
|
+
- A scheduler step does not exceed `max_num_batched_tokens`.
|
|
200
|
+
|
|
201
|
+
## Test and commit contract
|
|
202
|
+
|
|
203
|
+
For every milestone:
|
|
204
|
+
|
|
205
|
+
1. Write a failing, focused test.
|
|
206
|
+
2. Implement the smallest correct behavior.
|
|
207
|
+
3. Run the milestone tests and then all tests.
|
|
208
|
+
4. Make one Git commit only after all tests pass.
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
uv run pytest tests/unit -q
|
|
212
|
+
uv run pytest tests/integration -q
|
|
213
|
+
uv run pytest -q
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
Use a seeded `torch.Generator` whenever a test samples randomly. Unit tests must never need CUDA, an internet connection, or downloaded weights. Real Qwen parity tests have both `integration` and `model` markers; CUDA-only tests additionally have `cuda`.
|
|
217
|
+
|
|
218
|
+
## Delivery plan and estimated size
|
|
219
|
+
|
|
220
|
+
The roadmap deliberately starts with a compact Qwen3 core replica, then expands only after it works. It excludes model weights, dependencies, generated reports, and deployment infrastructure.
|
|
221
|
+
|
|
222
|
+
| Scope | Production | Tests | Total |
|
|
223
|
+
| --- | ---: | ---: | ---: |
|
|
224
|
+
| Track A M0–M5: eager Qwen3 core | 1,500–2,500 LOC | 600–1,200 LOC | 2,100–3,700 LOC |
|
|
225
|
+
| Track A M6: optional CUDA fast path | +300–800 LOC | +150–350 LOC | +450–1,150 LOC |
|
|
226
|
+
| Track B: hardening, API, metrics, scale-out | +1,500–3,000 LOC | +1,500–2,500 LOC | +3,000–5,500 LOC |
|
|
227
|
+
|
|
228
|
+
The immediate goal is Track A only. It is deliberately close to the compact reference-project scale, but still includes enough focused tests to make scheduler and block-manager bugs visible.
|