pico-vllm 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. pico_vllm-0.1.0/.github/workflows/ci.yml +33 -0
  2. pico_vllm-0.1.0/.gitignore +30 -0
  3. pico_vllm-0.1.0/PKG-INFO +258 -0
  4. pico_vllm-0.1.0/README.md +228 -0
  5. pico_vllm-0.1.0/ROADMAP.md +213 -0
  6. pico_vllm-0.1.0/docs/concepts/decode-attention-and-cuda-graph.md +153 -0
  7. pico_vllm-0.1.0/docs/design-vs-nano-vllm.md +45 -0
  8. pico_vllm-0.1.0/docs/experiments/m6-triton-kv-store.md +152 -0
  9. pico_vllm-0.1.0/docs/m6-optimizations.md +74 -0
  10. pico_vllm-0.1.0/example.py +51 -0
  11. pico_vllm-0.1.0/pyproject.toml +80 -0
  12. pico_vllm-0.1.0/src/pico_vllm/__init__.py +5 -0
  13. pico_vllm-0.1.0/src/pico_vllm/benchmark.py +232 -0
  14. pico_vllm-0.1.0/src/pico_vllm/cli.py +29 -0
  15. pico_vllm-0.1.0/src/pico_vllm/config.py +80 -0
  16. pico_vllm-0.1.0/src/pico_vllm/core/__init__.py +1 -0
  17. pico_vllm-0.1.0/src/pico_vllm/core/block_manager.py +196 -0
  18. pico_vllm-0.1.0/src/pico_vllm/core/paged_kv_cache.py +90 -0
  19. pico_vllm-0.1.0/src/pico_vllm/core/scheduler.py +139 -0
  20. pico_vllm-0.1.0/src/pico_vllm/engine/__init__.py +1 -0
  21. pico_vllm-0.1.0/src/pico_vllm/engine/llm_engine.py +139 -0
  22. pico_vllm-0.1.0/src/pico_vllm/kernels/__init__.py +1 -0
  23. pico_vllm-0.1.0/src/pico_vllm/kernels/kv_store.py +79 -0
  24. pico_vllm-0.1.0/src/pico_vllm/layers/__init__.py +1 -0
  25. pico_vllm-0.1.0/src/pico_vllm/layers/activation.py +13 -0
  26. pico_vllm-0.1.0/src/pico_vllm/layers/attention.py +80 -0
  27. pico_vllm-0.1.0/src/pico_vllm/layers/layernorm.py +19 -0
  28. pico_vllm-0.1.0/src/pico_vllm/layers/linear.py +47 -0
  29. pico_vllm-0.1.0/src/pico_vllm/layers/rotary_embedding.py +30 -0
  30. pico_vllm-0.1.0/src/pico_vllm/llm.py +54 -0
  31. pico_vllm-0.1.0/src/pico_vllm/models/__init__.py +1 -0
  32. pico_vllm-0.1.0/src/pico_vllm/models/qwen3.py +140 -0
  33. pico_vllm-0.1.0/src/pico_vllm/sampler.py +31 -0
  34. pico_vllm-0.1.0/src/pico_vllm/sampling_params.py +27 -0
  35. pico_vllm-0.1.0/src/pico_vllm/sequence.py +81 -0
  36. pico_vllm-0.1.0/src/pico_vllm/utils/__init__.py +1 -0
  37. pico_vllm-0.1.0/src/pico_vllm/utils/context.py +78 -0
  38. pico_vllm-0.1.0/src/pico_vllm/utils/loader.py +83 -0
  39. pico_vllm-0.1.0/src/pico_vllm/worker/__init__.py +1 -0
  40. pico_vllm-0.1.0/src/pico_vllm/worker/model_runner.py +85 -0
  41. pico_vllm-0.1.0/tests/conftest.py +13 -0
  42. pico_vllm-0.1.0/tests/cuda/test_triton_store.py +87 -0
  43. pico_vllm-0.1.0/tests/integration/test_benchmark.py +46 -0
  44. pico_vllm-0.1.0/tests/integration/test_llm_engine.py +235 -0
  45. pico_vllm-0.1.0/tests/integration/test_runner_fake_model.py +189 -0
  46. pico_vllm-0.1.0/tests/test_cli.py +29 -0
  47. pico_vllm-0.1.0/tests/test_package.py +7 -0
  48. pico_vllm-0.1.0/tests/unit/test_attention.py +108 -0
  49. pico_vllm-0.1.0/tests/unit/test_block_manager.py +450 -0
  50. pico_vllm-0.1.0/tests/unit/test_device_dtype.py +100 -0
  51. pico_vllm-0.1.0/tests/unit/test_paged_kv_cache.py +137 -0
  52. pico_vllm-0.1.0/tests/unit/test_qwen3_parity.py +96 -0
  53. pico_vllm-0.1.0/tests/unit/test_qwen3_shapes.py +157 -0
  54. pico_vllm-0.1.0/tests/unit/test_rms_norm.py +69 -0
  55. pico_vllm-0.1.0/tests/unit/test_rope.py +99 -0
  56. pico_vllm-0.1.0/tests/unit/test_sampler.py +81 -0
  57. pico_vllm-0.1.0/tests/unit/test_sampling_params.py +27 -0
  58. pico_vllm-0.1.0/tests/unit/test_scheduler.py +385 -0
  59. pico_vllm-0.1.0/tests/unit/test_sequence.py +56 -0
  60. pico_vllm-0.1.0/uv.lock +3510 -0
@@ -0,0 +1,33 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ fail-fast: false
13
+ matrix:
14
+ python-version: ["3.10", "3.12"]
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+
18
+ - uses: astral-sh/setup-uv@v6
19
+ with:
20
+ python-version: ${{ matrix.python-version }}
21
+
22
+ # CPU-only torch: tests never need CUDA, and the default Linux wheels pull
23
+ # several GB of NVIDIA libraries.
24
+ - name: Install dependencies
25
+ run: |
26
+ uv venv
27
+ uv pip install --torch-backend cpu -e ".[dev,model]"
28
+
29
+ - name: Lint
30
+ run: uv run --no-sync ruff check src tests
31
+
32
+ - name: Test
33
+ run: uv run --no-sync pytest -q
@@ -0,0 +1,30 @@
1
+ # Python bytecode and test artifacts
2
+ __pycache__/
3
+ *.py[cod]
4
+ .pytest_cache/
5
+ .mypy_cache/
6
+ .ruff_cache/
7
+ .coverage
8
+ coverage.xml
9
+ htmlcov/
10
+
11
+ # Virtual environments and build artifacts
12
+ .venv/
13
+ venv/
14
+ dist/
15
+ build/
16
+ *.egg-info/
17
+
18
+ # IDE and OS files
19
+ .idea/
20
+ .vscode/
21
+ .DS_Store
22
+
23
+ # Local model artifacts (never commit model weights)
24
+ checkpoints/
25
+ weights/
26
+ .models/
27
+ *.safetensors
28
+ *.bin
29
+ *.pt
30
+ *.pth
@@ -0,0 +1,258 @@
1
+ Metadata-Version: 2.5
2
+ Name: pico-vllm
3
+ Version: 0.1.0
4
+ Summary: A small, test-driven vLLM-inspired inference engine for Qwen3 models
5
+ Author: pico-vLLM contributors
6
+ License: Apache-2.0
7
+ Classifier: Development Status :: 2 - Pre-Alpha
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: Programming Language :: Python :: 3.10
10
+ Classifier: Programming Language :: Python :: 3.11
11
+ Requires-Python: >=3.10
12
+ Requires-Dist: numpy>=2.2.6
13
+ Requires-Dist: torch>=2.2
14
+ Requires-Dist: xxhash>=3.5
15
+ Provides-Extra: api
16
+ Requires-Dist: fastapi>=0.115; extra == 'api'
17
+ Requires-Dist: uvicorn[standard]>=0.30; extra == 'api'
18
+ Provides-Extra: cuda
19
+ Requires-Dist: triton>=3.0; (sys_platform == 'linux') and extra == 'cuda'
20
+ Provides-Extra: dev
21
+ Requires-Dist: mypy>=1.11; extra == 'dev'
22
+ Requires-Dist: pytest-cov>=5.0; extra == 'dev'
23
+ Requires-Dist: pytest>=8.3; extra == 'dev'
24
+ Requires-Dist: ruff>=0.6; extra == 'dev'
25
+ Provides-Extra: model
26
+ Requires-Dist: safetensors>=0.4; extra == 'model'
27
+ Requires-Dist: tokenizers>=0.20; extra == 'model'
28
+ Requires-Dist: transformers>=4.51; extra == 'model'
29
+ Description-Content-Type: text/markdown
30
+
31
+ # pico-vLLM
32
+
33
+ `pico-vLLM` is a single-GPU, single-process inference engine for learning the core ideas behind vLLM. The project deliberately focuses on the execution path that matters most: request lifecycle, continuous batching, KV-cache allocation, paged memory, and testable text generation.
34
+
35
+ The immediate target is a compact core replica for a dense **Qwen3 decoder-only** checkpoint, loaded from a Hugging Face-compatible local directory. Qwen3 provides the important real inference features—RMSNorm, RoPE, grouped-query attention (GQA), Q/K normalization, and SwiGLU—without multimodal or MoE complexity.
36
+
37
+ The milestone-by-milestone plan is in [ROADMAP.md](ROADMAP.md).
38
+
39
+ ## Recommended reading
40
+
41
+ - [AIInfraGuide — AI Infra 全栈知识库](https://caomaolufei.github.io/AIInfraGuide/): a Chinese guide to the AI infrastructure stack, covering CUDA kernel optimization, distributed training, inference optimization, and performance analysis. Useful background for the paged KV cache, FlashAttention, and CUDA Graph work in this project.
42
+
43
+ ## Scope
44
+
45
+ The first track will support:
46
+
47
+ - decoder-only Qwen-style autoregressive generation;
48
+ - continuous batching across several requests;
49
+ - paged KV-cache allocation, sharing, release, and copy-on-write;
50
+ - greedy, temperature, top-k, top-p, EOS, and `max_tokens` controls;
51
+ - deterministic CPU tests with a fake model runner;
52
+ - a single-GPU Qwen3 runner and an offline Python API.
53
+
54
+ It intentionally defers distributed execution, quantization, LoRA, multimodal models, mixture-of-experts, speculative decoding, HTTP serving, custom CUDA/Triton kernels, and production authentication. They belong to the later hardening track, not the initial core replica.
55
+
56
+ ## Repository layout
57
+
58
+ ```text
59
+ .
60
+ ├── pyproject.toml
61
+ ├── README.md
62
+ ├── ROADMAP.md
63
+ ├── src/pico_vllm/
64
+ │ ├── config.py
65
+ │ ├── sampling_params.py
66
+ │ ├── sequence.py
67
+ │ ├── core/ # scheduler, blocks, KV cache
68
+ │ ├── engine/ # LLMEngine and output events
69
+ │ ├── worker/ # model runners
70
+ │ ├── model_executor/ # Qwen layers and weight loader
71
+ │ └── entrypoints/ # CLI and HTTP API
72
+ └── tests/
73
+ ├── unit/
74
+ ├── integration/
75
+ └── conftest.py
76
+ ```
77
+
78
+ ## Installation
79
+
80
+ Install [uv](https://docs.astral.sh/uv/getting-started/installation/) once. From the repository root, create the project environment and install the development tools:
81
+
82
+ ```bash
83
+ uv sync --extra dev
84
+ uv run pytest -q
85
+ ```
86
+
87
+ `uv sync` creates `.venv` and installs the project in editable mode. `uv run` uses that environment even if your shell still shows Conda `(base)`, avoiding unrelated global pytest plugins.
88
+
89
+ Install the Qwen loading dependencies at M3. The web dependencies are only needed at M9:
90
+
91
+ ```bash
92
+ uv sync --extra dev --extra model
93
+ uv sync --extra dev --extra model --extra api
94
+ ```
95
+
96
+ `pyproject.toml` deliberately keeps `transformers`, FastAPI, and Uvicorn optional. The scheduler/cache unit tests must remain fast and independent of network access, model downloads, and CUDA.
97
+
98
+ ## Qwen3 model boundary
99
+
100
+ Start with a small dense Qwen3 model, preferably `Qwen/Qwen3-0.6B`. Parse `config.json` into a local `QwenConfig`; do not make scheduler and cache code depend on Hugging Face configuration objects.
101
+
102
+ ```python
103
+ @dataclass(frozen=True)
104
+ class Qwen3Config:
105
+ vocab_size: int
106
+ hidden_size: int
107
+ intermediate_size: int
108
+ num_hidden_layers: int
109
+ num_attention_heads: int
110
+ num_key_value_heads: int
111
+ head_dim: int
112
+ rms_norm_eps: float
113
+ rope_theta: float
114
+ max_position_embeddings: int
115
+ tie_word_embeddings: bool
116
+ attention_bias: bool
117
+ use_qk_norm: bool
118
+ ```
119
+
120
+ The forward path is:
121
+
122
+ ```text
123
+ token ids → embedding
124
+ → repeated [RMSNorm → GQA attention with RoPE → residual
125
+ RMSNorm → SwiGLU MLP → residual]
126
+ → final RMSNorm → LM head → logits
127
+ ```
128
+
129
+ Important implementation details:
130
+
131
+ - GQA means `num_key_value_heads` may be smaller than `num_attention_heads`.
132
+ - RoPE positions must be identical in prefill and decode paths.
133
+ - Qwen3 needs its configuration read exactly: Q/K normalization, projection bias, and RoPE settings must come from the selected checkpoint rather than from hard-coded assumptions.
134
+ - The initial implementation supports the checkpoint's normal RoPE settings. Long-context RoPE scaling is a later extension.
135
+ - Unit tests use a fake runner only. Real-model parity tests receive a local checkpoint through `PICO_VLLM_QWEN_MODEL=/absolute/path` and skip if absent.
136
+
137
+ ## Component boundaries
138
+
139
+ | Component | Owns | Must not own |
140
+ | --- | --- | --- |
141
+ | `SamplingParams` | validation of generation settings | scheduling |
142
+ | `Sequence` | one request's tokens, state, output | physical blocks |
143
+ | `Scheduler` | waiting/running selection under budgets | model execution |
144
+ | `BlockSpaceManager` | block allocation, sharing, refcounts | request priority |
145
+ | `ModelRunner` | Qwen3 prefill/decode and logits | request lifecycle |
146
+ | `LLMEngine` | orchestration and output events | HTTP protocol |
147
+
148
+ Use the state machine below and allow only the scheduler/engine to transition states:
149
+
150
+ ```text
151
+ WAITING → RUNNING → FINISHED_STOPPED
152
+ └→ FINISHED_LENGTH_CAPPED
153
+ └→ FINISHED_ABORTED
154
+ ```
155
+
156
+ ## How the components fit together
157
+
158
+ Two independent halves meet only through `Sequence.block_table`, a plain list of block ids:
159
+
160
+ - the **bookkeeping half** (CPU, Python objects) decides *which* block ids a sequence owns;
161
+ - the **storage half** (CPU or GPU tensors) holds the actual K/V values at those ids.
162
+
163
+ `BlockManager` never touches a tensor, and `PagedKVCache` never decides who owns a block. In particular, `BlockManager` does **not** call `PagedKVCache`.
164
+
165
+ ```text
166
+ LLMEngine (M5) — calls both halves once per step
167
+ / \
168
+ bookkeeping (CPU) storage + compute (device)
169
+ ───────────────── ──────────────────────────
170
+ Scheduler (M2) ModelRunner (M4)
171
+ │ owns │ owns │ owns
172
+ ▼ ▼ ▼
173
+ BlockManager (M2) Qwen3ForCausalLM PagedKVCache (M4)
174
+ · free queue, ref_count, hash (M3) k_cache / v_cache
175
+ · writes block ids into ─┐ │ each layer [L, N, B, H, D]
176
+ │ ▼ ▲
177
+ ▼ Attention (M4) ────────┘
178
+ Sequence.block_table reads Context: write()/read()
179
+ e.g. [3, 8] slot_mapping,
180
+ │ block_tables, ...
181
+ │ ▲
182
+ └──► ModelRunner.prepare_inputs builds Context
183
+ (set_context / get_context, utils/context.py)
184
+ ```
185
+
186
+ Who calls whom:
187
+
188
+ | Caller | Callee | Why |
189
+ | --- | --- | --- |
190
+ | `LLMEngine.step` | `Scheduler.schedule` | pick the seqs for this step; blocks are reserved here |
191
+ | `Scheduler` | `BlockManager.can_allocate / allocate / may_append / deallocate / hash_blocks` | manage block ids, write `seq.block_table` |
192
+ | `LLMEngine.step` | `ModelRunner.run(seqs, is_prefill)` | run the model on the scheduled batch |
193
+ | `ModelRunner.prepare_inputs` | reads `seq.block_table`, `num_cached_tokens`, ... | turn block ids into `slot_mapping` and a `Context` |
194
+ | `ModelRunner.run` | `set_context` → `model(...)` → `reset_context` | expose the step's metadata to every layer |
195
+ | `Attention.forward` | `get_context().kv_cache.write / read` | store this step's K/V and read the full context |
196
+ | `LLMEngine.step` | `Scheduler.postprocess` | append sampled tokens, hash full blocks, finish/free seqs |
197
+
198
+ One step, end to end:
199
+
200
+ ```text
201
+ 1. Scheduler.schedule() BlockManager gives seq A blocks [3, 8] (ids only)
202
+ 2. ModelRunner.prepare_inputs pos 0..4 of A → slots 12,13,14,15,32 (ids → slots)
203
+ 3. Attention, every layer kv_cache.write(slots, k, v) (values land in k_cache)
204
+ kv_cache.read(block_table=[3, 8], len=5)
205
+ 4. Scheduler.postprocess append token; finished? → deallocate (ids return to free queue;
206
+ k_cache is left untouched)
207
+ ```
208
+
209
+ `PagedKVCache` is allocated once at startup with `num_blocks` equal to the `BlockManager`'s `num_blocks`; the two must agree, since block id `i` in one is index `i` in the other.
210
+
211
+ ## Paged KV cache
212
+
213
+ For `block_size = 4`, ten tokens occupy three logical blocks:
214
+
215
+ ```text
216
+ logical token positions: 0 1 2 3 | 4 5 6 7 | 8 9
217
+ logical block index: 0 | 1 | 2
218
+ block table: [12, 7, 19]
219
+ physical block id: 12 7 19
220
+ ```
221
+
222
+ `Sequence.block_table` maps logical blocks to physical blocks. Prompt-sharing sequences may reference the same blocks. If a sequence appends to a shared partial tail block, it first receives a copied block and changes only its own table: copy-on-write. This behavior needs focused tests; an end-to-end generation assertion is not enough.
223
+
224
+ Core invariants:
225
+
226
+ - A physical block contains exactly `block_size` token positions.
227
+ - `num_computed_tokens <= len(sequence.token_ids)`.
228
+ - A block returns to the free queue only when its reference count becomes zero.
229
+ - A scheduler step does not exceed `max_num_batched_tokens`.
230
+
231
+ ## Test and commit contract
232
+
233
+ For every milestone:
234
+
235
+ 1. Write a failing, focused test.
236
+ 2. Implement the smallest correct behavior.
237
+ 3. Run the milestone tests and then all tests.
238
+ 4. Make one Git commit only after all tests pass.
239
+
240
+ ```bash
241
+ uv run pytest tests/unit -q
242
+ uv run pytest tests/integration -q
243
+ uv run pytest -q
244
+ ```
245
+
246
+ Use a seeded `torch.Generator` whenever a test samples randomly. Unit tests must never need CUDA, an internet connection, or downloaded weights. Real Qwen parity tests have both `integration` and `model` markers; CUDA-only tests additionally have `cuda`.
247
+
248
+ ## Delivery plan and estimated size
249
+
250
+ The roadmap deliberately starts with a compact Qwen3 core replica, then expands only after it works. It excludes model weights, dependencies, generated reports, and deployment infrastructure.
251
+
252
+ | Scope | Production | Tests | Total |
253
+ | --- | ---: | ---: | ---: |
254
+ | Track A M0–M5: eager Qwen3 core | 1,500–2,500 LOC | 600–1,200 LOC | 2,100–3,700 LOC |
255
+ | Track A M6: optional CUDA fast path | +300–800 LOC | +150–350 LOC | +450–1,150 LOC |
256
+ | Track B: hardening, API, metrics, scale-out | +1,500–3,000 LOC | +1,500–2,500 LOC | +3,000–5,500 LOC |
257
+
258
+ The immediate goal is Track A only. It is deliberately close to the compact reference-project scale, but still includes enough focused tests to make scheduler and block-manager bugs visible.
@@ -0,0 +1,228 @@
1
+ # pico-vLLM
2
+
3
+ `pico-vLLM` is a single-GPU, single-process inference engine for learning the core ideas behind vLLM. The project deliberately focuses on the execution path that matters most: request lifecycle, continuous batching, KV-cache allocation, paged memory, and testable text generation.
4
+
5
+ The immediate target is a compact core replica for a dense **Qwen3 decoder-only** checkpoint, loaded from a Hugging Face-compatible local directory. Qwen3 provides the important real inference features—RMSNorm, RoPE, grouped-query attention (GQA), Q/K normalization, and SwiGLU—without multimodal or MoE complexity.
6
+
7
+ The milestone-by-milestone plan is in [ROADMAP.md](ROADMAP.md).
8
+
9
+ ## Recommended reading
10
+
11
+ - [AIInfraGuide — AI Infra 全栈知识库](https://caomaolufei.github.io/AIInfraGuide/): a Chinese guide to the AI infrastructure stack, covering CUDA kernel optimization, distributed training, inference optimization, and performance analysis. Useful background for the paged KV cache, FlashAttention, and CUDA Graph work in this project.
12
+
13
+ ## Scope
14
+
15
+ The first track will support:
16
+
17
+ - decoder-only Qwen-style autoregressive generation;
18
+ - continuous batching across several requests;
19
+ - paged KV-cache allocation, sharing, release, and copy-on-write;
20
+ - greedy, temperature, top-k, top-p, EOS, and `max_tokens` controls;
21
+ - deterministic CPU tests with a fake model runner;
22
+ - a single-GPU Qwen3 runner and an offline Python API.
23
+
24
+ It intentionally defers distributed execution, quantization, LoRA, multimodal models, mixture-of-experts, speculative decoding, HTTP serving, custom CUDA/Triton kernels, and production authentication. They belong to the later hardening track, not the initial core replica.
25
+
26
+ ## Repository layout
27
+
28
+ ```text
29
+ .
30
+ ├── pyproject.toml
31
+ ├── README.md
32
+ ├── ROADMAP.md
33
+ ├── src/pico_vllm/
34
+ │ ├── config.py
35
+ │ ├── sampling_params.py
36
+ │ ├── sequence.py
37
+ │ ├── core/ # scheduler, blocks, KV cache
38
+ │ ├── engine/ # LLMEngine and output events
39
+ │ ├── worker/ # model runners
40
+ │ ├── model_executor/ # Qwen layers and weight loader
41
+ │ └── entrypoints/ # CLI and HTTP API
42
+ └── tests/
43
+ ├── unit/
44
+ ├── integration/
45
+ └── conftest.py
46
+ ```
47
+
48
+ ## Installation
49
+
50
+ Install [uv](https://docs.astral.sh/uv/getting-started/installation/) once. From the repository root, create the project environment and install the development tools:
51
+
52
+ ```bash
53
+ uv sync --extra dev
54
+ uv run pytest -q
55
+ ```
56
+
57
+ `uv sync` creates `.venv` and installs the project in editable mode. `uv run` uses that environment even if your shell still shows Conda `(base)`, avoiding unrelated global pytest plugins.
58
+
59
+ Install the Qwen loading dependencies at M3. The web dependencies are only needed at M9:
60
+
61
+ ```bash
62
+ uv sync --extra dev --extra model
63
+ uv sync --extra dev --extra model --extra api
64
+ ```
65
+
66
+ `pyproject.toml` deliberately keeps `transformers`, FastAPI, and Uvicorn optional. The scheduler/cache unit tests must remain fast and independent of network access, model downloads, and CUDA.
67
+
68
+ ## Qwen3 model boundary
69
+
70
+ Start with a small dense Qwen3 model, preferably `Qwen/Qwen3-0.6B`. Parse `config.json` into a local `QwenConfig`; do not make scheduler and cache code depend on Hugging Face configuration objects.
71
+
72
+ ```python
73
+ @dataclass(frozen=True)
74
+ class Qwen3Config:
75
+ vocab_size: int
76
+ hidden_size: int
77
+ intermediate_size: int
78
+ num_hidden_layers: int
79
+ num_attention_heads: int
80
+ num_key_value_heads: int
81
+ head_dim: int
82
+ rms_norm_eps: float
83
+ rope_theta: float
84
+ max_position_embeddings: int
85
+ tie_word_embeddings: bool
86
+ attention_bias: bool
87
+ use_qk_norm: bool
88
+ ```
89
+
90
+ The forward path is:
91
+
92
+ ```text
93
+ token ids → embedding
94
+ → repeated [RMSNorm → GQA attention with RoPE → residual
95
+ RMSNorm → SwiGLU MLP → residual]
96
+ → final RMSNorm → LM head → logits
97
+ ```
98
+
99
+ Important implementation details:
100
+
101
+ - GQA means `num_key_value_heads` may be smaller than `num_attention_heads`.
102
+ - RoPE positions must be identical in prefill and decode paths.
103
+ - Qwen3 needs its configuration read exactly: Q/K normalization, projection bias, and RoPE settings must come from the selected checkpoint rather than from hard-coded assumptions.
104
+ - The initial implementation supports the checkpoint's normal RoPE settings. Long-context RoPE scaling is a later extension.
105
+ - Unit tests use a fake runner only. Real-model parity tests receive a local checkpoint through `PICO_VLLM_QWEN_MODEL=/absolute/path` and skip if absent.
106
+
107
+ ## Component boundaries
108
+
109
+ | Component | Owns | Must not own |
110
+ | --- | --- | --- |
111
+ | `SamplingParams` | validation of generation settings | scheduling |
112
+ | `Sequence` | one request's tokens, state, output | physical blocks |
113
+ | `Scheduler` | waiting/running selection under budgets | model execution |
114
+ | `BlockSpaceManager` | block allocation, sharing, refcounts | request priority |
115
+ | `ModelRunner` | Qwen3 prefill/decode and logits | request lifecycle |
116
+ | `LLMEngine` | orchestration and output events | HTTP protocol |
117
+
118
+ Use the state machine below and allow only the scheduler/engine to transition states:
119
+
120
+ ```text
121
+ WAITING → RUNNING → FINISHED_STOPPED
122
+ └→ FINISHED_LENGTH_CAPPED
123
+ └→ FINISHED_ABORTED
124
+ ```
125
+
126
+ ## How the components fit together
127
+
128
+ Two independent halves meet only through `Sequence.block_table`, a plain list of block ids:
129
+
130
+ - the **bookkeeping half** (CPU, Python objects) decides *which* block ids a sequence owns;
131
+ - the **storage half** (CPU or GPU tensors) holds the actual K/V values at those ids.
132
+
133
+ `BlockManager` never touches a tensor, and `PagedKVCache` never decides who owns a block. In particular, `BlockManager` does **not** call `PagedKVCache`.
134
+
135
+ ```text
136
+ LLMEngine (M5) — calls both halves once per step
137
+ / \
138
+ bookkeeping (CPU) storage + compute (device)
139
+ ───────────────── ──────────────────────────
140
+ Scheduler (M2) ModelRunner (M4)
141
+ │ owns │ owns │ owns
142
+ ▼ ▼ ▼
143
+ BlockManager (M2) Qwen3ForCausalLM PagedKVCache (M4)
144
+ · free queue, ref_count, hash (M3) k_cache / v_cache
145
+ · writes block ids into ─┐ │ each layer [L, N, B, H, D]
146
+ │ ▼ ▲
147
+ ▼ Attention (M4) ────────┘
148
+ Sequence.block_table reads Context: write()/read()
149
+ e.g. [3, 8] slot_mapping,
150
+ │ block_tables, ...
151
+ │ ▲
152
+ └──► ModelRunner.prepare_inputs builds Context
153
+ (set_context / get_context, utils/context.py)
154
+ ```
155
+
156
+ Who calls whom:
157
+
158
+ | Caller | Callee | Why |
159
+ | --- | --- | --- |
160
+ | `LLMEngine.step` | `Scheduler.schedule` | pick the seqs for this step; blocks are reserved here |
161
+ | `Scheduler` | `BlockManager.can_allocate / allocate / may_append / deallocate / hash_blocks` | manage block ids, write `seq.block_table` |
162
+ | `LLMEngine.step` | `ModelRunner.run(seqs, is_prefill)` | run the model on the scheduled batch |
163
+ | `ModelRunner.prepare_inputs` | reads `seq.block_table`, `num_cached_tokens`, ... | turn block ids into `slot_mapping` and a `Context` |
164
+ | `ModelRunner.run` | `set_context` → `model(...)` → `reset_context` | expose the step's metadata to every layer |
165
+ | `Attention.forward` | `get_context().kv_cache.write / read` | store this step's K/V and read the full context |
166
+ | `LLMEngine.step` | `Scheduler.postprocess` | append sampled tokens, hash full blocks, finish/free seqs |
167
+
168
+ One step, end to end:
169
+
170
+ ```text
171
+ 1. Scheduler.schedule() BlockManager gives seq A blocks [3, 8] (ids only)
172
+ 2. ModelRunner.prepare_inputs pos 0..4 of A → slots 12,13,14,15,32 (ids → slots)
173
+ 3. Attention, every layer kv_cache.write(slots, k, v) (values land in k_cache)
174
+ kv_cache.read(block_table=[3, 8], len=5)
175
+ 4. Scheduler.postprocess append token; finished? → deallocate (ids return to free queue;
176
+ k_cache is left untouched)
177
+ ```
178
+
179
+ `PagedKVCache` is allocated once at startup with `num_blocks` equal to the `BlockManager`'s `num_blocks`; the two must agree, since block id `i` in one is index `i` in the other.
180
+
181
+ ## Paged KV cache
182
+
183
+ For `block_size = 4`, ten tokens occupy three logical blocks:
184
+
185
+ ```text
186
+ logical token positions: 0 1 2 3 | 4 5 6 7 | 8 9
187
+ logical block index: 0 | 1 | 2
188
+ block table: [12, 7, 19]
189
+ physical block id: 12 7 19
190
+ ```
191
+
192
+ `Sequence.block_table` maps logical blocks to physical blocks. Prompt-sharing sequences may reference the same blocks. If a sequence appends to a shared partial tail block, it first receives a copied block and changes only its own table: copy-on-write. This behavior needs focused tests; an end-to-end generation assertion is not enough.
193
+
194
+ Core invariants:
195
+
196
+ - A physical block contains exactly `block_size` token positions.
197
+ - `num_computed_tokens <= len(sequence.token_ids)`.
198
+ - A block returns to the free queue only when its reference count becomes zero.
199
+ - A scheduler step does not exceed `max_num_batched_tokens`.
200
+
201
+ ## Test and commit contract
202
+
203
+ For every milestone:
204
+
205
+ 1. Write a failing, focused test.
206
+ 2. Implement the smallest correct behavior.
207
+ 3. Run the milestone tests and then all tests.
208
+ 4. Make one Git commit only after all tests pass.
209
+
210
+ ```bash
211
+ uv run pytest tests/unit -q
212
+ uv run pytest tests/integration -q
213
+ uv run pytest -q
214
+ ```
215
+
216
+ Use a seeded `torch.Generator` whenever a test samples randomly. Unit tests must never need CUDA, an internet connection, or downloaded weights. Real Qwen parity tests have both `integration` and `model` markers; CUDA-only tests additionally have `cuda`.
217
+
218
+ ## Delivery plan and estimated size
219
+
220
+ The roadmap deliberately starts with a compact Qwen3 core replica, then expands only after it works. It excludes model weights, dependencies, generated reports, and deployment infrastructure.
221
+
222
+ | Scope | Production | Tests | Total |
223
+ | --- | ---: | ---: | ---: |
224
+ | Track A M0–M5: eager Qwen3 core | 1,500–2,500 LOC | 600–1,200 LOC | 2,100–3,700 LOC |
225
+ | Track A M6: optional CUDA fast path | +300–800 LOC | +150–350 LOC | +450–1,150 LOC |
226
+ | Track B: hardening, API, metrics, scale-out | +1,500–3,000 LOC | +1,500–2,500 LOC | +3,000–5,500 LOC |
227
+
228
+ The immediate goal is Track A only. It is deliberately close to the compact reference-project scale, but still includes enough focused tests to make scheduler and block-manager bugs visible.