thoughtformula 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- thoughtformula-0.3.0/PKG-INFO +152 -0
- thoughtformula-0.3.0/README.md +129 -0
- thoughtformula-0.3.0/pyproject.toml +51 -0
- thoughtformula-0.3.0/pyproject.toml.orig +48 -0
- thoughtformula-0.3.0/src/thoughtformula/__init__.py +39 -0
- thoughtformula-0.3.0/src/thoughtformula/batching.py +220 -0
- thoughtformula-0.3.0/src/thoughtformula/cache.py +318 -0
- thoughtformula-0.3.0/src/thoughtformula/cli.py +357 -0
- thoughtformula-0.3.0/src/thoughtformula/draft.py +57 -0
- thoughtformula-0.3.0/src/thoughtformula/gradcheck.py +152 -0
- thoughtformula-0.3.0/src/thoughtformula/medusa.py +138 -0
- thoughtformula-0.3.0/src/thoughtformula/micro_gpt_checkpoint.json +1 -0
- thoughtformula-0.3.0/src/thoughtformula/micro_gpt_v8.py +12 -0
- thoughtformula-0.3.0/src/thoughtformula/model.py +1198 -0
- thoughtformula-0.3.0/src/thoughtformula/rag.py +321 -0
- thoughtformula-0.3.0/src/thoughtformula/tensor.py +1192 -0
- thoughtformula-0.3.0/src/thoughtformula/tokenizer.py +171 -0
- thoughtformula-0.3.0/src/thoughtformula/training.py +246 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: thoughtformula
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Pure-Python generative Transformer architecture with FlashAttention, Paged KV-Cache, BitNet b1.58, KTO, Medusa, and Continuous Batching
|
|
5
|
+
Keywords: transformer,pure-python,deep-learning,attention,flashattention,moe,lora,speculative-decoding,kto,dpo,bitnet,rag
|
|
6
|
+
Author: ellainng
|
|
7
|
+
Author-email: ellainng <299958882+ellainng@users.noreply.github.com>
|
|
8
|
+
License: MIT
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Project-URL: Homepage, https://github.com/wispsy/thoughtformula
|
|
19
|
+
Project-URL: Repository, https://github.com/wispsy/thoughtformula
|
|
20
|
+
Project-URL: Issues, https://github.com/wispsy/thoughtformula/issues
|
|
21
|
+
Project-URL: Documentation, https://github.com/wispsy/thoughtformula#readme
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# Thoughtformula ðŸ§
|
|
25
|
+
|
|
26
|
+
> Pure-Python generative Transformer architecture featuring FlashAttention online softmax tiling, vLLM-style Paged KV-Cache, BitNet b1.58 ternary quantization, Grouped Query Attention (GQA), Rotary Positional Embeddings (RoPE), Sparse Mixture of Experts (MoE SwiGLU), LoRA continual learning, Kahneman-Tversky Optimization (KTO), Medusa speculative decoding, persistent binary index (.tfidx), and continuous batching. Built exclusively with Python standard library built-ins.
|
|
27
|
+
|
|
28
|
+
[]()
|
|
29
|
+
[]()
|
|
30
|
+
[-success.svg)]()
|
|
31
|
+
[]()
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## Key Features
|
|
36
|
+
|
|
37
|
+
- **Standard Library Autograd (`FlatTensor`)**: 1D contiguous float64 training buffer with symmetric Int8 inference quantization, BitNet b1.58 ternary quantization, and reverse-mode automatic differentiation.
|
|
38
|
+
- **FlashAttention Online Softmax Tiling**: Dao et al. streaming online softmax eliminates $T \times T$ attention matrix materialization, maintaining $O(N)$ memory across long context windows.
|
|
39
|
+
- **vLLM-Style Paged KV-Cache (`PagedKVCache`)**: Fixed-size physical memory pages (e.g., 16 tokens/block), block allocator, and page tables supporting non-contiguous allocation and $O(1)$ prompt prefix sharing.
|
|
40
|
+
- **BitNet b1.58 Ternary Quantization**: Weights in $\{-1, 0, 1\}$ with scale $\gamma = \text{mean}(|W|)$, 2-bit bitpacking (4 weights/byte), and addition-only matrix multiplication.
|
|
41
|
+
- **Kahneman-Tversky Optimization (KTO)**: Unpaired binary human preference alignment (`q :: a ++` and `q :: a --`) using prospect theory utility loss and frozen base Int8 weights as reference.
|
|
42
|
+
- **Interactive Streaming Token Generator**: Real-time typewriter output streaming via `generate_stream()`, `generate_speculative_stream()`, and `generate_medusa_stream()`.
|
|
43
|
+
- **Medusa Multi-Head Speculative Drafter**: Lightweight auxiliary heads predicting multiple future tokens simultaneously directly from the final hidden state without extra transformer layers.
|
|
44
|
+
- **Persistent Binary Index Format (`.tfidx`)**: High-performance binary container persisting precomputed dense embeddings and term inverted postings for large-scale hybrid RAG retrieval.
|
|
45
|
+
- **Continuous Batching Engine (`ContinuousBatchScheduler`)**: Synchronous iteration-level continuous batcher packing active requests into unified forward passes, amortizing parameter loading across concurrent sessions.
|
|
46
|
+
- **C-Speed Matrix Multiplication**: Accelerated with compiled C `math.sumprod` over contiguous transposed weight slices (5x-12x pure-Python speedup).
|
|
47
|
+
- **Grouped Query Attention (GQA) & Precomputed RoPE**: Memory-efficient query/key-value head grouping with precomputed `cos_cache` / `sin_cache` trigonometric lookup tables.
|
|
48
|
+
- **Sparse Mixture of Experts (MoE)**: 4 SwiGLU experts with true sparse token dispatch and Switch Transformer auxiliary load-balancing loss preventing expert collapse.
|
|
49
|
+
- **Zero-Memory Direct Preference Optimization (DPO)**: Align model responses (`q :: good || bad`) using frozen Int8 base weights as reference policy $\pi_{\text{ref}}$ with 0 extra RAM.
|
|
50
|
+
- **Decoupled Weight Decay (AdamW)**: Bounded weight magnitudes preventing Int8 scale drift.
|
|
51
|
+
- **Lossless Speculative Decoding**: Leviathan et al. (2023) rejection sampling with N-gram proposal distributions and Int8 KV-cache rollback.
|
|
52
|
+
- **Hybrid Dense + Lexical RAG**: Okapi BM25 combined with mean-pooled `tok_emb` dense embeddings via Reciprocal Rank Fusion (RRF).
|
|
53
|
+
- **Binary `.mgpt` Container**: Python standard library `zipfile` binary container eliminating 33% Base64 storage overhead.
|
|
54
|
+
- **Built-in Gradient Checker**: Finite-difference calculus test harness (`/test`) verifying all analytical gradients to $< 10^{-10}$.
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Quickstart
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
# Install via pip
|
|
62
|
+
pip install thoughtformula
|
|
63
|
+
|
|
64
|
+
# Run CLI directly
|
|
65
|
+
thoughtformula --benchmark
|
|
66
|
+
|
|
67
|
+
# Or run interactive typewriter REPL
|
|
68
|
+
thoughtformula
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### Run from Source
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
git clone https://github.com/wispsy/thoughtformula.git
|
|
75
|
+
cd thoughtformula
|
|
76
|
+
|
|
77
|
+
# Run speculative decoding benchmark
|
|
78
|
+
PYTHONPATH=src python3 -m thoughtformula.cli --benchmark
|
|
79
|
+
|
|
80
|
+
# Run interactive typewriter REPL
|
|
81
|
+
PYTHONPATH=src python3 -m thoughtformula.cli
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### REPL Commands & Syntax
|
|
85
|
+
|
|
86
|
+
| Syntax | Action |
|
|
87
|
+
|---|---|
|
|
88
|
+
| `question?` | RAG retrieval + speculative decoding typewriter output |
|
|
89
|
+
| `q :: answer` | Continual online LoRA adaptation |
|
|
90
|
+
| `q :: good \|\| bad` | Zero-Memory LoRA Direct Preference Optimization (DPO) |
|
|
91
|
+
| `q :: answer ++` | Kahneman-Tversky Optimization (KTO) Desirable feedback |
|
|
92
|
+
| `q :: answer --` | Kahneman-Tversky Optimization (KTO) Undesirable feedback |
|
|
93
|
+
| `/test` | Run finite-difference calculus gradient verification |
|
|
94
|
+
| `/merge` | Merge LoRA adapters into base weights |
|
|
95
|
+
| `/stats` | Display model parameter count and cache metrics |
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## Project Structure
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
thoughtformula/
|
|
103
|
+
├── pyproject.toml
|
|
104
|
+
├── README.md
|
|
105
|
+
├── docs/
|
|
106
|
+
│ ├── architecture.md
|
|
107
|
+
│ ├── modules.md
|
|
108
|
+
│ └── quickstart.md
|
|
109
|
+
├── src/
|
|
110
|
+
│ └── thoughtformula/
|
|
111
|
+
│ ├── __init__.py
|
|
112
|
+
│ ├── batching.py
|
|
113
|
+
│ ├── cache.py
|
|
114
|
+
│ ├── cli.py
|
|
115
|
+
│ ├── draft.py
|
|
116
|
+
│ ├── gradcheck.py
|
|
117
|
+
│ ├── medusa.py
|
|
118
|
+
│ ├── micro_gpt_checkpoint.json
|
|
119
|
+
│ ├── micro_gpt_v8.py
|
|
120
|
+
│ ├── model.py
|
|
121
|
+
│ ├── rag.py
|
|
122
|
+
│ ├── tensor.py
|
|
123
|
+
│ ├── tokenizer.py
|
|
124
|
+
│ └── training.py
|
|
125
|
+
└── tests/
|
|
126
|
+
├── test_batching.py
|
|
127
|
+
├── test_cache.py
|
|
128
|
+
├── test_draft.py
|
|
129
|
+
├── test_gradcheck.py
|
|
130
|
+
├── test_medusa.py
|
|
131
|
+
├── test_model.py
|
|
132
|
+
├── test_rag.py
|
|
133
|
+
├── test_tensor.py
|
|
134
|
+
├── test_tokenizer.py
|
|
135
|
+
└── test_training.py
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## Running Tests
|
|
141
|
+
|
|
142
|
+
All 64 unit tests run using Python's built-in `unittest` framework with zero external dependencies:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
PYTHONPATH=src python3 -m unittest discover tests
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
## License
|
|
151
|
+
|
|
152
|
+
MIT
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# Thoughtformula ðŸ§
|
|
2
|
+
|
|
3
|
+
> Pure-Python generative Transformer architecture featuring FlashAttention online softmax tiling, vLLM-style Paged KV-Cache, BitNet b1.58 ternary quantization, Grouped Query Attention (GQA), Rotary Positional Embeddings (RoPE), Sparse Mixture of Experts (MoE SwiGLU), LoRA continual learning, Kahneman-Tversky Optimization (KTO), Medusa speculative decoding, persistent binary index (.tfidx), and continuous batching. Built exclusively with Python standard library built-ins.
|
|
4
|
+
|
|
5
|
+
[]()
|
|
6
|
+
[]()
|
|
7
|
+
[-success.svg)]()
|
|
8
|
+
[]()
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Key Features
|
|
13
|
+
|
|
14
|
+
- **Standard Library Autograd (`FlatTensor`)**: 1D contiguous float64 training buffer with symmetric Int8 inference quantization, BitNet b1.58 ternary quantization, and reverse-mode automatic differentiation.
|
|
15
|
+
- **FlashAttention Online Softmax Tiling**: Dao et al. streaming online softmax eliminates $T \times T$ attention matrix materialization, maintaining $O(N)$ memory across long context windows.
|
|
16
|
+
- **vLLM-Style Paged KV-Cache (`PagedKVCache`)**: Fixed-size physical memory pages (e.g., 16 tokens/block), block allocator, and page tables supporting non-contiguous allocation and $O(1)$ prompt prefix sharing.
|
|
17
|
+
- **BitNet b1.58 Ternary Quantization**: Weights in $\{-1, 0, 1\}$ with scale $\gamma = \text{mean}(|W|)$, 2-bit bitpacking (4 weights/byte), and addition-only matrix multiplication.
|
|
18
|
+
- **Kahneman-Tversky Optimization (KTO)**: Unpaired binary human preference alignment (`q :: a ++` and `q :: a --`) using prospect theory utility loss and frozen base Int8 weights as reference.
|
|
19
|
+
- **Interactive Streaming Token Generator**: Real-time typewriter output streaming via `generate_stream()`, `generate_speculative_stream()`, and `generate_medusa_stream()`.
|
|
20
|
+
- **Medusa Multi-Head Speculative Drafter**: Lightweight auxiliary heads predicting multiple future tokens simultaneously directly from the final hidden state without extra transformer layers.
|
|
21
|
+
- **Persistent Binary Index Format (`.tfidx`)**: High-performance binary container persisting precomputed dense embeddings and term inverted postings for large-scale hybrid RAG retrieval.
|
|
22
|
+
- **Continuous Batching Engine (`ContinuousBatchScheduler`)**: Synchronous iteration-level continuous batcher packing active requests into unified forward passes, amortizing parameter loading across concurrent sessions.
|
|
23
|
+
- **C-Speed Matrix Multiplication**: Accelerated with compiled C `math.sumprod` over contiguous transposed weight slices (5x-12x pure-Python speedup).
|
|
24
|
+
- **Grouped Query Attention (GQA) & Precomputed RoPE**: Memory-efficient query/key-value head grouping with precomputed `cos_cache` / `sin_cache` trigonometric lookup tables.
|
|
25
|
+
- **Sparse Mixture of Experts (MoE)**: 4 SwiGLU experts with true sparse token dispatch and Switch Transformer auxiliary load-balancing loss preventing expert collapse.
|
|
26
|
+
- **Zero-Memory Direct Preference Optimization (DPO)**: Align model responses (`q :: good || bad`) using frozen Int8 base weights as reference policy $\pi_{\text{ref}}$ with 0 extra RAM.
|
|
27
|
+
- **Decoupled Weight Decay (AdamW)**: Bounded weight magnitudes preventing Int8 scale drift.
|
|
28
|
+
- **Lossless Speculative Decoding**: Leviathan et al. (2023) rejection sampling with N-gram proposal distributions and Int8 KV-cache rollback.
|
|
29
|
+
- **Hybrid Dense + Lexical RAG**: Okapi BM25 combined with mean-pooled `tok_emb` dense embeddings via Reciprocal Rank Fusion (RRF).
|
|
30
|
+
- **Binary `.mgpt` Container**: Python standard library `zipfile` binary container eliminating 33% Base64 storage overhead.
|
|
31
|
+
- **Built-in Gradient Checker**: Finite-difference calculus test harness (`/test`) verifying all analytical gradients to $< 10^{-10}$.
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## Quickstart
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
# Install via pip
|
|
39
|
+
pip install thoughtformula
|
|
40
|
+
|
|
41
|
+
# Run CLI directly
|
|
42
|
+
thoughtformula --benchmark
|
|
43
|
+
|
|
44
|
+
# Or run interactive typewriter REPL
|
|
45
|
+
thoughtformula
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
### Run from Source
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
git clone https://github.com/wispsy/thoughtformula.git
|
|
52
|
+
cd thoughtformula
|
|
53
|
+
|
|
54
|
+
# Run speculative decoding benchmark
|
|
55
|
+
PYTHONPATH=src python3 -m thoughtformula.cli --benchmark
|
|
56
|
+
|
|
57
|
+
# Run interactive typewriter REPL
|
|
58
|
+
PYTHONPATH=src python3 -m thoughtformula.cli
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### REPL Commands & Syntax
|
|
62
|
+
|
|
63
|
+
| Syntax | Action |
|
|
64
|
+
|---|---|
|
|
65
|
+
| `question?` | RAG retrieval + speculative decoding typewriter output |
|
|
66
|
+
| `q :: answer` | Continual online LoRA adaptation |
|
|
67
|
+
| `q :: good \|\| bad` | Zero-Memory LoRA Direct Preference Optimization (DPO) |
|
|
68
|
+
| `q :: answer ++` | Kahneman-Tversky Optimization (KTO) Desirable feedback |
|
|
69
|
+
| `q :: answer --` | Kahneman-Tversky Optimization (KTO) Undesirable feedback |
|
|
70
|
+
| `/test` | Run finite-difference calculus gradient verification |
|
|
71
|
+
| `/merge` | Merge LoRA adapters into base weights |
|
|
72
|
+
| `/stats` | Display model parameter count and cache metrics |
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
## Project Structure
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
thoughtformula/
|
|
80
|
+
├── pyproject.toml
|
|
81
|
+
├── README.md
|
|
82
|
+
├── docs/
|
|
83
|
+
│ ├── architecture.md
|
|
84
|
+
│ ├── modules.md
|
|
85
|
+
│ └── quickstart.md
|
|
86
|
+
├── src/
|
|
87
|
+
│ └── thoughtformula/
|
|
88
|
+
│ ├── __init__.py
|
|
89
|
+
│ ├── batching.py
|
|
90
|
+
│ ├── cache.py
|
|
91
|
+
│ ├── cli.py
|
|
92
|
+
│ ├── draft.py
|
|
93
|
+
│ ├── gradcheck.py
|
|
94
|
+
│ ├── medusa.py
|
|
95
|
+
│ ├── micro_gpt_checkpoint.json
|
|
96
|
+
│ ├── micro_gpt_v8.py
|
|
97
|
+
│ ├── model.py
|
|
98
|
+
│ ├── rag.py
|
|
99
|
+
│ ├── tensor.py
|
|
100
|
+
│ ├── tokenizer.py
|
|
101
|
+
│ └── training.py
|
|
102
|
+
└── tests/
|
|
103
|
+
├── test_batching.py
|
|
104
|
+
├── test_cache.py
|
|
105
|
+
├── test_draft.py
|
|
106
|
+
├── test_gradcheck.py
|
|
107
|
+
├── test_medusa.py
|
|
108
|
+
├── test_model.py
|
|
109
|
+
├── test_rag.py
|
|
110
|
+
├── test_tensor.py
|
|
111
|
+
├── test_tokenizer.py
|
|
112
|
+
└── test_training.py
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## Running Tests
|
|
118
|
+
|
|
119
|
+
All 64 unit tests run using Python's built-in `unittest` framework with zero external dependencies:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
PYTHONPATH=src python3 -m unittest discover tests
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
## License
|
|
128
|
+
|
|
129
|
+
MIT
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "thoughtformula"
|
|
3
|
+
version = "0.3.0"
|
|
4
|
+
description = "Pure-Python generative Transformer architecture with FlashAttention, Paged KV-Cache, BitNet b1.58, KTO, Medusa, and Continuous Batching"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
dependencies = []
|
|
8
|
+
keywords = [
|
|
9
|
+
"transformer",
|
|
10
|
+
"pure-python",
|
|
11
|
+
"deep-learning",
|
|
12
|
+
"attention",
|
|
13
|
+
"flashattention",
|
|
14
|
+
"moe",
|
|
15
|
+
"lora",
|
|
16
|
+
"speculative-decoding",
|
|
17
|
+
"kto",
|
|
18
|
+
"dpo",
|
|
19
|
+
"bitnet",
|
|
20
|
+
"rag",
|
|
21
|
+
]
|
|
22
|
+
classifiers = [
|
|
23
|
+
"Development Status :: 4 - Beta",
|
|
24
|
+
"Intended Audience :: Developers",
|
|
25
|
+
"Intended Audience :: Science/Research",
|
|
26
|
+
"Programming Language :: Python :: 3",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Programming Language :: Python :: 3.13",
|
|
29
|
+
"Programming Language :: Python :: 3.14",
|
|
30
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.license]
|
|
34
|
+
text = "MIT"
|
|
35
|
+
|
|
36
|
+
[[project.authors]]
|
|
37
|
+
name = "ellainng"
|
|
38
|
+
email = "299958882+ellainng@users.noreply.github.com"
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/wispsy/thoughtformula"
|
|
42
|
+
Repository = "https://github.com/wispsy/thoughtformula"
|
|
43
|
+
Issues = "https://github.com/wispsy/thoughtformula/issues"
|
|
44
|
+
Documentation = "https://github.com/wispsy/thoughtformula#readme"
|
|
45
|
+
|
|
46
|
+
[project.scripts]
|
|
47
|
+
thoughtformula = "thoughtformula.cli:main"
|
|
48
|
+
|
|
49
|
+
[build-system]
|
|
50
|
+
requires = ["uv_build>=0.13.6,<0.14.0"]
|
|
51
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "thoughtformula"
|
|
3
|
+
version = "0.3.0"
|
|
4
|
+
description = "Pure-Python generative Transformer architecture with FlashAttention, Paged KV-Cache, BitNet b1.58, KTO, Medusa, and Continuous Batching"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = { text = "MIT" }
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "ellainng", email = "299958882+ellainng@users.noreply.github.com" }
|
|
9
|
+
]
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
dependencies = []
|
|
12
|
+
keywords = [
|
|
13
|
+
"transformer",
|
|
14
|
+
"pure-python",
|
|
15
|
+
"deep-learning",
|
|
16
|
+
"attention",
|
|
17
|
+
"flashattention",
|
|
18
|
+
"moe",
|
|
19
|
+
"lora",
|
|
20
|
+
"speculative-decoding",
|
|
21
|
+
"kto",
|
|
22
|
+
"dpo",
|
|
23
|
+
"bitnet",
|
|
24
|
+
"rag",
|
|
25
|
+
]
|
|
26
|
+
classifiers = [
|
|
27
|
+
"Development Status :: 4 - Beta",
|
|
28
|
+
"Intended Audience :: Developers",
|
|
29
|
+
"Intended Audience :: Science/Research",
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Programming Language :: Python :: 3.12",
|
|
32
|
+
"Programming Language :: Python :: 3.13",
|
|
33
|
+
"Programming Language :: Python :: 3.14",
|
|
34
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://github.com/wispsy/thoughtformula"
|
|
39
|
+
Repository = "https://github.com/wispsy/thoughtformula"
|
|
40
|
+
Issues = "https://github.com/wispsy/thoughtformula/issues"
|
|
41
|
+
Documentation = "https://github.com/wispsy/thoughtformula#readme"
|
|
42
|
+
|
|
43
|
+
[project.scripts]
|
|
44
|
+
thoughtformula = "thoughtformula.cli:main"
|
|
45
|
+
|
|
46
|
+
[build-system]
|
|
47
|
+
requires = ["uv_build>=0.13.6,<0.14.0"]
|
|
48
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Thoughtformula: Pure-Python Transformer with GQA, Sparse MoE SwiGLU, Int8 quantization, LoRA, and Speculative Decoding."""
|
|
2
|
+
|
|
3
|
+
from .tensor import FlatTensor
|
|
4
|
+
from .draft import NGramDraftModel
|
|
5
|
+
from .rag import PurePythonBM25RAG, HybridDenseBM25RAG
|
|
6
|
+
from .tokenizer import BPETokenizer
|
|
7
|
+
from .model import ModernMicroGPT
|
|
8
|
+
from .medusa import MedusaDrafter, MedusaHead
|
|
9
|
+
from .cache import PagedKVCache, PagedKVCacheManager, BlockAllocator
|
|
10
|
+
from .batching import ContinuousBatchScheduler, InferenceRequest
|
|
11
|
+
from .training import get_cosine_lr, build_training_sequences, continual_learn_lora_turn, train_dpo_turn, train_kto_turn
|
|
12
|
+
from .gradcheck import run_gradient_checks
|
|
13
|
+
from .cli import main
|
|
14
|
+
|
|
15
|
+
__version__ = "0.3.0"
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"FlatTensor",
|
|
19
|
+
"NGramDraftModel",
|
|
20
|
+
"PurePythonBM25RAG",
|
|
21
|
+
"HybridDenseBM25RAG",
|
|
22
|
+
"BPETokenizer",
|
|
23
|
+
"ModernMicroGPT",
|
|
24
|
+
"MedusaDrafter",
|
|
25
|
+
"MedusaHead",
|
|
26
|
+
"PagedKVCache",
|
|
27
|
+
"PagedKVCacheManager",
|
|
28
|
+
"BlockAllocator",
|
|
29
|
+
"ContinuousBatchScheduler",
|
|
30
|
+
"InferenceRequest",
|
|
31
|
+
"get_cosine_lr",
|
|
32
|
+
"build_training_sequences",
|
|
33
|
+
"continual_learn_lora_turn",
|
|
34
|
+
"train_dpo_turn",
|
|
35
|
+
"train_kto_turn",
|
|
36
|
+
"run_gradient_checks",
|
|
37
|
+
"main",
|
|
38
|
+
"__version__",
|
|
39
|
+
]
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Pure-Python continuous batching scheduler for concurrent multi-request inference.
|
|
2
|
+
|
|
3
|
+
Implements iteration-level scheduling (Orca / vLLM style):
|
|
4
|
+
- Packs concurrent active decode tokens into unified forward passes.
|
|
5
|
+
- Dynamically admits pending requests as active requests finish.
|
|
6
|
+
- Isolates attention memory per sequence KV-cache while amortizing linear weight projections.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from array import array
|
|
10
|
+
import uuid
|
|
11
|
+
from typing import Generator
|
|
12
|
+
|
|
13
|
+
from .model import ModernMicroGPT
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class InferenceRequest:
|
|
17
|
+
"""Represents a single generation request managed by the continuous batcher."""
|
|
18
|
+
|
|
19
|
+
__slots__ = (
|
|
20
|
+
"request_id",
|
|
21
|
+
"prompt_ids",
|
|
22
|
+
"max_new_tokens",
|
|
23
|
+
"temperature",
|
|
24
|
+
"top_k",
|
|
25
|
+
"top_p",
|
|
26
|
+
"stop_token_id",
|
|
27
|
+
"use_int8",
|
|
28
|
+
"kv_cache",
|
|
29
|
+
"output_tokens",
|
|
30
|
+
"is_finished",
|
|
31
|
+
"prefilled",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
prompt_ids: list[int],
|
|
37
|
+
request_id: str | int | None = None,
|
|
38
|
+
max_new_tokens: int = 24,
|
|
39
|
+
temperature: float = 0.05,
|
|
40
|
+
top_k: int = 6,
|
|
41
|
+
top_p: float = 0.9,
|
|
42
|
+
stop_token_id: int | None = None,
|
|
43
|
+
use_int8: bool = True,
|
|
44
|
+
kv_cache=None,
|
|
45
|
+
):
|
|
46
|
+
self.request_id = str(uuid.uuid4())[:8] if request_id is None else request_id
|
|
47
|
+
self.prompt_ids = list(prompt_ids)
|
|
48
|
+
self.max_new_tokens = max_new_tokens
|
|
49
|
+
self.temperature = temperature
|
|
50
|
+
self.top_k = top_k
|
|
51
|
+
self.top_p = top_p
|
|
52
|
+
self.stop_token_id = stop_token_id
|
|
53
|
+
self.use_int8 = use_int8
|
|
54
|
+
self.kv_cache = kv_cache
|
|
55
|
+
self.output_tokens: list[int] = []
|
|
56
|
+
self.is_finished = False
|
|
57
|
+
self.prefilled = False
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ContinuousBatchScheduler:
|
|
61
|
+
"""
|
|
62
|
+
Synchronous continuous batching scheduler:
|
|
63
|
+
Packs decode tokens from up to max_batch_size active requests into single forward passes,
|
|
64
|
+
admitting new requests on the fly without waiting for previous requests to finish.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def __init__(
|
|
68
|
+
self,
|
|
69
|
+
model: ModernMicroGPT,
|
|
70
|
+
max_batch_size: int = 4,
|
|
71
|
+
use_paged_cache: bool = True,
|
|
72
|
+
use_int8: bool = True,
|
|
73
|
+
):
|
|
74
|
+
self.model = model
|
|
75
|
+
self.max_batch_size = max_batch_size
|
|
76
|
+
self.use_paged_cache = use_paged_cache
|
|
77
|
+
self.use_int8 = use_int8
|
|
78
|
+
self.pending_queue: list[InferenceRequest] = []
|
|
79
|
+
self.active_requests: list[InferenceRequest] = []
|
|
80
|
+
self.completed_requests: dict[str | int, list[int]] = {}
|
|
81
|
+
|
|
82
|
+
def add_request(
|
|
83
|
+
self,
|
|
84
|
+
prompt_ids: list[int],
|
|
85
|
+
request_id: str | int | None = None,
|
|
86
|
+
max_new_tokens: int = 24,
|
|
87
|
+
temperature: float = 0.05,
|
|
88
|
+
top_k: int = 6,
|
|
89
|
+
top_p: float = 0.9,
|
|
90
|
+
stop_token_id: int | None = None,
|
|
91
|
+
) -> InferenceRequest:
|
|
92
|
+
"""Enqueues a new generation request into the pending queue."""
|
|
93
|
+
cache = self.model.init_kv_cache(use_paged=self.use_paged_cache)
|
|
94
|
+
req = InferenceRequest(
|
|
95
|
+
prompt_ids=prompt_ids,
|
|
96
|
+
request_id=request_id,
|
|
97
|
+
max_new_tokens=max_new_tokens,
|
|
98
|
+
temperature=temperature,
|
|
99
|
+
top_k=top_k,
|
|
100
|
+
top_p=top_p,
|
|
101
|
+
stop_token_id=stop_token_id,
|
|
102
|
+
use_int8=self.use_int8,
|
|
103
|
+
kv_cache=cache,
|
|
104
|
+
)
|
|
105
|
+
self.pending_queue.append(req)
|
|
106
|
+
return req
|
|
107
|
+
|
|
108
|
+
def has_work(self) -> bool:
|
|
109
|
+
"""Returns True if there are pending or active requests being processed."""
|
|
110
|
+
return bool(self.pending_queue or self.active_requests)
|
|
111
|
+
|
|
112
|
+
def step(self) -> dict[str | int, int]:
|
|
113
|
+
"""
|
|
114
|
+
Executes one continuous batching iteration:
|
|
115
|
+
1. Admits pending requests up to max_batch_size.
|
|
116
|
+
2. Runs prompt prefill for newly admitted requests.
|
|
117
|
+
3. Packs active decode requests into a single forward pass.
|
|
118
|
+
4. Evicts finished requests and returns {request_id: token_id} for this step.
|
|
119
|
+
"""
|
|
120
|
+
if not self.has_work():
|
|
121
|
+
return {}
|
|
122
|
+
|
|
123
|
+
step_outputs: dict[str | int, int] = {}
|
|
124
|
+
V = self.model.vocab_size
|
|
125
|
+
|
|
126
|
+
# 1. Admit pending requests into active batch
|
|
127
|
+
while len(self.active_requests) < self.max_batch_size and self.pending_queue:
|
|
128
|
+
admitted = self.pending_queue.pop(0)
|
|
129
|
+
self.active_requests.append(admitted)
|
|
130
|
+
|
|
131
|
+
# 2. Prefill newly admitted requests (1 request at a time to populate its KV cache)
|
|
132
|
+
decode_candidates: list[InferenceRequest] = []
|
|
133
|
+
for req in self.active_requests:
|
|
134
|
+
if not req.prefilled:
|
|
135
|
+
if not req.prompt_ids:
|
|
136
|
+
req.is_finished = True
|
|
137
|
+
continue
|
|
138
|
+
logits = self.model.forward(
|
|
139
|
+
req.prompt_ids,
|
|
140
|
+
start_pos=0,
|
|
141
|
+
kv_caches=req.kv_cache,
|
|
142
|
+
use_int8=req.use_int8,
|
|
143
|
+
)
|
|
144
|
+
last_row = logits.data[(logits.rows - 1) * V : logits.rows * V]
|
|
145
|
+
next_id = self.model.sample_top_k_top_p(
|
|
146
|
+
last_row, req.temperature, req.top_k, req.top_p
|
|
147
|
+
)
|
|
148
|
+
req.output_tokens.append(next_id)
|
|
149
|
+
req.prefilled = True
|
|
150
|
+
step_outputs[req.request_id] = next_id
|
|
151
|
+
|
|
152
|
+
if (req.stop_token_id is not None and next_id == req.stop_token_id) or len(
|
|
153
|
+
req.output_tokens
|
|
154
|
+
) >= req.max_new_tokens:
|
|
155
|
+
req.is_finished = True
|
|
156
|
+
else:
|
|
157
|
+
decode_candidates.append(req)
|
|
158
|
+
|
|
159
|
+
# 3. Packed continuous decode forward pass for active requests
|
|
160
|
+
if decode_candidates:
|
|
161
|
+
tokens = [req.output_tokens[-1] for req in decode_candidates]
|
|
162
|
+
caches = [req.kv_cache for req in decode_candidates]
|
|
163
|
+
positions = [
|
|
164
|
+
req.kv_cache.seq_len
|
|
165
|
+
if getattr(req.kv_cache, "is_paged", False)
|
|
166
|
+
else req.kv_cache[0]["seq_len"]
|
|
167
|
+
for req in decode_candidates
|
|
168
|
+
]
|
|
169
|
+
|
|
170
|
+
batched_logits = self.model.forward_batched(
|
|
171
|
+
tokens, caches, positions, use_int8=self.use_int8
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
for idx, req in enumerate(decode_candidates):
|
|
175
|
+
row = batched_logits.data[idx * V : (idx + 1) * V]
|
|
176
|
+
next_id = self.model.sample_top_k_top_p(
|
|
177
|
+
row, req.temperature, req.top_k, req.top_p
|
|
178
|
+
)
|
|
179
|
+
req.output_tokens.append(next_id)
|
|
180
|
+
step_outputs[req.request_id] = next_id
|
|
181
|
+
|
|
182
|
+
if (req.stop_token_id is not None and next_id == req.stop_token_id) or len(
|
|
183
|
+
req.output_tokens
|
|
184
|
+
) >= req.max_new_tokens:
|
|
185
|
+
req.is_finished = True
|
|
186
|
+
|
|
187
|
+
# 4. Evict finished requests and free their KV-caches
|
|
188
|
+
remaining_active = []
|
|
189
|
+
for req in self.active_requests:
|
|
190
|
+
if req.is_finished:
|
|
191
|
+
self.completed_requests[req.request_id] = list(req.output_tokens)
|
|
192
|
+
if hasattr(req.kv_cache, "release"):
|
|
193
|
+
req.kv_cache.release()
|
|
194
|
+
else:
|
|
195
|
+
remaining_active.append(req)
|
|
196
|
+
self.active_requests = remaining_active
|
|
197
|
+
|
|
198
|
+
return step_outputs
|
|
199
|
+
|
|
200
|
+
def run_all(self) -> dict[str | int, list[int]]:
|
|
201
|
+
"""Runs the continuous batcher to completion across all enqueued requests."""
|
|
202
|
+
while self.has_work():
|
|
203
|
+
self.step()
|
|
204
|
+
return self.completed_requests
|
|
205
|
+
|
|
206
|
+
def stream(self) -> Generator[dict[str | int, int], None, None]:
|
|
207
|
+
"""Yields step_outputs dict at each decoding iteration until all requests finish."""
|
|
208
|
+
while self.has_work():
|
|
209
|
+
outputs = self.step()
|
|
210
|
+
if outputs:
|
|
211
|
+
yield outputs
|
|
212
|
+
|
|
213
|
+
def stats(self) -> dict:
|
|
214
|
+
"""Returns scheduler queues and active batch telemetry."""
|
|
215
|
+
return {
|
|
216
|
+
"pending_count": len(self.pending_queue),
|
|
217
|
+
"active_count": len(self.active_requests),
|
|
218
|
+
"completed_count": len(self.completed_requests),
|
|
219
|
+
"max_batch_size": self.max_batch_size,
|
|
220
|
+
}
|