surgecore 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- surgecore-0.0.1/Cargo.lock +228 -0
- surgecore-0.0.1/Cargo.toml +22 -0
- surgecore-0.0.1/PKG-INFO +89 -0
- surgecore-0.0.1/README.md +71 -0
- surgecore-0.0.1/pyproject.toml +64 -0
- surgecore-0.0.1/rustfmt.toml +2 -0
- surgecore-0.0.1/src/engine/async_spec.rs +602 -0
- surgecore-0.0.1/src/engine/dp.rs +356 -0
- surgecore-0.0.1/src/engine/golden.rs +670 -0
- surgecore-0.0.1/src/engine/mod.rs +939 -0
- surgecore-0.0.1/src/engine/plan.rs +191 -0
- surgecore-0.0.1/src/engine/record.rs +263 -0
- surgecore-0.0.1/src/engine/request.rs +450 -0
- surgecore-0.0.1/src/engine/sched/budget.rs +106 -0
- surgecore-0.0.1/src/engine/sched/commit.rs +282 -0
- surgecore-0.0.1/src/engine/sched/mod.rs +135 -0
- surgecore-0.0.1/src/engine/sched/queue.rs +186 -0
- surgecore-0.0.1/src/engine/sched/schedule.rs +596 -0
- surgecore-0.0.1/src/engine/sched/trim.rs +223 -0
- surgecore-0.0.1/src/engine/snapshot_tests.rs +754 -0
- surgecore-0.0.1/src/engine/verify.rs +556 -0
- surgecore-0.0.1/src/kv/group.rs +302 -0
- surgecore-0.0.1/src/kv/hash.rs +100 -0
- surgecore-0.0.1/src/kv/mod.rs +866 -0
- surgecore-0.0.1/src/kv/pool.rs +401 -0
- surgecore-0.0.1/src/kv/search.rs +326 -0
- surgecore-0.0.1/src/kv/snapshot.rs +294 -0
- surgecore-0.0.1/src/lib.rs +28 -0
- surgecore-0.0.1/src/payload.rs +1295 -0
- surgecore-0.0.1/src/python/detok.rs +456 -0
- surgecore-0.0.1/src/python/engine.rs +755 -0
- surgecore-0.0.1/src/python/kv.rs +349 -0
- surgecore-0.0.1/src/python/mod.rs +37 -0
- surgecore-0.0.1/src/python/payload.rs +398 -0
- surgecore-0.0.1/surgecore/__init__.py +73 -0
- surgecore-0.0.1/surgecore/_native.pyi +650 -0
- surgecore-0.0.1/tests/data/engine_goldens.npz +0 -0
- surgecore-0.0.1/tests/data/engine_goldens_ext.npz +0 -0
- surgecore-0.0.1/tests/data/host_payload_goldens.npz +0 -0
- surgecore-0.0.1/tests/data/kv_goldens.npz +0 -0
- surgecore-0.0.1/tests/test_detokenizer.py +161 -0
- surgecore-0.0.1/tests/test_engine_behaviour.py +1215 -0
- surgecore-0.0.1/tests/test_engine_goldens.py +251 -0
- surgecore-0.0.1/tests/test_engine_goldens_ext.py +231 -0
- surgecore-0.0.1/tests/test_host_payload.py +317 -0
- surgecore-0.0.1/tests/test_kv_invariants.py +122 -0
- surgecore-0.0.1/tests/test_kv_parity.py +520 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "autocfg"
|
|
7
|
+
version = "1.5.1"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
|
10
|
+
|
|
11
|
+
[[package]]
|
|
12
|
+
name = "heck"
|
|
13
|
+
version = "0.5.0"
|
|
14
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
15
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
16
|
+
|
|
17
|
+
[[package]]
|
|
18
|
+
name = "libc"
|
|
19
|
+
version = "0.2.191"
|
|
20
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
21
|
+
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
|
22
|
+
|
|
23
|
+
[[package]]
|
|
24
|
+
name = "matrixmultiply"
|
|
25
|
+
version = "0.3.11"
|
|
26
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
27
|
+
checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
|
|
28
|
+
dependencies = [
|
|
29
|
+
"autocfg",
|
|
30
|
+
"rawpointer",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[[package]]
|
|
34
|
+
name = "ndarray"
|
|
35
|
+
version = "0.17.2"
|
|
36
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
37
|
+
checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
|
|
38
|
+
dependencies = [
|
|
39
|
+
"matrixmultiply",
|
|
40
|
+
"num-complex",
|
|
41
|
+
"num-integer",
|
|
42
|
+
"num-traits",
|
|
43
|
+
"portable-atomic",
|
|
44
|
+
"portable-atomic-util",
|
|
45
|
+
"rawpointer",
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
[[package]]
|
|
49
|
+
name = "num-complex"
|
|
50
|
+
version = "0.4.6"
|
|
51
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
52
|
+
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
|
|
53
|
+
dependencies = [
|
|
54
|
+
"num-traits",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[[package]]
|
|
58
|
+
name = "num-integer"
|
|
59
|
+
version = "0.1.47"
|
|
60
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
61
|
+
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
|
|
62
|
+
dependencies = [
|
|
63
|
+
"num-traits",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
[[package]]
|
|
67
|
+
name = "num-traits"
|
|
68
|
+
version = "0.2.19"
|
|
69
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
70
|
+
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
|
71
|
+
dependencies = [
|
|
72
|
+
"autocfg",
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
[[package]]
|
|
76
|
+
name = "numpy"
|
|
77
|
+
version = "0.30.0"
|
|
78
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
79
|
+
checksum = "aa0b9111761885449e894981fa8051dcf69e0bd114aecbd4bcdd830b6be4ed92"
|
|
80
|
+
dependencies = [
|
|
81
|
+
"libc",
|
|
82
|
+
"ndarray",
|
|
83
|
+
"num-complex",
|
|
84
|
+
"num-integer",
|
|
85
|
+
"num-traits",
|
|
86
|
+
"pyo3",
|
|
87
|
+
"pyo3-build-config",
|
|
88
|
+
"rustc-hash",
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
[[package]]
|
|
92
|
+
name = "once_cell"
|
|
93
|
+
version = "1.21.4"
|
|
94
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
95
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
96
|
+
|
|
97
|
+
[[package]]
|
|
98
|
+
name = "portable-atomic"
|
|
99
|
+
version = "1.15.0"
|
|
100
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
101
|
+
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
102
|
+
|
|
103
|
+
[[package]]
|
|
104
|
+
name = "portable-atomic-util"
|
|
105
|
+
version = "0.2.8"
|
|
106
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
107
|
+
checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715"
|
|
108
|
+
dependencies = [
|
|
109
|
+
"portable-atomic",
|
|
110
|
+
]
|
|
111
|
+
|
|
112
|
+
[[package]]
|
|
113
|
+
name = "proc-macro2"
|
|
114
|
+
version = "1.0.107"
|
|
115
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
116
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
117
|
+
dependencies = [
|
|
118
|
+
"unicode-ident",
|
|
119
|
+
]
|
|
120
|
+
|
|
121
|
+
[[package]]
|
|
122
|
+
name = "pyo3"
|
|
123
|
+
version = "0.30.1"
|
|
124
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
125
|
+
checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
|
|
126
|
+
dependencies = [
|
|
127
|
+
"libc",
|
|
128
|
+
"once_cell",
|
|
129
|
+
"portable-atomic",
|
|
130
|
+
"pyo3-build-config",
|
|
131
|
+
"pyo3-ffi",
|
|
132
|
+
"pyo3-macros",
|
|
133
|
+
]
|
|
134
|
+
|
|
135
|
+
[[package]]
|
|
136
|
+
name = "pyo3-build-config"
|
|
137
|
+
version = "0.30.1"
|
|
138
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
139
|
+
checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
|
|
140
|
+
dependencies = [
|
|
141
|
+
"target-lexicon",
|
|
142
|
+
]
|
|
143
|
+
|
|
144
|
+
[[package]]
|
|
145
|
+
name = "pyo3-ffi"
|
|
146
|
+
version = "0.30.1"
|
|
147
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
148
|
+
checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
|
|
149
|
+
dependencies = [
|
|
150
|
+
"libc",
|
|
151
|
+
"pyo3-build-config",
|
|
152
|
+
]
|
|
153
|
+
|
|
154
|
+
[[package]]
|
|
155
|
+
name = "pyo3-macros"
|
|
156
|
+
version = "0.30.1"
|
|
157
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
158
|
+
checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
|
|
159
|
+
dependencies = [
|
|
160
|
+
"proc-macro2",
|
|
161
|
+
"pyo3-macros-backend",
|
|
162
|
+
"quote",
|
|
163
|
+
"syn",
|
|
164
|
+
]
|
|
165
|
+
|
|
166
|
+
[[package]]
|
|
167
|
+
name = "pyo3-macros-backend"
|
|
168
|
+
version = "0.30.1"
|
|
169
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
170
|
+
checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
|
|
171
|
+
dependencies = [
|
|
172
|
+
"heck",
|
|
173
|
+
"proc-macro2",
|
|
174
|
+
"quote",
|
|
175
|
+
"syn",
|
|
176
|
+
]
|
|
177
|
+
|
|
178
|
+
[[package]]
|
|
179
|
+
name = "quote"
|
|
180
|
+
version = "1.0.47"
|
|
181
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
182
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
183
|
+
dependencies = [
|
|
184
|
+
"proc-macro2",
|
|
185
|
+
]
|
|
186
|
+
|
|
187
|
+
[[package]]
|
|
188
|
+
name = "rawpointer"
|
|
189
|
+
version = "0.2.1"
|
|
190
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
191
|
+
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
|
|
192
|
+
|
|
193
|
+
[[package]]
|
|
194
|
+
name = "rustc-hash"
|
|
195
|
+
version = "2.1.3"
|
|
196
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
197
|
+
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
|
198
|
+
|
|
199
|
+
[[package]]
|
|
200
|
+
name = "surgecore"
|
|
201
|
+
version = "0.0.1"
|
|
202
|
+
dependencies = [
|
|
203
|
+
"numpy",
|
|
204
|
+
"pyo3",
|
|
205
|
+
]
|
|
206
|
+
|
|
207
|
+
[[package]]
|
|
208
|
+
name = "syn"
|
|
209
|
+
version = "3.0.9"
|
|
210
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
211
|
+
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
212
|
+
dependencies = [
|
|
213
|
+
"proc-macro2",
|
|
214
|
+
"quote",
|
|
215
|
+
"unicode-ident",
|
|
216
|
+
]
|
|
217
|
+
|
|
218
|
+
[[package]]
|
|
219
|
+
name = "target-lexicon"
|
|
220
|
+
version = "0.13.5"
|
|
221
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
222
|
+
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
223
|
+
|
|
224
|
+
[[package]]
|
|
225
|
+
name = "unicode-ident"
|
|
226
|
+
version = "1.0.27"
|
|
227
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
228
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "surgecore"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
rust-version = "1.85"
|
|
6
|
+
license = "Apache-2.0"
|
|
7
|
+
description = "EasyDeL's native host library for the eSurge inference engine"
|
|
8
|
+
publish = false
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
|
|
11
|
+
[lib]
|
|
12
|
+
name = "surgecore"
|
|
13
|
+
crate-type = ["cdylib"]
|
|
14
|
+
|
|
15
|
+
[features]
|
|
16
|
+
# The Python bindings. `cargo test --no-default-features` tests the pure-Rust core without a Python interpreter.
|
|
17
|
+
default = ["python"]
|
|
18
|
+
python = ["dep:pyo3", "dep:numpy"]
|
|
19
|
+
|
|
20
|
+
[dependencies]
|
|
21
|
+
pyo3 = { version = "0.30.1", features = ["abi3-py311"], optional = true }
|
|
22
|
+
numpy = { version = "0.30.0", optional = true }
|
surgecore-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: surgecore
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Classifier: Development Status :: 3 - Alpha
|
|
5
|
+
Classifier: Intended Audience :: Developers
|
|
6
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
10
|
+
Classifier: Programming Language :: Rust
|
|
11
|
+
Requires-Dist: numpy
|
|
12
|
+
Summary: EasyDeL's native host library for the eSurge inference engine.
|
|
13
|
+
Author-email: Erfan Zare Chavoshi <Erfanzare810@gmail.com>
|
|
14
|
+
License: Apache-2.0
|
|
15
|
+
Requires-Python: >=3.11, <3.14
|
|
16
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
17
|
+
|
|
18
|
+
# surgecore
|
|
19
|
+
|
|
20
|
+
EasyDeL's native host library for the eSurge inference engine: a Rust crate (PyO3 + maturin) whose extension module
|
|
21
|
+
`surgecore._native` holds eSurge's host-side state and per-step host computation.
|
|
22
|
+
|
|
23
|
+
It holds:
|
|
24
|
+
- `HostPayloadBuilder`, which fills a batch preparer's preallocated staging arrays with one forward window's step
|
|
25
|
+
inputs, with the GIL released: token and position packing, query start locations and sequence lengths, rank-major DP
|
|
26
|
+
packing, page-table rows, the DP-local page check, the v2 KV slot mapping, the positional request distribution and
|
|
27
|
+
the packed sampling vectors.
|
|
28
|
+
- `Detokenizer`, in-process batched incremental detokenization over a HuggingFace tokenizer. Rust owns request token
|
|
29
|
+
history, context windows, incomplete UTF-8 buffering, bounded FIFO state and terminal suffix reconciliation; normal
|
|
30
|
+
results carry only new text, with a full snapshot at finish or resynchronization.
|
|
31
|
+
- `KvCore`, eSurge's KV-cache core: the page pool with its LRU free list and null page 0, the chained page keys and the
|
|
32
|
+
prefix map, the full, sliding-window and chunked-local group rules, transactional allocation with rollback, and
|
|
33
|
+
publication of pages once their KV is computed. eSurge's `CacheManager` is its binding.
|
|
34
|
+
- `SnapshotIndex`, the slots of recurrent-state snapshots taken at prefix-cache page boundaries, which let
|
|
35
|
+
linear-attention and SSM models reuse prefix-cache hits.
|
|
36
|
+
- `EngineCore`, eSurge's host state machine: the request table, every data-parallel rank's queues, token budget, KV
|
|
37
|
+
core and snapshot index, and the scheduler over them (FCFS or priority, chunked prefill, the long-prefill cap,
|
|
38
|
+
preemption, whole-prompt admission, image-span alignment, snapshot boundaries, async output placeholders, request
|
|
39
|
+
buckets) with synchronous speculative drafts (draft trimming, EAGLE lookahead, acceptance rollback) and the stop
|
|
40
|
+
checks. `schedule()` returns a `Plan`; `commit()` takes the runner's sampled lists, accepted counts and new drafts
|
|
41
|
+
and returns `Events`. Every call releases the GIL.
|
|
42
|
+
- The verification budget of synchronous speculative steps (`EngineCore.set_verify_budget`): from a step-time table
|
|
43
|
+
per compiled token bucket (measured by the runner after compile, corrected online from measured step times) and
|
|
44
|
+
per-request acceptance estimates per draft position, each schedule verifies 0..k drafts per request, the count
|
|
45
|
+
that maximises expected emitted tokens per millisecond, and sets `Plan.num_draft_tokens_next`.
|
|
46
|
+
- Async speculative steps: a verify window of `1 + k` may be scheduled while the request's previous window is in
|
|
47
|
+
flight. The core advances `computed` as if every draft were accepted (`Plan.spec_slots` per entry), each commit
|
|
48
|
+
rolls back only its own rejected drafts, and the device corrects the later step's start. Windowed groups drop
|
|
49
|
+
pages only below the lowest position the request may really be at.
|
|
50
|
+
|
|
51
|
+
Only eSurge imports it; `import easydel` and training do not need it.
|
|
52
|
+
|
|
53
|
+
## Build
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
cd libs/surgecore
|
|
57
|
+
maturin build --release # wheel with surgecore/_native.abi3.so (abi3: one wheel covers CPython 3.11-3.13)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Install the wheel, or extract its `surgecore/_native.abi3.so` into `libs/surgecore/surgecore/` and put
|
|
61
|
+
`libs/surgecore` on `PYTHONPATH`. `pyproject.toml` enables `pyo3/extension-module` through `[tool.maturin] features`,
|
|
62
|
+
so plain cargo builds link against libpython.
|
|
63
|
+
|
|
64
|
+
## Test
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
cd libs/surgecore
|
|
68
|
+
export PYO3_PYTHON=$(which python) # a Python 3.11-3.13 interpreter: clippy and the tests link libpython
|
|
69
|
+
# (if the test binary cannot find libpython at run time, add that interpreter's LIBDIR to LD_LIBRARY_PATH)
|
|
70
|
+
cargo fmt --check
|
|
71
|
+
cargo clippy --locked --all-targets -- -D warnings
|
|
72
|
+
cargo test --locked # Rust unit tests (with the Python bindings compiled)
|
|
73
|
+
cargo test --locked --no-default-features # the pure-Rust core alone, no Python needed
|
|
74
|
+
python -m pytest tests # parity against the goldens in tests/data plus invariant tests (CPU)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Workspace CI (`.github/workflows/workspace-ci.yaml`, job `surgecore`) runs these and a wheel build whenever
|
|
78
|
+
`libs/surgecore` changes. Releases go through `scripts/release.sh surgecore <version>` and
|
|
79
|
+
`scripts/publish.sh surgecore` (see the workspace's `WORKSPACE.md`); a `surgecore-v*` tag builds and publishes the
|
|
80
|
+
wheels and the sdist.
|
|
81
|
+
|
|
82
|
+
The pytest files carry `pytest.mark.cpu_ok`: they do no JAX computation. The fixtures in `tests/data` hold inputs and
|
|
83
|
+
outputs captured from the Python code each piece replaced, and each header names its source commit:
|
|
84
|
+
`host_payload_goldens.npz` comes from eSurge's former step-input builder, `kv_goldens.npz` from its former KV core,
|
|
85
|
+
`engine_goldens.npz` and `engine_goldens_ext.npz` from its former scheduler (the second with speculative drafts,
|
|
86
|
+
runner-output gaps, multimodal page keys, request buckets, the finished set and the structured-output flag).
|
|
87
|
+
`test_engine_behaviour.py` states the former scheduler's behaviour tests against `EngineCore`.
|
|
88
|
+
The parity tests compare the native code with them bitwise.
|
|
89
|
+
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# surgecore
|
|
2
|
+
|
|
3
|
+
EasyDeL's native host library for the eSurge inference engine: a Rust crate (PyO3 + maturin) whose extension module
|
|
4
|
+
`surgecore._native` holds eSurge's host-side state and per-step host computation.
|
|
5
|
+
|
|
6
|
+
It holds:
|
|
7
|
+
- `HostPayloadBuilder`, which fills a batch preparer's preallocated staging arrays with one forward window's step
|
|
8
|
+
inputs, with the GIL released: token and position packing, query start locations and sequence lengths, rank-major DP
|
|
9
|
+
packing, page-table rows, the DP-local page check, the v2 KV slot mapping, the positional request distribution and
|
|
10
|
+
the packed sampling vectors.
|
|
11
|
+
- `Detokenizer`, in-process batched incremental detokenization over a HuggingFace tokenizer. Rust owns request token
|
|
12
|
+
history, context windows, incomplete UTF-8 buffering, bounded FIFO state and terminal suffix reconciliation; normal
|
|
13
|
+
results carry only new text, with a full snapshot at finish or resynchronization.
|
|
14
|
+
- `KvCore`, eSurge's KV-cache core: the page pool with its LRU free list and null page 0, the chained page keys and the
|
|
15
|
+
prefix map, the full, sliding-window and chunked-local group rules, transactional allocation with rollback, and
|
|
16
|
+
publication of pages once their KV is computed. eSurge's `CacheManager` is its binding.
|
|
17
|
+
- `SnapshotIndex`, the slots of recurrent-state snapshots taken at prefix-cache page boundaries, which let
|
|
18
|
+
linear-attention and SSM models reuse prefix-cache hits.
|
|
19
|
+
- `EngineCore`, eSurge's host state machine: the request table, every data-parallel rank's queues, token budget, KV
|
|
20
|
+
core and snapshot index, and the scheduler over them (FCFS or priority, chunked prefill, the long-prefill cap,
|
|
21
|
+
preemption, whole-prompt admission, image-span alignment, snapshot boundaries, async output placeholders, request
|
|
22
|
+
buckets) with synchronous speculative drafts (draft trimming, EAGLE lookahead, acceptance rollback) and the stop
|
|
23
|
+
checks. `schedule()` returns a `Plan`; `commit()` takes the runner's sampled lists, accepted counts and new drafts
|
|
24
|
+
and returns `Events`. Every call releases the GIL.
|
|
25
|
+
- The verification budget of synchronous speculative steps (`EngineCore.set_verify_budget`): from a step-time table
|
|
26
|
+
per compiled token bucket (measured by the runner after compile, corrected online from measured step times) and
|
|
27
|
+
per-request acceptance estimates per draft position, each schedule verifies 0..k drafts per request, the count
|
|
28
|
+
that maximises expected emitted tokens per millisecond, and sets `Plan.num_draft_tokens_next`.
|
|
29
|
+
- Async speculative steps: a verify window of `1 + k` may be scheduled while the request's previous window is in
|
|
30
|
+
flight. The core advances `computed` as if every draft were accepted (`Plan.spec_slots` per entry), each commit
|
|
31
|
+
rolls back only its own rejected drafts, and the device corrects the later step's start. Windowed groups drop
|
|
32
|
+
pages only below the lowest position the request may really be at.
|
|
33
|
+
|
|
34
|
+
Only eSurge imports it; `import easydel` and training do not need it.
|
|
35
|
+
|
|
36
|
+
## Build
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
cd libs/surgecore
|
|
40
|
+
maturin build --release # wheel with surgecore/_native.abi3.so (abi3: one wheel covers CPython 3.11-3.13)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Install the wheel, or extract its `surgecore/_native.abi3.so` into `libs/surgecore/surgecore/` and put
|
|
44
|
+
`libs/surgecore` on `PYTHONPATH`. `pyproject.toml` enables `pyo3/extension-module` through `[tool.maturin] features`,
|
|
45
|
+
so plain cargo builds link against libpython.
|
|
46
|
+
|
|
47
|
+
## Test
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
cd libs/surgecore
|
|
51
|
+
export PYO3_PYTHON=$(which python) # a Python 3.11-3.13 interpreter: clippy and the tests link libpython
|
|
52
|
+
# (if the test binary cannot find libpython at run time, add that interpreter's LIBDIR to LD_LIBRARY_PATH)
|
|
53
|
+
cargo fmt --check
|
|
54
|
+
cargo clippy --locked --all-targets -- -D warnings
|
|
55
|
+
cargo test --locked # Rust unit tests (with the Python bindings compiled)
|
|
56
|
+
cargo test --locked --no-default-features # the pure-Rust core alone, no Python needed
|
|
57
|
+
python -m pytest tests # parity against the goldens in tests/data plus invariant tests (CPU)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Workspace CI (`.github/workflows/workspace-ci.yaml`, job `surgecore`) runs these and a wheel build whenever
|
|
61
|
+
`libs/surgecore` changes. Releases go through `scripts/release.sh surgecore <version>` and
|
|
62
|
+
`scripts/publish.sh surgecore` (see the workspace's `WORKSPACE.md`); a `surgecore-v*` tag builds and publishes the
|
|
63
|
+
wheels and the sdist.
|
|
64
|
+
|
|
65
|
+
The pytest files carry `pytest.mark.cpu_ok`: they do no JAX computation. The fixtures in `tests/data` hold inputs and
|
|
66
|
+
outputs captured from the Python code each piece replaced, and each header names its source commit:
|
|
67
|
+
`host_payload_goldens.npz` comes from eSurge's former step-input builder, `kv_goldens.npz` from its former KV core,
|
|
68
|
+
`engine_goldens.npz` and `engine_goldens_ext.npz` from its former scheduler (the second with speculative drafts,
|
|
69
|
+
runner-output gaps, multimodal page keys, request buckets, the finished set and the structured-output flag).
|
|
70
|
+
`test_engine_behaviour.py` states the former scheduler's behaviour tests against `EngineCore`.
|
|
71
|
+
The parity tests compare the native code with them bitwise.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["maturin>=1.7,<2"]
|
|
3
|
+
build-backend = "maturin"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "surgecore"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
authors = [{ name = "Erfan Zare Chavoshi", email = "Erfanzare810@gmail.com" }]
|
|
9
|
+
description = "EasyDeL's native host library for the eSurge inference engine."
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
requires-python = ">=3.11,<3.14"
|
|
12
|
+
license = { text = "Apache-2.0" }
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"License :: OSI Approved :: Apache Software License",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Programming Language :: Rust",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["numpy"]
|
|
23
|
+
|
|
24
|
+
[tool.maturin]
|
|
25
|
+
module-name = "surgecore._native"
|
|
26
|
+
features = ["pyo3/extension-module"]
|
|
27
|
+
|
|
28
|
+
[tool.uv]
|
|
29
|
+
# Rebuild the extension when the crate changes, not only when this file does.
|
|
30
|
+
cache-keys = [
|
|
31
|
+
{ file = "pyproject.toml" },
|
|
32
|
+
{ file = "Cargo.toml" },
|
|
33
|
+
{ file = "Cargo.lock" },
|
|
34
|
+
{ file = "src/**/*.rs" },
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[tool.ruff]
|
|
38
|
+
line-length = 121
|
|
39
|
+
target-version = "py311"
|
|
40
|
+
src = []
|
|
41
|
+
|
|
42
|
+
[tool.ruff.lint]
|
|
43
|
+
select = ["A", "B", "E", "F", "I", "NPY", "RUF", "UP", "W"]
|
|
44
|
+
ignore = [
|
|
45
|
+
"F722", # Forward annotations needed for jaxtyping
|
|
46
|
+
"B008", # Do not perform function calls in argument defaults
|
|
47
|
+
"UP015", # Unnecessary open mode
|
|
48
|
+
"A005", # Module-level shadowing
|
|
49
|
+
"UP037", # Remove quotes from type annotation - needed for jaxtyping string literals
|
|
50
|
+
"E501", # Line too long (handled by formatter)
|
|
51
|
+
"RUF001", # Ruff's own complexity check - handled by other tools
|
|
52
|
+
"RUF002", # Ruff's own complexity check - handled by other tools
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
[tool.ruff.lint.per-file-ignores]
|
|
56
|
+
"__init__.py" = ["E402", "F401"]
|
|
57
|
+
"**/modeling_*.py" = [
|
|
58
|
+
"UP037",
|
|
59
|
+
"F821",
|
|
60
|
+
] # Allow quoted type annotations and forward refs in model files
|
|
61
|
+
|
|
62
|
+
[tool.pytest.ini_options]
|
|
63
|
+
markers = ["cpu_ok: the test does no array computation, so it may run on a CPU backend"]
|
|
64
|
+
testpaths = ["tests"]
|