tuetoken 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,8 @@
1
+ # Builds are PORTABLE by default (generic target) so released wheels run on any
2
+ # x86-64 CPU — mirroring the C package, whose cibuildwheel build also drops
3
+ # -march=native. Measured cost of portability for this tokenizer: ~0% (C) to
4
+ # ~5-10% (Rust); the hot path doesn't auto-vectorize, so native buys little.
5
+ #
6
+ # For a max-speed LOCAL build (e.g. a fair benchmark vs the C version, which
7
+ # uses -march=native for local installs), opt in:
8
+ # RUSTFLAGS="-C target-cpu=native" maturin develop --release
@@ -0,0 +1,39 @@
1
+ name: ci
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+
7
+ jobs:
8
+ test:
9
+ runs-on: ubuntu-latest
10
+ steps:
11
+ - uses: actions/checkout@v4
12
+ - uses: actions/setup-python@v5
13
+ with:
14
+ python-version: "3.12"
15
+ - uses: dtolnay/rust-toolchain@stable
16
+
17
+ # Self-contained correctness gate: the streaming-vs-scan-vs-canonical merge
18
+ # invariants on random tables, rank-inversion, etc. No Python or model data.
19
+ - name: Rust tests
20
+ run: cargo test --release
21
+
22
+ - name: Build and install extension
23
+ run: |
24
+ python -m pip install --upgrade pip maturin
25
+ maturin build --release --out dist --interpreter python
26
+ python -m pip install --force-reinstall dist/*.whl
27
+
28
+ # Smoke test the installed wheel: imports cleanly, tokenizes, round-trips.
29
+ # from_tiktoken downloads a small rank file from OpenAI's CDN (no tiktoken dep).
30
+ - name: Python smoke test
31
+ run: |
32
+ python - <<'PY'
33
+ from tuetoken import Tokenizer, AutoTokenizer
34
+ enc = Tokenizer.from_tiktoken("gpt2")
35
+ ids = enc.encode_ordinary("Hello world 1234")
36
+ assert enc.decode(ids) == "Hello world 1234", "round-trip failed"
37
+ assert enc.count_tokens("Hello world 1234") == len(ids)
38
+ print("smoke OK:", ids)
39
+ PY
@@ -0,0 +1,124 @@
1
+ name: wheels
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ push:
6
+ tags:
7
+ - "v*"
8
+ release:
9
+ types: [published]
10
+
11
+ jobs:
12
+ build-wheels:
13
+ name: ${{ matrix.platform.name }} py${{ matrix.python-version }}
14
+ runs-on: ${{ matrix.platform.os }}
15
+ strategy:
16
+ fail-fast: false
17
+ matrix:
18
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
19
+ platform:
20
+ - name: linux-x86_64
21
+ os: ubuntu-latest
22
+ target: x86_64
23
+ - name: linux-aarch64
24
+ os: ubuntu-latest
25
+ target: aarch64
26
+ - name: macos-arm64
27
+ os: macos-14
28
+ - name: windows-x86_64
29
+ os: windows-2022
30
+
31
+ steps:
32
+ - uses: actions/checkout@v4
33
+
34
+ - uses: actions/setup-python@v5
35
+ with:
36
+ python-version: ${{ matrix.python-version }}
37
+
38
+ - uses: dtolnay/rust-toolchain@stable
39
+
40
+ - name: Build Linux wheel
41
+ if: runner.os == 'Linux'
42
+ uses: PyO3/maturin-action@v1
43
+ with:
44
+ command: build
45
+ target: ${{ matrix.platform.target }} # x86_64 (native) or aarch64 (cross)
46
+ args: --release --out dist -i python${{ matrix.python-version }}
47
+ manylinux: "2014"
48
+
49
+ - name: Build host wheel
50
+ if: runner.os != 'Linux'
51
+ uses: PyO3/maturin-action@v1
52
+ with:
53
+ command: build
54
+ args: --release --out dist --interpreter python
55
+
56
+ - name: Upload wheel artifact
57
+ uses: actions/upload-artifact@v4
58
+ with:
59
+ name: tuetoken-${{ matrix.platform.name }}-py${{ matrix.python-version }}
60
+ path: dist/*.whl
61
+ if-no-files-found: error
62
+
63
+ build-sdist:
64
+ name: source distribution
65
+ runs-on: ubuntu-latest
66
+ steps:
67
+ - uses: actions/checkout@v4
68
+
69
+ - uses: actions/setup-python@v5
70
+ with:
71
+ python-version: "3.12"
72
+
73
+ - uses: dtolnay/rust-toolchain@stable
74
+
75
+ - name: Build sdist
76
+ uses: PyO3/maturin-action@v1
77
+ with:
78
+ command: sdist
79
+ args: --out dist
80
+
81
+ - name: Upload sdist artifact
82
+ uses: actions/upload-artifact@v4
83
+ with:
84
+ name: tuetoken-sdist
85
+ path: dist/*.tar.gz
86
+ if-no-files-found: error
87
+
88
+ upload-release:
89
+ name: attach assets to release
90
+ needs: [build-wheels, build-sdist]
91
+ if: github.event_name == 'release' && github.event.action == 'published'
92
+ runs-on: ubuntu-latest
93
+ permissions:
94
+ contents: write
95
+ steps:
96
+ - uses: actions/download-artifact@v4
97
+ with:
98
+ path: dist
99
+ merge-multiple: true
100
+
101
+ - name: Attach wheels and sdist to GitHub release
102
+ uses: softprops/action-gh-release@v2
103
+ with:
104
+ files: |
105
+ dist/*.whl
106
+ dist/*.tar.gz
107
+
108
+ publish-pypi:
109
+ name: publish to PyPI
110
+ needs: [build-wheels, build-sdist]
111
+ if: github.event_name == 'release' && github.event.action == 'published'
112
+ runs-on: ubuntu-latest
113
+ environment: pypi
114
+ permissions:
115
+ contents: read
116
+ id-token: write
117
+ steps:
118
+ - uses: actions/download-artifact@v4
119
+ with:
120
+ path: dist
121
+ merge-multiple: true
122
+
123
+ - name: Publish package distributions to PyPI
124
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,5 @@
1
+ /target
2
+ *.so
3
+ __pycache__/
4
+ *.egg-info/
5
+ dist/
@@ -0,0 +1,332 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 4
4
+
5
+ [[package]]
6
+ name = "aho-corasick"
7
+ version = "1.1.4"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
10
+ dependencies = [
11
+ "memchr",
12
+ ]
13
+
14
+ [[package]]
15
+ name = "bit-set"
16
+ version = "0.8.0"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
19
+ dependencies = [
20
+ "bit-vec",
21
+ ]
22
+
23
+ [[package]]
24
+ name = "bit-vec"
25
+ version = "0.8.0"
26
+ source = "registry+https://github.com/rust-lang/crates.io-index"
27
+ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
28
+
29
+ [[package]]
30
+ name = "crossbeam-deque"
31
+ version = "0.8.6"
32
+ source = "registry+https://github.com/rust-lang/crates.io-index"
33
+ checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
34
+ dependencies = [
35
+ "crossbeam-epoch",
36
+ "crossbeam-utils",
37
+ ]
38
+
39
+ [[package]]
40
+ name = "crossbeam-epoch"
41
+ version = "0.9.18"
42
+ source = "registry+https://github.com/rust-lang/crates.io-index"
43
+ checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
44
+ dependencies = [
45
+ "crossbeam-utils",
46
+ ]
47
+
48
+ [[package]]
49
+ name = "crossbeam-utils"
50
+ version = "0.8.21"
51
+ source = "registry+https://github.com/rust-lang/crates.io-index"
52
+ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
53
+
54
+ [[package]]
55
+ name = "either"
56
+ version = "1.16.0"
57
+ source = "registry+https://github.com/rust-lang/crates.io-index"
58
+ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
59
+
60
+ [[package]]
61
+ name = "fancy-regex"
62
+ version = "0.14.0"
63
+ source = "registry+https://github.com/rust-lang/crates.io-index"
64
+ checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
65
+ dependencies = [
66
+ "bit-set",
67
+ "regex-automata",
68
+ "regex-syntax",
69
+ ]
70
+
71
+ [[package]]
72
+ name = "heck"
73
+ version = "0.5.0"
74
+ source = "registry+https://github.com/rust-lang/crates.io-index"
75
+ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
76
+
77
+ [[package]]
78
+ name = "itoa"
79
+ version = "1.0.18"
80
+ source = "registry+https://github.com/rust-lang/crates.io-index"
81
+ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
82
+
83
+ [[package]]
84
+ name = "libc"
85
+ version = "0.2.186"
86
+ source = "registry+https://github.com/rust-lang/crates.io-index"
87
+ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
88
+
89
+ [[package]]
90
+ name = "memchr"
91
+ version = "2.8.1"
92
+ source = "registry+https://github.com/rust-lang/crates.io-index"
93
+ checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
94
+
95
+ [[package]]
96
+ name = "once_cell"
97
+ version = "1.21.4"
98
+ source = "registry+https://github.com/rust-lang/crates.io-index"
99
+ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
100
+
101
+ [[package]]
102
+ name = "portable-atomic"
103
+ version = "1.13.1"
104
+ source = "registry+https://github.com/rust-lang/crates.io-index"
105
+ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
106
+
107
+ [[package]]
108
+ name = "proc-macro2"
109
+ version = "1.0.106"
110
+ source = "registry+https://github.com/rust-lang/crates.io-index"
111
+ checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
112
+ dependencies = [
113
+ "unicode-ident",
114
+ ]
115
+
116
+ [[package]]
117
+ name = "pyo3"
118
+ version = "0.28.3"
119
+ source = "registry+https://github.com/rust-lang/crates.io-index"
120
+ checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12"
121
+ dependencies = [
122
+ "libc",
123
+ "once_cell",
124
+ "portable-atomic",
125
+ "pyo3-build-config",
126
+ "pyo3-ffi",
127
+ "pyo3-macros",
128
+ ]
129
+
130
+ [[package]]
131
+ name = "pyo3-build-config"
132
+ version = "0.28.3"
133
+ source = "registry+https://github.com/rust-lang/crates.io-index"
134
+ checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e"
135
+ dependencies = [
136
+ "target-lexicon",
137
+ ]
138
+
139
+ [[package]]
140
+ name = "pyo3-ffi"
141
+ version = "0.28.3"
142
+ source = "registry+https://github.com/rust-lang/crates.io-index"
143
+ checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e"
144
+ dependencies = [
145
+ "libc",
146
+ "pyo3-build-config",
147
+ ]
148
+
149
+ [[package]]
150
+ name = "pyo3-macros"
151
+ version = "0.28.3"
152
+ source = "registry+https://github.com/rust-lang/crates.io-index"
153
+ checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813"
154
+ dependencies = [
155
+ "proc-macro2",
156
+ "pyo3-macros-backend",
157
+ "quote",
158
+ "syn",
159
+ ]
160
+
161
+ [[package]]
162
+ name = "pyo3-macros-backend"
163
+ version = "0.28.3"
164
+ source = "registry+https://github.com/rust-lang/crates.io-index"
165
+ checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb"
166
+ dependencies = [
167
+ "heck",
168
+ "proc-macro2",
169
+ "pyo3-build-config",
170
+ "quote",
171
+ "syn",
172
+ ]
173
+
174
+ [[package]]
175
+ name = "quote"
176
+ version = "1.0.45"
177
+ source = "registry+https://github.com/rust-lang/crates.io-index"
178
+ checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
179
+ dependencies = [
180
+ "proc-macro2",
181
+ ]
182
+
183
+ [[package]]
184
+ name = "rayon"
185
+ version = "1.12.0"
186
+ source = "registry+https://github.com/rust-lang/crates.io-index"
187
+ checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
188
+ dependencies = [
189
+ "either",
190
+ "rayon-core",
191
+ ]
192
+
193
+ [[package]]
194
+ name = "rayon-core"
195
+ version = "1.13.0"
196
+ source = "registry+https://github.com/rust-lang/crates.io-index"
197
+ checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
198
+ dependencies = [
199
+ "crossbeam-deque",
200
+ "crossbeam-utils",
201
+ ]
202
+
203
+ [[package]]
204
+ name = "regex-automata"
205
+ version = "0.4.14"
206
+ source = "registry+https://github.com/rust-lang/crates.io-index"
207
+ checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
208
+ dependencies = [
209
+ "aho-corasick",
210
+ "memchr",
211
+ "regex-syntax",
212
+ ]
213
+
214
+ [[package]]
215
+ name = "regex-syntax"
216
+ version = "0.8.10"
217
+ source = "registry+https://github.com/rust-lang/crates.io-index"
218
+ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
219
+
220
+ [[package]]
221
+ name = "rustc-hash"
222
+ version = "2.1.2"
223
+ source = "registry+https://github.com/rust-lang/crates.io-index"
224
+ checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
225
+
226
+ [[package]]
227
+ name = "serde"
228
+ version = "1.0.228"
229
+ source = "registry+https://github.com/rust-lang/crates.io-index"
230
+ checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
231
+ dependencies = [
232
+ "serde_core",
233
+ ]
234
+
235
+ [[package]]
236
+ name = "serde_core"
237
+ version = "1.0.228"
238
+ source = "registry+https://github.com/rust-lang/crates.io-index"
239
+ checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
240
+ dependencies = [
241
+ "serde_derive",
242
+ ]
243
+
244
+ [[package]]
245
+ name = "serde_derive"
246
+ version = "1.0.228"
247
+ source = "registry+https://github.com/rust-lang/crates.io-index"
248
+ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
249
+ dependencies = [
250
+ "proc-macro2",
251
+ "quote",
252
+ "syn",
253
+ ]
254
+
255
+ [[package]]
256
+ name = "serde_json"
257
+ version = "1.0.150"
258
+ source = "registry+https://github.com/rust-lang/crates.io-index"
259
+ checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
260
+ dependencies = [
261
+ "itoa",
262
+ "memchr",
263
+ "serde",
264
+ "serde_core",
265
+ "zmij",
266
+ ]
267
+
268
+ [[package]]
269
+ name = "syn"
270
+ version = "2.0.117"
271
+ source = "registry+https://github.com/rust-lang/crates.io-index"
272
+ checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
273
+ dependencies = [
274
+ "proc-macro2",
275
+ "quote",
276
+ "unicode-ident",
277
+ ]
278
+
279
+ [[package]]
280
+ name = "target-lexicon"
281
+ version = "0.13.5"
282
+ source = "registry+https://github.com/rust-lang/crates.io-index"
283
+ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
284
+
285
+ [[package]]
286
+ name = "tinyvec"
287
+ version = "1.11.0"
288
+ source = "registry+https://github.com/rust-lang/crates.io-index"
289
+ checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
290
+ dependencies = [
291
+ "tinyvec_macros",
292
+ ]
293
+
294
+ [[package]]
295
+ name = "tinyvec_macros"
296
+ version = "0.1.1"
297
+ source = "registry+https://github.com/rust-lang/crates.io-index"
298
+ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
299
+
300
+ [[package]]
301
+ name = "tuetoken"
302
+ version = "0.1.0"
303
+ dependencies = [
304
+ "aho-corasick",
305
+ "fancy-regex",
306
+ "pyo3",
307
+ "rayon",
308
+ "rustc-hash",
309
+ "serde_json",
310
+ "unicode-normalization",
311
+ ]
312
+
313
+ [[package]]
314
+ name = "unicode-ident"
315
+ version = "1.0.24"
316
+ source = "registry+https://github.com/rust-lang/crates.io-index"
317
+ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
318
+
319
+ [[package]]
320
+ name = "unicode-normalization"
321
+ version = "0.1.25"
322
+ source = "registry+https://github.com/rust-lang/crates.io-index"
323
+ checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
324
+ dependencies = [
325
+ "tinyvec",
326
+ ]
327
+
328
+ [[package]]
329
+ name = "zmij"
330
+ version = "1.0.21"
331
+ source = "registry+https://github.com/rust-lang/crates.io-index"
332
+ checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
@@ -0,0 +1,39 @@
1
+ [package]
2
+ name = "tuetoken"
3
+ version = "0.1.0"
4
+ edition = "2021"
5
+ readme = "README.md"
6
+
7
+ [lib]
8
+ name = "tuetoken"
9
+ # rlib so `cargo test` can build a test harness; cdylib for the Python wheel.
10
+ crate-type = ["cdylib", "rlib"]
11
+
12
+ [dependencies]
13
+ # extension-module is enabled only for the wheel (via the `extension-module`
14
+ # feature, set in pyproject [tool.maturin]); leaving it off lets `cargo test`
15
+ # link libpython normally instead of failing on missing symbols.
16
+ pyo3 = { version = "0.28" }
17
+ serde_json = "1"
18
+ rayon = "1"
19
+ rustc-hash = "2"
20
+ # Universal pre-tokenizer fallback for ByteLevel Split patterns the hand-written
21
+ # machines don't recognize (supports the lookarounds the std `regex` crate can't).
22
+ # Same engine family HF tokenizers / tiktoken use. Only on the slow path.
23
+ fancy-regex = "0.14"
24
+ # Unicode normalization (NFC/NFD/NFKC/NFKD) for SentencePiece/byte_fallback models.
25
+ unicode-normalization = "0.1"
26
+ # Leftmost-longest multi-pattern matcher for special-token splitting in the
27
+ # AutoTokenizer path (so a whole chat string is tokenized in one Rust pass).
28
+ aho-corasick = "1"
29
+
30
+ [features]
31
+ extension-module = ["pyo3/extension-module"]
32
+
33
+ [profile.release]
34
+ opt-level = 3
35
+ lto = true
36
+ codegen-units = 1
37
+ # NB: do NOT set panic = "abort" — PyO3 catches Rust panics at the pymethods
38
+ # boundary and turns them into Python exceptions; "abort" would crash the host
39
+ # interpreter instead. Default unwind keeps a panic recoverable from Python.
tuetoken-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Vadim Borisov
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.