tuetoken 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tuetoken-0.1.0/.cargo/config.toml +8 -0
- tuetoken-0.1.0/.github/workflows/ci.yml +39 -0
- tuetoken-0.1.0/.github/workflows/wheels.yml +124 -0
- tuetoken-0.1.0/.gitignore +5 -0
- tuetoken-0.1.0/Cargo.lock +332 -0
- tuetoken-0.1.0/Cargo.toml +39 -0
- tuetoken-0.1.0/LICENSE +21 -0
- tuetoken-0.1.0/PKG-INFO +163 -0
- tuetoken-0.1.0/README.md +121 -0
- tuetoken-0.1.0/pyproject.toml +47 -0
- tuetoken-0.1.0/python/tuetoken/__init__.py +27 -0
- tuetoken-0.1.0/python/tuetoken/__init__.pyi +65 -0
- tuetoken-0.1.0/python/tuetoken/_auto.py +539 -0
- tuetoken-0.1.0/python/tuetoken/_tiktoken.py +538 -0
- tuetoken-0.1.0/python/tuetoken/py.typed +0 -0
- tuetoken-0.1.0/src/bpe.rs +677 -0
- tuetoken-0.1.0/src/cjk.rs +256 -0
- tuetoken-0.1.0/src/lib.rs +1752 -0
- tuetoken-0.1.0/src/pretok.rs +388 -0
- tuetoken-0.1.0/src/unicode.rs +235 -0
- tuetoken-0.1.0/src/unicode_tables.rs +918 -0
- tuetoken-0.1.0/tests/README.md +27 -0
- tuetoken-0.1.0/tests/_tiktoken_json.py +350 -0
- tuetoken-0.1.0/tests/bench_chat.py +107 -0
- tuetoken-0.1.0/tests/bench_models.py +105 -0
- tuetoken-0.1.0/tests/bench_three_way_rust.png +0 -0
- tuetoken-0.1.0/tests/bench_three_way_rust.py +604 -0
- tuetoken-0.1.0/tests/stress_diverse.py +357 -0
- tuetoken-0.1.0/tests/test_tuetoken.py +538 -0
- tuetoken-0.1.0/tools/gen_unicode_rs.py +96 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Builds are PORTABLE by default (generic target) so released wheels run on any
|
|
2
|
+
# x86-64 CPU — mirroring the C package, whose cibuildwheel build also drops
|
|
3
|
+
# -march=native. Measured cost of portability for this tokenizer: ~0% (C) to
|
|
4
|
+
# ~5-10% (Rust); the hot path doesn't auto-vectorize, so native buys little.
|
|
5
|
+
#
|
|
6
|
+
# For a max-speed LOCAL build (e.g. a fair benchmark vs the C version, which
|
|
7
|
+
# uses -march=native for local installs), opt in:
|
|
8
|
+
# RUSTFLAGS="-C target-cpu=native" maturin develop --release
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
test:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: actions/setup-python@v5
|
|
13
|
+
with:
|
|
14
|
+
python-version: "3.12"
|
|
15
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
16
|
+
|
|
17
|
+
# Self-contained correctness gate: the streaming-vs-scan-vs-canonical merge
|
|
18
|
+
# invariants on random tables, rank-inversion, etc. No Python or model data.
|
|
19
|
+
- name: Rust tests
|
|
20
|
+
run: cargo test --release
|
|
21
|
+
|
|
22
|
+
- name: Build and install extension
|
|
23
|
+
run: |
|
|
24
|
+
python -m pip install --upgrade pip maturin
|
|
25
|
+
maturin build --release --out dist --interpreter python
|
|
26
|
+
python -m pip install --force-reinstall dist/*.whl
|
|
27
|
+
|
|
28
|
+
# Smoke test the installed wheel: imports cleanly, tokenizes, round-trips.
|
|
29
|
+
# from_tiktoken downloads a small rank file from OpenAI's CDN (no tiktoken dep).
|
|
30
|
+
- name: Python smoke test
|
|
31
|
+
run: |
|
|
32
|
+
python - <<'PY'
|
|
33
|
+
from tuetoken import Tokenizer, AutoTokenizer
|
|
34
|
+
enc = Tokenizer.from_tiktoken("gpt2")
|
|
35
|
+
ids = enc.encode_ordinary("Hello world 1234")
|
|
36
|
+
assert enc.decode(ids) == "Hello world 1234", "round-trip failed"
|
|
37
|
+
assert enc.count_tokens("Hello world 1234") == len(ids)
|
|
38
|
+
print("smoke OK:", ids)
|
|
39
|
+
PY
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
name: wheels
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
push:
|
|
6
|
+
tags:
|
|
7
|
+
- "v*"
|
|
8
|
+
release:
|
|
9
|
+
types: [published]
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build-wheels:
|
|
13
|
+
name: ${{ matrix.platform.name }} py${{ matrix.python-version }}
|
|
14
|
+
runs-on: ${{ matrix.platform.os }}
|
|
15
|
+
strategy:
|
|
16
|
+
fail-fast: false
|
|
17
|
+
matrix:
|
|
18
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
19
|
+
platform:
|
|
20
|
+
- name: linux-x86_64
|
|
21
|
+
os: ubuntu-latest
|
|
22
|
+
target: x86_64
|
|
23
|
+
- name: linux-aarch64
|
|
24
|
+
os: ubuntu-latest
|
|
25
|
+
target: aarch64
|
|
26
|
+
- name: macos-arm64
|
|
27
|
+
os: macos-14
|
|
28
|
+
- name: windows-x86_64
|
|
29
|
+
os: windows-2022
|
|
30
|
+
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v4
|
|
33
|
+
|
|
34
|
+
- uses: actions/setup-python@v5
|
|
35
|
+
with:
|
|
36
|
+
python-version: ${{ matrix.python-version }}
|
|
37
|
+
|
|
38
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
39
|
+
|
|
40
|
+
- name: Build Linux wheel
|
|
41
|
+
if: runner.os == 'Linux'
|
|
42
|
+
uses: PyO3/maturin-action@v1
|
|
43
|
+
with:
|
|
44
|
+
command: build
|
|
45
|
+
target: ${{ matrix.platform.target }} # x86_64 (native) or aarch64 (cross)
|
|
46
|
+
args: --release --out dist -i python${{ matrix.python-version }}
|
|
47
|
+
manylinux: "2014"
|
|
48
|
+
|
|
49
|
+
- name: Build host wheel
|
|
50
|
+
if: runner.os != 'Linux'
|
|
51
|
+
uses: PyO3/maturin-action@v1
|
|
52
|
+
with:
|
|
53
|
+
command: build
|
|
54
|
+
args: --release --out dist --interpreter python
|
|
55
|
+
|
|
56
|
+
- name: Upload wheel artifact
|
|
57
|
+
uses: actions/upload-artifact@v4
|
|
58
|
+
with:
|
|
59
|
+
name: tuetoken-${{ matrix.platform.name }}-py${{ matrix.python-version }}
|
|
60
|
+
path: dist/*.whl
|
|
61
|
+
if-no-files-found: error
|
|
62
|
+
|
|
63
|
+
build-sdist:
|
|
64
|
+
name: source distribution
|
|
65
|
+
runs-on: ubuntu-latest
|
|
66
|
+
steps:
|
|
67
|
+
- uses: actions/checkout@v4
|
|
68
|
+
|
|
69
|
+
- uses: actions/setup-python@v5
|
|
70
|
+
with:
|
|
71
|
+
python-version: "3.12"
|
|
72
|
+
|
|
73
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
74
|
+
|
|
75
|
+
- name: Build sdist
|
|
76
|
+
uses: PyO3/maturin-action@v1
|
|
77
|
+
with:
|
|
78
|
+
command: sdist
|
|
79
|
+
args: --out dist
|
|
80
|
+
|
|
81
|
+
- name: Upload sdist artifact
|
|
82
|
+
uses: actions/upload-artifact@v4
|
|
83
|
+
with:
|
|
84
|
+
name: tuetoken-sdist
|
|
85
|
+
path: dist/*.tar.gz
|
|
86
|
+
if-no-files-found: error
|
|
87
|
+
|
|
88
|
+
upload-release:
|
|
89
|
+
name: attach assets to release
|
|
90
|
+
needs: [build-wheels, build-sdist]
|
|
91
|
+
if: github.event_name == 'release' && github.event.action == 'published'
|
|
92
|
+
runs-on: ubuntu-latest
|
|
93
|
+
permissions:
|
|
94
|
+
contents: write
|
|
95
|
+
steps:
|
|
96
|
+
- uses: actions/download-artifact@v4
|
|
97
|
+
with:
|
|
98
|
+
path: dist
|
|
99
|
+
merge-multiple: true
|
|
100
|
+
|
|
101
|
+
- name: Attach wheels and sdist to GitHub release
|
|
102
|
+
uses: softprops/action-gh-release@v2
|
|
103
|
+
with:
|
|
104
|
+
files: |
|
|
105
|
+
dist/*.whl
|
|
106
|
+
dist/*.tar.gz
|
|
107
|
+
|
|
108
|
+
publish-pypi:
|
|
109
|
+
name: publish to PyPI
|
|
110
|
+
needs: [build-wheels, build-sdist]
|
|
111
|
+
if: github.event_name == 'release' && github.event.action == 'published'
|
|
112
|
+
runs-on: ubuntu-latest
|
|
113
|
+
environment: pypi
|
|
114
|
+
permissions:
|
|
115
|
+
contents: read
|
|
116
|
+
id-token: write
|
|
117
|
+
steps:
|
|
118
|
+
- uses: actions/download-artifact@v4
|
|
119
|
+
with:
|
|
120
|
+
path: dist
|
|
121
|
+
merge-multiple: true
|
|
122
|
+
|
|
123
|
+
- name: Publish package distributions to PyPI
|
|
124
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "aho-corasick"
|
|
7
|
+
version = "1.1.4"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"memchr",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[[package]]
|
|
15
|
+
name = "bit-set"
|
|
16
|
+
version = "0.8.0"
|
|
17
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
+
checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
|
|
19
|
+
dependencies = [
|
|
20
|
+
"bit-vec",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[[package]]
|
|
24
|
+
name = "bit-vec"
|
|
25
|
+
version = "0.8.0"
|
|
26
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
27
|
+
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
|
28
|
+
|
|
29
|
+
[[package]]
|
|
30
|
+
name = "crossbeam-deque"
|
|
31
|
+
version = "0.8.6"
|
|
32
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
33
|
+
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
|
|
34
|
+
dependencies = [
|
|
35
|
+
"crossbeam-epoch",
|
|
36
|
+
"crossbeam-utils",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[[package]]
|
|
40
|
+
name = "crossbeam-epoch"
|
|
41
|
+
version = "0.9.18"
|
|
42
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
43
|
+
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
|
44
|
+
dependencies = [
|
|
45
|
+
"crossbeam-utils",
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
[[package]]
|
|
49
|
+
name = "crossbeam-utils"
|
|
50
|
+
version = "0.8.21"
|
|
51
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
52
|
+
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
|
53
|
+
|
|
54
|
+
[[package]]
|
|
55
|
+
name = "either"
|
|
56
|
+
version = "1.16.0"
|
|
57
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
58
|
+
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
|
|
59
|
+
|
|
60
|
+
[[package]]
|
|
61
|
+
name = "fancy-regex"
|
|
62
|
+
version = "0.14.0"
|
|
63
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
64
|
+
checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
|
|
65
|
+
dependencies = [
|
|
66
|
+
"bit-set",
|
|
67
|
+
"regex-automata",
|
|
68
|
+
"regex-syntax",
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
[[package]]
|
|
72
|
+
name = "heck"
|
|
73
|
+
version = "0.5.0"
|
|
74
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
75
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
76
|
+
|
|
77
|
+
[[package]]
|
|
78
|
+
name = "itoa"
|
|
79
|
+
version = "1.0.18"
|
|
80
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
81
|
+
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|
82
|
+
|
|
83
|
+
[[package]]
|
|
84
|
+
name = "libc"
|
|
85
|
+
version = "0.2.186"
|
|
86
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
87
|
+
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
|
88
|
+
|
|
89
|
+
[[package]]
|
|
90
|
+
name = "memchr"
|
|
91
|
+
version = "2.8.1"
|
|
92
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
93
|
+
checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
|
|
94
|
+
|
|
95
|
+
[[package]]
|
|
96
|
+
name = "once_cell"
|
|
97
|
+
version = "1.21.4"
|
|
98
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
99
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
100
|
+
|
|
101
|
+
[[package]]
|
|
102
|
+
name = "portable-atomic"
|
|
103
|
+
version = "1.13.1"
|
|
104
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
105
|
+
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
|
|
106
|
+
|
|
107
|
+
[[package]]
|
|
108
|
+
name = "proc-macro2"
|
|
109
|
+
version = "1.0.106"
|
|
110
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
111
|
+
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
|
112
|
+
dependencies = [
|
|
113
|
+
"unicode-ident",
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
[[package]]
|
|
117
|
+
name = "pyo3"
|
|
118
|
+
version = "0.28.3"
|
|
119
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
120
|
+
checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12"
|
|
121
|
+
dependencies = [
|
|
122
|
+
"libc",
|
|
123
|
+
"once_cell",
|
|
124
|
+
"portable-atomic",
|
|
125
|
+
"pyo3-build-config",
|
|
126
|
+
"pyo3-ffi",
|
|
127
|
+
"pyo3-macros",
|
|
128
|
+
]
|
|
129
|
+
|
|
130
|
+
[[package]]
|
|
131
|
+
name = "pyo3-build-config"
|
|
132
|
+
version = "0.28.3"
|
|
133
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
134
|
+
checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e"
|
|
135
|
+
dependencies = [
|
|
136
|
+
"target-lexicon",
|
|
137
|
+
]
|
|
138
|
+
|
|
139
|
+
[[package]]
|
|
140
|
+
name = "pyo3-ffi"
|
|
141
|
+
version = "0.28.3"
|
|
142
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
143
|
+
checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e"
|
|
144
|
+
dependencies = [
|
|
145
|
+
"libc",
|
|
146
|
+
"pyo3-build-config",
|
|
147
|
+
]
|
|
148
|
+
|
|
149
|
+
[[package]]
|
|
150
|
+
name = "pyo3-macros"
|
|
151
|
+
version = "0.28.3"
|
|
152
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
153
|
+
checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813"
|
|
154
|
+
dependencies = [
|
|
155
|
+
"proc-macro2",
|
|
156
|
+
"pyo3-macros-backend",
|
|
157
|
+
"quote",
|
|
158
|
+
"syn",
|
|
159
|
+
]
|
|
160
|
+
|
|
161
|
+
[[package]]
|
|
162
|
+
name = "pyo3-macros-backend"
|
|
163
|
+
version = "0.28.3"
|
|
164
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
165
|
+
checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb"
|
|
166
|
+
dependencies = [
|
|
167
|
+
"heck",
|
|
168
|
+
"proc-macro2",
|
|
169
|
+
"pyo3-build-config",
|
|
170
|
+
"quote",
|
|
171
|
+
"syn",
|
|
172
|
+
]
|
|
173
|
+
|
|
174
|
+
[[package]]
|
|
175
|
+
name = "quote"
|
|
176
|
+
version = "1.0.45"
|
|
177
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
178
|
+
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
|
179
|
+
dependencies = [
|
|
180
|
+
"proc-macro2",
|
|
181
|
+
]
|
|
182
|
+
|
|
183
|
+
[[package]]
|
|
184
|
+
name = "rayon"
|
|
185
|
+
version = "1.12.0"
|
|
186
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
187
|
+
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
|
188
|
+
dependencies = [
|
|
189
|
+
"either",
|
|
190
|
+
"rayon-core",
|
|
191
|
+
]
|
|
192
|
+
|
|
193
|
+
[[package]]
|
|
194
|
+
name = "rayon-core"
|
|
195
|
+
version = "1.13.0"
|
|
196
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
197
|
+
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
|
198
|
+
dependencies = [
|
|
199
|
+
"crossbeam-deque",
|
|
200
|
+
"crossbeam-utils",
|
|
201
|
+
]
|
|
202
|
+
|
|
203
|
+
[[package]]
|
|
204
|
+
name = "regex-automata"
|
|
205
|
+
version = "0.4.14"
|
|
206
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
207
|
+
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
|
208
|
+
dependencies = [
|
|
209
|
+
"aho-corasick",
|
|
210
|
+
"memchr",
|
|
211
|
+
"regex-syntax",
|
|
212
|
+
]
|
|
213
|
+
|
|
214
|
+
[[package]]
|
|
215
|
+
name = "regex-syntax"
|
|
216
|
+
version = "0.8.10"
|
|
217
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
218
|
+
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
|
219
|
+
|
|
220
|
+
[[package]]
|
|
221
|
+
name = "rustc-hash"
|
|
222
|
+
version = "2.1.2"
|
|
223
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
224
|
+
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
|
225
|
+
|
|
226
|
+
[[package]]
|
|
227
|
+
name = "serde"
|
|
228
|
+
version = "1.0.228"
|
|
229
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
230
|
+
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
|
231
|
+
dependencies = [
|
|
232
|
+
"serde_core",
|
|
233
|
+
]
|
|
234
|
+
|
|
235
|
+
[[package]]
|
|
236
|
+
name = "serde_core"
|
|
237
|
+
version = "1.0.228"
|
|
238
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
239
|
+
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
|
240
|
+
dependencies = [
|
|
241
|
+
"serde_derive",
|
|
242
|
+
]
|
|
243
|
+
|
|
244
|
+
[[package]]
|
|
245
|
+
name = "serde_derive"
|
|
246
|
+
version = "1.0.228"
|
|
247
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
248
|
+
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
|
249
|
+
dependencies = [
|
|
250
|
+
"proc-macro2",
|
|
251
|
+
"quote",
|
|
252
|
+
"syn",
|
|
253
|
+
]
|
|
254
|
+
|
|
255
|
+
[[package]]
|
|
256
|
+
name = "serde_json"
|
|
257
|
+
version = "1.0.150"
|
|
258
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
259
|
+
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
|
260
|
+
dependencies = [
|
|
261
|
+
"itoa",
|
|
262
|
+
"memchr",
|
|
263
|
+
"serde",
|
|
264
|
+
"serde_core",
|
|
265
|
+
"zmij",
|
|
266
|
+
]
|
|
267
|
+
|
|
268
|
+
[[package]]
|
|
269
|
+
name = "syn"
|
|
270
|
+
version = "2.0.117"
|
|
271
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
272
|
+
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
|
273
|
+
dependencies = [
|
|
274
|
+
"proc-macro2",
|
|
275
|
+
"quote",
|
|
276
|
+
"unicode-ident",
|
|
277
|
+
]
|
|
278
|
+
|
|
279
|
+
[[package]]
|
|
280
|
+
name = "target-lexicon"
|
|
281
|
+
version = "0.13.5"
|
|
282
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
283
|
+
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
284
|
+
|
|
285
|
+
[[package]]
|
|
286
|
+
name = "tinyvec"
|
|
287
|
+
version = "1.11.0"
|
|
288
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
289
|
+
checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
|
|
290
|
+
dependencies = [
|
|
291
|
+
"tinyvec_macros",
|
|
292
|
+
]
|
|
293
|
+
|
|
294
|
+
[[package]]
|
|
295
|
+
name = "tinyvec_macros"
|
|
296
|
+
version = "0.1.1"
|
|
297
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
298
|
+
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
|
299
|
+
|
|
300
|
+
[[package]]
|
|
301
|
+
name = "tuetoken"
|
|
302
|
+
version = "0.1.0"
|
|
303
|
+
dependencies = [
|
|
304
|
+
"aho-corasick",
|
|
305
|
+
"fancy-regex",
|
|
306
|
+
"pyo3",
|
|
307
|
+
"rayon",
|
|
308
|
+
"rustc-hash",
|
|
309
|
+
"serde_json",
|
|
310
|
+
"unicode-normalization",
|
|
311
|
+
]
|
|
312
|
+
|
|
313
|
+
[[package]]
|
|
314
|
+
name = "unicode-ident"
|
|
315
|
+
version = "1.0.24"
|
|
316
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
317
|
+
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
|
318
|
+
|
|
319
|
+
[[package]]
|
|
320
|
+
name = "unicode-normalization"
|
|
321
|
+
version = "0.1.25"
|
|
322
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
323
|
+
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
|
|
324
|
+
dependencies = [
|
|
325
|
+
"tinyvec",
|
|
326
|
+
]
|
|
327
|
+
|
|
328
|
+
[[package]]
|
|
329
|
+
name = "zmij"
|
|
330
|
+
version = "1.0.21"
|
|
331
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
332
|
+
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "tuetoken"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
|
|
7
|
+
[lib]
|
|
8
|
+
name = "tuetoken"
|
|
9
|
+
# rlib so `cargo test` can build a test harness; cdylib for the Python wheel.
|
|
10
|
+
crate-type = ["cdylib", "rlib"]
|
|
11
|
+
|
|
12
|
+
[dependencies]
|
|
13
|
+
# extension-module is enabled only for the wheel (via the `extension-module`
|
|
14
|
+
# feature, set in pyproject [tool.maturin]); leaving it off lets `cargo test`
|
|
15
|
+
# link libpython normally instead of failing on missing symbols.
|
|
16
|
+
pyo3 = { version = "0.28" }
|
|
17
|
+
serde_json = "1"
|
|
18
|
+
rayon = "1"
|
|
19
|
+
rustc-hash = "2"
|
|
20
|
+
# Universal pre-tokenizer fallback for ByteLevel Split patterns the hand-written
|
|
21
|
+
# machines don't recognize (supports the lookarounds the std `regex` crate can't).
|
|
22
|
+
# Same engine family HF tokenizers / tiktoken use. Only on the slow path.
|
|
23
|
+
fancy-regex = "0.14"
|
|
24
|
+
# Unicode normalization (NFC/NFD/NFKC/NFKD) for SentencePiece/byte_fallback models.
|
|
25
|
+
unicode-normalization = "0.1"
|
|
26
|
+
# Leftmost-longest multi-pattern matcher for special-token splitting in the
|
|
27
|
+
# AutoTokenizer path (so a whole chat string is tokenized in one Rust pass).
|
|
28
|
+
aho-corasick = "1"
|
|
29
|
+
|
|
30
|
+
[features]
|
|
31
|
+
extension-module = ["pyo3/extension-module"]
|
|
32
|
+
|
|
33
|
+
[profile.release]
|
|
34
|
+
opt-level = 3
|
|
35
|
+
lto = true
|
|
36
|
+
codegen-units = 1
|
|
37
|
+
# NB: do NOT set panic = "abort" — PyO3 catches Rust panics at the pymethods
|
|
38
|
+
# boundary and turns them into Python exceptions; "abort" would crash the host
|
|
39
|
+
# interpreter instead. Default unwind keeps a panic recoverable from Python.
|
tuetoken-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Vadim Borisov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|