bpe-tokenizer-editor 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bpe_tokenizer_editor-0.1.0/.github/workflows/CI.yml +182 -0
- bpe_tokenizer_editor-0.1.0/.gitignore +53 -0
- bpe_tokenizer_editor-0.1.0/Cargo.lock +397 -0
- bpe_tokenizer_editor-0.1.0/Cargo.toml +30 -0
- bpe_tokenizer_editor-0.1.0/LICENSE +21 -0
- bpe_tokenizer_editor-0.1.0/Makefile +56 -0
- bpe_tokenizer_editor-0.1.0/PKG-INFO +359 -0
- bpe_tokenizer_editor-0.1.0/README.md +404 -0
- bpe_tokenizer_editor-0.1.0/README_PYTHON.md +326 -0
- bpe_tokenizer_editor-0.1.0/examples/example.py +111 -0
- bpe_tokenizer_editor-0.1.0/pyproject.toml +89 -0
- bpe_tokenizer_editor-0.1.0/python/bpe_tokenizer_editor/__init__.py +27 -0
- bpe_tokenizer_editor-0.1.0/python/bpe_tokenizer_editor/__init__.pyi +524 -0
- bpe_tokenizer_editor-0.1.0/python/bpe_tokenizer_editor/py.typed +2 -0
- bpe_tokenizer_editor-0.1.0/src/cli.rs +182 -0
- bpe_tokenizer_editor-0.1.0/src/editor/addition.rs +191 -0
- bpe_tokenizer_editor-0.1.0/src/editor/core.rs +111 -0
- bpe_tokenizer_editor-0.1.0/src/editor/management.rs +238 -0
- bpe_tokenizer_editor-0.1.0/src/editor/mod.rs +11 -0
- bpe_tokenizer_editor-0.1.0/src/editor/reindex.rs +309 -0
- bpe_tokenizer_editor-0.1.0/src/editor/removal.rs +77 -0
- bpe_tokenizer_editor-0.1.0/src/editor/sync.rs +323 -0
- bpe_tokenizer_editor-0.1.0/src/editor/validation.rs +48 -0
- bpe_tokenizer_editor-0.1.0/src/lib.rs +18 -0
- bpe_tokenizer_editor-0.1.0/src/main.rs +622 -0
- bpe_tokenizer_editor-0.1.0/src/python.rs +640 -0
- bpe_tokenizer_editor-0.1.0/src/tokenizer.rs +73 -0
- bpe_tokenizer_editor-0.1.0/src/types.rs +128 -0
- bpe_tokenizer_editor-0.1.0/tests/test_python.py +290 -0
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# Generated with maturin v1.14.1 and customized for regular CPython abi3 wheels.
|
|
2
|
+
# To regenerate the base workflow, run
|
|
3
|
+
#
|
|
4
|
+
# maturin generate-ci github
|
|
5
|
+
#
|
|
6
|
+
name: CI
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
branches:
|
|
11
|
+
- main
|
|
12
|
+
- master
|
|
13
|
+
tags:
|
|
14
|
+
- '*'
|
|
15
|
+
pull_request:
|
|
16
|
+
workflow_dispatch:
|
|
17
|
+
|
|
18
|
+
permissions:
|
|
19
|
+
contents: read
|
|
20
|
+
|
|
21
|
+
jobs:
|
|
22
|
+
linux:
|
|
23
|
+
runs-on: ${{ matrix.platform.runner }}
|
|
24
|
+
strategy:
|
|
25
|
+
matrix:
|
|
26
|
+
platform:
|
|
27
|
+
- runner: ubuntu-22.04
|
|
28
|
+
target: x86_64
|
|
29
|
+
- runner: ubuntu-22.04
|
|
30
|
+
target: x86
|
|
31
|
+
- runner: ubuntu-22.04
|
|
32
|
+
target: aarch64
|
|
33
|
+
- runner: ubuntu-22.04
|
|
34
|
+
target: armv7
|
|
35
|
+
- runner: ubuntu-22.04
|
|
36
|
+
target: s390x
|
|
37
|
+
- runner: ubuntu-22.04
|
|
38
|
+
target: ppc64le
|
|
39
|
+
steps:
|
|
40
|
+
- uses: actions/checkout@v6
|
|
41
|
+
- uses: actions/setup-python@v6
|
|
42
|
+
with:
|
|
43
|
+
python-version: "3.8"
|
|
44
|
+
- name: Build wheels
|
|
45
|
+
uses: PyO3/maturin-action@v1
|
|
46
|
+
with:
|
|
47
|
+
target: ${{ matrix.platform.target }}
|
|
48
|
+
args: --release --out dist
|
|
49
|
+
sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
|
50
|
+
manylinux: auto
|
|
51
|
+
- name: Upload wheels
|
|
52
|
+
uses: actions/upload-artifact@v6
|
|
53
|
+
with:
|
|
54
|
+
name: wheels-linux-${{ matrix.platform.target }}
|
|
55
|
+
path: dist
|
|
56
|
+
|
|
57
|
+
musllinux:
|
|
58
|
+
runs-on: ${{ matrix.platform.runner }}
|
|
59
|
+
strategy:
|
|
60
|
+
matrix:
|
|
61
|
+
platform:
|
|
62
|
+
- runner: ubuntu-22.04
|
|
63
|
+
target: x86_64
|
|
64
|
+
- runner: ubuntu-22.04
|
|
65
|
+
target: x86
|
|
66
|
+
- runner: ubuntu-22.04
|
|
67
|
+
target: aarch64
|
|
68
|
+
- runner: ubuntu-22.04
|
|
69
|
+
target: armv7
|
|
70
|
+
steps:
|
|
71
|
+
- uses: actions/checkout@v6
|
|
72
|
+
- uses: actions/setup-python@v6
|
|
73
|
+
with:
|
|
74
|
+
python-version: "3.8"
|
|
75
|
+
- name: Build wheels
|
|
76
|
+
uses: PyO3/maturin-action@v1
|
|
77
|
+
with:
|
|
78
|
+
target: ${{ matrix.platform.target }}
|
|
79
|
+
args: --release --out dist
|
|
80
|
+
sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
|
81
|
+
manylinux: musllinux_1_2
|
|
82
|
+
- name: Upload wheels
|
|
83
|
+
uses: actions/upload-artifact@v6
|
|
84
|
+
with:
|
|
85
|
+
name: wheels-musllinux-${{ matrix.platform.target }}
|
|
86
|
+
path: dist
|
|
87
|
+
|
|
88
|
+
windows:
|
|
89
|
+
runs-on: ${{ matrix.platform.runner }}
|
|
90
|
+
strategy:
|
|
91
|
+
matrix:
|
|
92
|
+
platform:
|
|
93
|
+
- runner: windows-latest
|
|
94
|
+
target: x64
|
|
95
|
+
python_arch: x64
|
|
96
|
+
- runner: windows-latest
|
|
97
|
+
target: x86
|
|
98
|
+
python_arch: x86
|
|
99
|
+
- runner: windows-11-arm
|
|
100
|
+
target: aarch64
|
|
101
|
+
python_arch: arm64
|
|
102
|
+
steps:
|
|
103
|
+
- uses: actions/checkout@v6
|
|
104
|
+
- uses: actions/setup-python@v6
|
|
105
|
+
with:
|
|
106
|
+
python-version: "3.13"
|
|
107
|
+
architecture: ${{ matrix.platform.python_arch }}
|
|
108
|
+
- name: Build wheels
|
|
109
|
+
uses: PyO3/maturin-action@v1
|
|
110
|
+
with:
|
|
111
|
+
target: ${{ matrix.platform.target }}
|
|
112
|
+
args: --release --out dist
|
|
113
|
+
sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
|
114
|
+
- name: Upload wheels
|
|
115
|
+
uses: actions/upload-artifact@v6
|
|
116
|
+
with:
|
|
117
|
+
name: wheels-windows-${{ matrix.platform.target }}
|
|
118
|
+
path: dist
|
|
119
|
+
|
|
120
|
+
macos:
|
|
121
|
+
runs-on: ${{ matrix.platform.runner }}
|
|
122
|
+
strategy:
|
|
123
|
+
matrix:
|
|
124
|
+
platform:
|
|
125
|
+
- runner: macos-15-intel
|
|
126
|
+
target: x86_64
|
|
127
|
+
- runner: macos-latest
|
|
128
|
+
target: aarch64
|
|
129
|
+
steps:
|
|
130
|
+
- uses: actions/checkout@v6
|
|
131
|
+
- uses: actions/setup-python@v6
|
|
132
|
+
with:
|
|
133
|
+
python-version: "3.8"
|
|
134
|
+
- name: Build wheels
|
|
135
|
+
uses: PyO3/maturin-action@v1
|
|
136
|
+
with:
|
|
137
|
+
target: ${{ matrix.platform.target }}
|
|
138
|
+
args: --release --out dist
|
|
139
|
+
sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
|
140
|
+
- name: Upload wheels
|
|
141
|
+
uses: actions/upload-artifact@v6
|
|
142
|
+
with:
|
|
143
|
+
name: wheels-macos-${{ matrix.platform.target }}
|
|
144
|
+
path: dist
|
|
145
|
+
|
|
146
|
+
sdist:
|
|
147
|
+
runs-on: ubuntu-latest
|
|
148
|
+
steps:
|
|
149
|
+
- uses: actions/checkout@v6
|
|
150
|
+
- name: Build sdist
|
|
151
|
+
uses: PyO3/maturin-action@v1
|
|
152
|
+
with:
|
|
153
|
+
command: sdist
|
|
154
|
+
args: --out dist
|
|
155
|
+
- name: Upload sdist
|
|
156
|
+
uses: actions/upload-artifact@v6
|
|
157
|
+
with:
|
|
158
|
+
name: wheels-sdist
|
|
159
|
+
path: dist
|
|
160
|
+
|
|
161
|
+
release:
|
|
162
|
+
name: Release
|
|
163
|
+
runs-on: ubuntu-latest
|
|
164
|
+
if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
|
|
165
|
+
needs: [linux, musllinux, windows, macos, sdist]
|
|
166
|
+
environment: pypi
|
|
167
|
+
permissions:
|
|
168
|
+
id-token: write
|
|
169
|
+
contents: write
|
|
170
|
+
attestations: write
|
|
171
|
+
steps:
|
|
172
|
+
- uses: actions/download-artifact@v7
|
|
173
|
+
- name: Generate artifact attestation
|
|
174
|
+
uses: actions/attest@v4
|
|
175
|
+
with:
|
|
176
|
+
subject-path: 'wheels-*/*'
|
|
177
|
+
- name: Install uv
|
|
178
|
+
if: ${{ startsWith(github.ref, 'refs/tags/') }}
|
|
179
|
+
uses: astral-sh/setup-uv@v7
|
|
180
|
+
- name: Publish to PyPI
|
|
181
|
+
if: ${{ startsWith(github.ref, 'refs/tags/') }}
|
|
182
|
+
run: uv publish --trusted-publishing always 'wheels-*/*'
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Rust related files
|
|
2
|
+
/target/
|
|
3
|
+
**/*.rs.bk
|
|
4
|
+
**/*.rs.swp
|
|
5
|
+
**/*.rs.swo
|
|
6
|
+
**/*.rlib
|
|
7
|
+
**/*.rmeta
|
|
8
|
+
**/*.dSYM/
|
|
9
|
+
**/*.cargo-lock
|
|
10
|
+
**/Cargo.lock
|
|
11
|
+
|
|
12
|
+
# Python
|
|
13
|
+
__pycache__/
|
|
14
|
+
*.py[cod]
|
|
15
|
+
*$py.class
|
|
16
|
+
*.so
|
|
17
|
+
*.pyd
|
|
18
|
+
.Python
|
|
19
|
+
build/
|
|
20
|
+
develop-eggs/
|
|
21
|
+
dist/
|
|
22
|
+
downloads/
|
|
23
|
+
eggs/
|
|
24
|
+
.eggs/
|
|
25
|
+
lib/
|
|
26
|
+
lib64/
|
|
27
|
+
parts/
|
|
28
|
+
sdist/
|
|
29
|
+
var/
|
|
30
|
+
wheels/
|
|
31
|
+
*.egg-info/
|
|
32
|
+
.installed.cfg
|
|
33
|
+
*.egg
|
|
34
|
+
.pytest_cache/
|
|
35
|
+
.mypy_cache/
|
|
36
|
+
*.whl
|
|
37
|
+
|
|
38
|
+
# Virtual environments
|
|
39
|
+
.venv/
|
|
40
|
+
venv/
|
|
41
|
+
ENV/
|
|
42
|
+
|
|
43
|
+
# IDE related files
|
|
44
|
+
/.idea/
|
|
45
|
+
*.iml
|
|
46
|
+
.vscode/
|
|
47
|
+
|
|
48
|
+
# OS generated files
|
|
49
|
+
.DS_Store
|
|
50
|
+
Thumbs.db
|
|
51
|
+
*~
|
|
52
|
+
|
|
53
|
+
.env
|
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "anstream"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"anstyle",
|
|
12
|
+
"anstyle-parse",
|
|
13
|
+
"anstyle-query",
|
|
14
|
+
"anstyle-wincon",
|
|
15
|
+
"colorchoice",
|
|
16
|
+
"is_terminal_polyfill",
|
|
17
|
+
"utf8parse",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[[package]]
|
|
21
|
+
name = "anstyle"
|
|
22
|
+
version = "1.0.14"
|
|
23
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
24
|
+
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
|
25
|
+
|
|
26
|
+
[[package]]
|
|
27
|
+
name = "anstyle-parse"
|
|
28
|
+
version = "1.0.0"
|
|
29
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
30
|
+
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
|
31
|
+
dependencies = [
|
|
32
|
+
"utf8parse",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[[package]]
|
|
36
|
+
name = "anstyle-query"
|
|
37
|
+
version = "1.1.5"
|
|
38
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
39
|
+
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
|
40
|
+
dependencies = [
|
|
41
|
+
"windows-sys",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[[package]]
|
|
45
|
+
name = "anstyle-wincon"
|
|
46
|
+
version = "3.0.11"
|
|
47
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
48
|
+
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
|
49
|
+
dependencies = [
|
|
50
|
+
"anstyle",
|
|
51
|
+
"once_cell_polyfill",
|
|
52
|
+
"windows-sys",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
[[package]]
|
|
56
|
+
name = "anyhow"
|
|
57
|
+
version = "1.0.104"
|
|
58
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
59
|
+
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
|
60
|
+
|
|
61
|
+
[[package]]
|
|
62
|
+
name = "autocfg"
|
|
63
|
+
version = "1.5.1"
|
|
64
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
65
|
+
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
|
66
|
+
|
|
67
|
+
[[package]]
|
|
68
|
+
name = "bpe-tokenizer-editor"
|
|
69
|
+
version = "0.1.0"
|
|
70
|
+
dependencies = [
|
|
71
|
+
"anyhow",
|
|
72
|
+
"clap",
|
|
73
|
+
"pyo3",
|
|
74
|
+
"serde",
|
|
75
|
+
"serde_json",
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
[[package]]
|
|
79
|
+
name = "cfg-if"
|
|
80
|
+
version = "1.0.4"
|
|
81
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
82
|
+
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
|
83
|
+
|
|
84
|
+
[[package]]
|
|
85
|
+
name = "clap"
|
|
86
|
+
version = "4.6.6"
|
|
87
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
88
|
+
checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca"
|
|
89
|
+
dependencies = [
|
|
90
|
+
"clap_builder",
|
|
91
|
+
"clap_derive",
|
|
92
|
+
]
|
|
93
|
+
|
|
94
|
+
[[package]]
|
|
95
|
+
name = "clap_builder"
|
|
96
|
+
version = "4.6.6"
|
|
97
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
98
|
+
checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889"
|
|
99
|
+
dependencies = [
|
|
100
|
+
"anstream",
|
|
101
|
+
"anstyle",
|
|
102
|
+
"clap_lex",
|
|
103
|
+
"strsim",
|
|
104
|
+
]
|
|
105
|
+
|
|
106
|
+
[[package]]
|
|
107
|
+
name = "clap_derive"
|
|
108
|
+
version = "4.6.4"
|
|
109
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
110
|
+
checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
|
|
111
|
+
dependencies = [
|
|
112
|
+
"heck",
|
|
113
|
+
"proc-macro2",
|
|
114
|
+
"quote",
|
|
115
|
+
"syn 3.0.3",
|
|
116
|
+
]
|
|
117
|
+
|
|
118
|
+
[[package]]
|
|
119
|
+
name = "clap_lex"
|
|
120
|
+
version = "1.1.0"
|
|
121
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
122
|
+
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
|
123
|
+
|
|
124
|
+
[[package]]
|
|
125
|
+
name = "colorchoice"
|
|
126
|
+
version = "1.0.5"
|
|
127
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
128
|
+
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
|
129
|
+
|
|
130
|
+
[[package]]
|
|
131
|
+
name = "heck"
|
|
132
|
+
version = "0.5.0"
|
|
133
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
134
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
135
|
+
|
|
136
|
+
[[package]]
|
|
137
|
+
name = "indoc"
|
|
138
|
+
version = "2.0.7"
|
|
139
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
140
|
+
checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
|
|
141
|
+
dependencies = [
|
|
142
|
+
"rustversion",
|
|
143
|
+
]
|
|
144
|
+
|
|
145
|
+
[[package]]
|
|
146
|
+
name = "is_terminal_polyfill"
|
|
147
|
+
version = "1.70.2"
|
|
148
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
149
|
+
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
|
150
|
+
|
|
151
|
+
[[package]]
|
|
152
|
+
name = "itoa"
|
|
153
|
+
version = "1.0.18"
|
|
154
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
155
|
+
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|
156
|
+
|
|
157
|
+
[[package]]
|
|
158
|
+
name = "libc"
|
|
159
|
+
version = "0.2.189"
|
|
160
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
161
|
+
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
|
162
|
+
|
|
163
|
+
[[package]]
|
|
164
|
+
name = "memchr"
|
|
165
|
+
version = "2.8.3"
|
|
166
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
167
|
+
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
168
|
+
|
|
169
|
+
[[package]]
|
|
170
|
+
name = "memoffset"
|
|
171
|
+
version = "0.9.1"
|
|
172
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
173
|
+
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
|
|
174
|
+
dependencies = [
|
|
175
|
+
"autocfg",
|
|
176
|
+
]
|
|
177
|
+
|
|
178
|
+
[[package]]
|
|
179
|
+
name = "once_cell"
|
|
180
|
+
version = "1.21.4"
|
|
181
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
182
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
183
|
+
|
|
184
|
+
[[package]]
|
|
185
|
+
name = "once_cell_polyfill"
|
|
186
|
+
version = "1.70.2"
|
|
187
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
188
|
+
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
|
189
|
+
|
|
190
|
+
[[package]]
|
|
191
|
+
name = "portable-atomic"
|
|
192
|
+
version = "1.15.0"
|
|
193
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
194
|
+
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
195
|
+
|
|
196
|
+
[[package]]
|
|
197
|
+
name = "proc-macro2"
|
|
198
|
+
version = "1.0.107"
|
|
199
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
200
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
201
|
+
dependencies = [
|
|
202
|
+
"unicode-ident",
|
|
203
|
+
]
|
|
204
|
+
|
|
205
|
+
[[package]]
|
|
206
|
+
name = "pyo3"
|
|
207
|
+
version = "0.22.6"
|
|
208
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
209
|
+
checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
|
|
210
|
+
dependencies = [
|
|
211
|
+
"cfg-if",
|
|
212
|
+
"indoc",
|
|
213
|
+
"libc",
|
|
214
|
+
"memoffset",
|
|
215
|
+
"once_cell",
|
|
216
|
+
"portable-atomic",
|
|
217
|
+
"pyo3-build-config",
|
|
218
|
+
"pyo3-ffi",
|
|
219
|
+
"pyo3-macros",
|
|
220
|
+
"unindent",
|
|
221
|
+
]
|
|
222
|
+
|
|
223
|
+
[[package]]
|
|
224
|
+
name = "pyo3-build-config"
|
|
225
|
+
version = "0.22.6"
|
|
226
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
227
|
+
checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38"
|
|
228
|
+
dependencies = [
|
|
229
|
+
"once_cell",
|
|
230
|
+
"target-lexicon",
|
|
231
|
+
]
|
|
232
|
+
|
|
233
|
+
[[package]]
|
|
234
|
+
name = "pyo3-ffi"
|
|
235
|
+
version = "0.22.6"
|
|
236
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
237
|
+
checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636"
|
|
238
|
+
dependencies = [
|
|
239
|
+
"libc",
|
|
240
|
+
"pyo3-build-config",
|
|
241
|
+
]
|
|
242
|
+
|
|
243
|
+
[[package]]
|
|
244
|
+
name = "pyo3-macros"
|
|
245
|
+
version = "0.22.6"
|
|
246
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
247
|
+
checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453"
|
|
248
|
+
dependencies = [
|
|
249
|
+
"proc-macro2",
|
|
250
|
+
"pyo3-macros-backend",
|
|
251
|
+
"quote",
|
|
252
|
+
"syn 2.0.119",
|
|
253
|
+
]
|
|
254
|
+
|
|
255
|
+
[[package]]
|
|
256
|
+
name = "pyo3-macros-backend"
|
|
257
|
+
version = "0.22.6"
|
|
258
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
259
|
+
checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe"
|
|
260
|
+
dependencies = [
|
|
261
|
+
"heck",
|
|
262
|
+
"proc-macro2",
|
|
263
|
+
"pyo3-build-config",
|
|
264
|
+
"quote",
|
|
265
|
+
"syn 2.0.119",
|
|
266
|
+
]
|
|
267
|
+
|
|
268
|
+
[[package]]
|
|
269
|
+
name = "quote"
|
|
270
|
+
version = "1.0.47"
|
|
271
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
272
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
273
|
+
dependencies = [
|
|
274
|
+
"proc-macro2",
|
|
275
|
+
]
|
|
276
|
+
|
|
277
|
+
[[package]]
|
|
278
|
+
name = "rustversion"
|
|
279
|
+
version = "1.0.23"
|
|
280
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
281
|
+
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
|
282
|
+
|
|
283
|
+
[[package]]
|
|
284
|
+
name = "serde"
|
|
285
|
+
version = "1.0.229"
|
|
286
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
287
|
+
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
|
288
|
+
dependencies = [
|
|
289
|
+
"serde_core",
|
|
290
|
+
"serde_derive",
|
|
291
|
+
]
|
|
292
|
+
|
|
293
|
+
[[package]]
|
|
294
|
+
name = "serde_core"
|
|
295
|
+
version = "1.0.229"
|
|
296
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
297
|
+
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
|
298
|
+
dependencies = [
|
|
299
|
+
"serde_derive",
|
|
300
|
+
]
|
|
301
|
+
|
|
302
|
+
[[package]]
|
|
303
|
+
name = "serde_derive"
|
|
304
|
+
version = "1.0.229"
|
|
305
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
306
|
+
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
307
|
+
dependencies = [
|
|
308
|
+
"proc-macro2",
|
|
309
|
+
"quote",
|
|
310
|
+
"syn 3.0.3",
|
|
311
|
+
]
|
|
312
|
+
|
|
313
|
+
[[package]]
|
|
314
|
+
name = "serde_json"
|
|
315
|
+
version = "1.0.151"
|
|
316
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
317
|
+
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
|
318
|
+
dependencies = [
|
|
319
|
+
"itoa",
|
|
320
|
+
"memchr",
|
|
321
|
+
"serde",
|
|
322
|
+
"serde_core",
|
|
323
|
+
"zmij",
|
|
324
|
+
]
|
|
325
|
+
|
|
326
|
+
[[package]]
|
|
327
|
+
name = "strsim"
|
|
328
|
+
version = "0.11.1"
|
|
329
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
330
|
+
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
|
331
|
+
|
|
332
|
+
[[package]]
|
|
333
|
+
name = "syn"
|
|
334
|
+
version = "2.0.119"
|
|
335
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
336
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
337
|
+
dependencies = [
|
|
338
|
+
"proc-macro2",
|
|
339
|
+
"quote",
|
|
340
|
+
"unicode-ident",
|
|
341
|
+
]
|
|
342
|
+
|
|
343
|
+
[[package]]
|
|
344
|
+
name = "syn"
|
|
345
|
+
version = "3.0.3"
|
|
346
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
347
|
+
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
|
|
348
|
+
dependencies = [
|
|
349
|
+
"proc-macro2",
|
|
350
|
+
"quote",
|
|
351
|
+
"unicode-ident",
|
|
352
|
+
]
|
|
353
|
+
|
|
354
|
+
[[package]]
|
|
355
|
+
name = "target-lexicon"
|
|
356
|
+
version = "0.12.16"
|
|
357
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
358
|
+
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
|
|
359
|
+
|
|
360
|
+
[[package]]
|
|
361
|
+
name = "unicode-ident"
|
|
362
|
+
version = "1.0.24"
|
|
363
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
364
|
+
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
|
365
|
+
|
|
366
|
+
[[package]]
|
|
367
|
+
name = "unindent"
|
|
368
|
+
version = "0.2.4"
|
|
369
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
370
|
+
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
|
|
371
|
+
|
|
372
|
+
[[package]]
|
|
373
|
+
name = "utf8parse"
|
|
374
|
+
version = "0.2.2"
|
|
375
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
376
|
+
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
|
377
|
+
|
|
378
|
+
[[package]]
|
|
379
|
+
name = "windows-link"
|
|
380
|
+
version = "0.2.1"
|
|
381
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
382
|
+
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
|
383
|
+
|
|
384
|
+
[[package]]
|
|
385
|
+
name = "windows-sys"
|
|
386
|
+
version = "0.61.2"
|
|
387
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
388
|
+
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
|
389
|
+
dependencies = [
|
|
390
|
+
"windows-link",
|
|
391
|
+
]
|
|
392
|
+
|
|
393
|
+
[[package]]
|
|
394
|
+
name = "zmij"
|
|
395
|
+
version = "1.0.23"
|
|
396
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
397
|
+
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "bpe-tokenizer-editor"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
description = "BPE Tokenizer Editor - Add/Remove tokens with consistency guarantees"
|
|
6
|
+
authors = ["Ali Bayram"]
|
|
7
|
+
license = "MIT"
|
|
8
|
+
repository = "https://github.com/malibayram/bpe-tokenizer-editor"
|
|
9
|
+
keywords = ["tokenizer", "bpe", "nlp", "huggingface"]
|
|
10
|
+
categories = ["text-processing", "science"]
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
|
|
13
|
+
[lib]
|
|
14
|
+
name = "bpe_tokenizer_editor"
|
|
15
|
+
crate-type = ["cdylib", "rlib"]
|
|
16
|
+
|
|
17
|
+
[[bin]]
|
|
18
|
+
name = "bpe-editor"
|
|
19
|
+
path = "src/main.rs"
|
|
20
|
+
|
|
21
|
+
[features]
|
|
22
|
+
default = []
|
|
23
|
+
python = ["pyo3/extension-module"]
|
|
24
|
+
|
|
25
|
+
[dependencies]
|
|
26
|
+
anyhow = "1"
|
|
27
|
+
clap = { version = "4", features = ["derive"] }
|
|
28
|
+
serde = { version = "1", features = ["derive"] }
|
|
29
|
+
serde_json = "1"
|
|
30
|
+
pyo3 = { version = "0.22", optional = true, features = ["abi3-py38"] }
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Ali Bayram
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|