sacremoses-rs 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,48 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ lint:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@v4
13
+ - uses: dtolnay/rust-toolchain@stable
14
+ with:
15
+ components: rustfmt, clippy
16
+ - run: cargo fmt --check
17
+ - run: cargo clippy --release -- -D warnings
18
+
19
+ test:
20
+ runs-on: ${{ matrix.os }}
21
+ strategy:
22
+ fail-fast: false
23
+ matrix:
24
+ os: [ubuntu-24.04, macos-latest, windows-latest]
25
+ python: ["3.9", "3.13"]
26
+ env:
27
+ PYTHONUTF8: "1"
28
+ steps:
29
+ - uses: actions/checkout@v4
30
+ - uses: actions/setup-python@v5
31
+ with:
32
+ python-version: ${{ matrix.python }}
33
+ - uses: dtolnay/rust-toolchain@stable
34
+ - name: Build wheel and install
35
+ shell: bash
36
+ run: |
37
+ python -m pip install maturin pytest "sacremoses>=0.2.0"
38
+ maturin build --release -o dist
39
+ python -m pip install --no-index --find-links dist sacremoses-rs
40
+ - name: Unit tests
41
+ run: python -m pytest tests/test_basic.py -q
42
+ - name: Differential test against released sacremoses
43
+ shell: bash
44
+ run: |
45
+ python -m pip download "sacremoses==0.2.0" --no-deps --no-binary :all: -d upstream
46
+ mkdir upstream/src && tar xzf upstream/sacremoses-0.2.0.tar.gz -C upstream/src
47
+ curl -fsSL https://norvig.com/big.txt -o big.txt || cp README.md big.txt
48
+ python -I tests/diff_test.py upstream/src/sacremoses-0.2.0 big.txt 600 1500
@@ -0,0 +1,61 @@
1
+ # Builds wheels + sdist and publishes to PyPI when a tag like v0.1.0 is pushed.
2
+ # Nothing here runs until you push such a tag. One-time setup before the first
3
+ # release: on PyPI, add a "trusted publisher" for this repository (workflow:
4
+ # release.yml, environment: pypi). No API token needs to be stored.
5
+ name: Release
6
+
7
+ on:
8
+ push:
9
+ tags: ["v*"]
10
+
11
+ permissions:
12
+ contents: read
13
+
14
+ jobs:
15
+ wheels:
16
+ runs-on: ${{ matrix.os }}
17
+ strategy:
18
+ matrix:
19
+ include:
20
+ - { os: ubuntu-latest, target: x86_64 }
21
+ - { os: ubuntu-latest, target: aarch64 }
22
+ - { os: macos-latest, target: x86_64 }
23
+ - { os: macos-latest, target: aarch64 }
24
+ - { os: windows-latest, target: x64 }
25
+ steps:
26
+ - uses: actions/checkout@v4
27
+ - uses: PyO3/maturin-action@v1
28
+ with:
29
+ target: ${{ matrix.target }}
30
+ args: --release --out dist
31
+ manylinux: auto
32
+ - uses: actions/upload-artifact@v4
33
+ with:
34
+ name: wheels-${{ matrix.os }}-${{ matrix.target }}
35
+ path: dist
36
+
37
+ sdist:
38
+ runs-on: ubuntu-latest
39
+ steps:
40
+ - uses: actions/checkout@v4
41
+ - uses: PyO3/maturin-action@v1
42
+ with:
43
+ command: sdist
44
+ args: --out dist
45
+ - uses: actions/upload-artifact@v4
46
+ with:
47
+ name: sdist
48
+ path: dist
49
+
50
+ publish:
51
+ needs: [wheels, sdist]
52
+ runs-on: ubuntu-latest
53
+ environment: pypi
54
+ permissions:
55
+ id-token: write
56
+ steps:
57
+ - uses: actions/download-artifact@v4
58
+ with:
59
+ path: dist
60
+ merge-multiple: true
61
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,4 @@
1
+ target/
2
+ dist/
3
+ __pycache__/
4
+ *.so
@@ -0,0 +1,229 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 4
4
+
5
+ [[package]]
6
+ name = "aho-corasick"
7
+ version = "1.1.5"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
10
+ dependencies = [
11
+ "memchr",
12
+ ]
13
+
14
+ [[package]]
15
+ name = "crossbeam-deque"
16
+ version = "0.8.8"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
19
+ dependencies = [
20
+ "crossbeam-epoch",
21
+ "crossbeam-utils",
22
+ ]
23
+
24
+ [[package]]
25
+ name = "crossbeam-epoch"
26
+ version = "0.9.21"
27
+ source = "registry+https://github.com/rust-lang/crates.io-index"
28
+ checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
29
+ dependencies = [
30
+ "crossbeam-utils",
31
+ ]
32
+
33
+ [[package]]
34
+ name = "crossbeam-utils"
35
+ version = "0.8.23"
36
+ source = "registry+https://github.com/rust-lang/crates.io-index"
37
+ checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
38
+
39
+ [[package]]
40
+ name = "either"
41
+ version = "1.19.0"
42
+ source = "registry+https://github.com/rust-lang/crates.io-index"
43
+ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
44
+
45
+ [[package]]
46
+ name = "heck"
47
+ version = "0.5.0"
48
+ source = "registry+https://github.com/rust-lang/crates.io-index"
49
+ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
50
+
51
+ [[package]]
52
+ name = "libc"
53
+ version = "0.2.191"
54
+ source = "registry+https://github.com/rust-lang/crates.io-index"
55
+ checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
56
+
57
+ [[package]]
58
+ name = "memchr"
59
+ version = "2.8.3"
60
+ source = "registry+https://github.com/rust-lang/crates.io-index"
61
+ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
62
+
63
+ [[package]]
64
+ name = "once_cell"
65
+ version = "1.21.4"
66
+ source = "registry+https://github.com/rust-lang/crates.io-index"
67
+ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
68
+
69
+ [[package]]
70
+ name = "portable-atomic"
71
+ version = "1.15.0"
72
+ source = "registry+https://github.com/rust-lang/crates.io-index"
73
+ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
74
+
75
+ [[package]]
76
+ name = "proc-macro2"
77
+ version = "1.0.107"
78
+ source = "registry+https://github.com/rust-lang/crates.io-index"
79
+ checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
80
+ dependencies = [
81
+ "unicode-ident",
82
+ ]
83
+
84
+ [[package]]
85
+ name = "pyo3"
86
+ version = "0.30.1"
87
+ source = "registry+https://github.com/rust-lang/crates.io-index"
88
+ checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
89
+ dependencies = [
90
+ "libc",
91
+ "once_cell",
92
+ "portable-atomic",
93
+ "pyo3-build-config",
94
+ "pyo3-ffi",
95
+ "pyo3-macros",
96
+ ]
97
+
98
+ [[package]]
99
+ name = "pyo3-build-config"
100
+ version = "0.30.1"
101
+ source = "registry+https://github.com/rust-lang/crates.io-index"
102
+ checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
103
+ dependencies = [
104
+ "target-lexicon",
105
+ ]
106
+
107
+ [[package]]
108
+ name = "pyo3-ffi"
109
+ version = "0.30.1"
110
+ source = "registry+https://github.com/rust-lang/crates.io-index"
111
+ checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
112
+ dependencies = [
113
+ "libc",
114
+ "pyo3-build-config",
115
+ ]
116
+
117
+ [[package]]
118
+ name = "pyo3-macros"
119
+ version = "0.30.1"
120
+ source = "registry+https://github.com/rust-lang/crates.io-index"
121
+ checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
122
+ dependencies = [
123
+ "proc-macro2",
124
+ "pyo3-macros-backend",
125
+ "quote",
126
+ "syn",
127
+ ]
128
+
129
+ [[package]]
130
+ name = "pyo3-macros-backend"
131
+ version = "0.30.1"
132
+ source = "registry+https://github.com/rust-lang/crates.io-index"
133
+ checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
134
+ dependencies = [
135
+ "heck",
136
+ "proc-macro2",
137
+ "quote",
138
+ "syn",
139
+ ]
140
+
141
+ [[package]]
142
+ name = "quote"
143
+ version = "1.0.47"
144
+ source = "registry+https://github.com/rust-lang/crates.io-index"
145
+ checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
146
+ dependencies = [
147
+ "proc-macro2",
148
+ ]
149
+
150
+ [[package]]
151
+ name = "rayon"
152
+ version = "1.12.0"
153
+ source = "registry+https://github.com/rust-lang/crates.io-index"
154
+ checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
155
+ dependencies = [
156
+ "either",
157
+ "rayon-core",
158
+ ]
159
+
160
+ [[package]]
161
+ name = "rayon-core"
162
+ version = "1.13.0"
163
+ source = "registry+https://github.com/rust-lang/crates.io-index"
164
+ checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
165
+ dependencies = [
166
+ "crossbeam-deque",
167
+ "crossbeam-utils",
168
+ ]
169
+
170
+ [[package]]
171
+ name = "regex"
172
+ version = "1.13.1"
173
+ source = "registry+https://github.com/rust-lang/crates.io-index"
174
+ checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
175
+ dependencies = [
176
+ "aho-corasick",
177
+ "memchr",
178
+ "regex-automata",
179
+ "regex-syntax",
180
+ ]
181
+
182
+ [[package]]
183
+ name = "regex-automata"
184
+ version = "0.4.18"
185
+ source = "registry+https://github.com/rust-lang/crates.io-index"
186
+ checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
187
+ dependencies = [
188
+ "aho-corasick",
189
+ "memchr",
190
+ "regex-syntax",
191
+ ]
192
+
193
+ [[package]]
194
+ name = "regex-syntax"
195
+ version = "0.8.11"
196
+ source = "registry+https://github.com/rust-lang/crates.io-index"
197
+ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
198
+
199
+ [[package]]
200
+ name = "sacremoses-rs"
201
+ version = "0.1.0"
202
+ dependencies = [
203
+ "pyo3",
204
+ "rayon",
205
+ "regex",
206
+ ]
207
+
208
+ [[package]]
209
+ name = "syn"
210
+ version = "3.0.10"
211
+ source = "registry+https://github.com/rust-lang/crates.io-index"
212
+ checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"
213
+ dependencies = [
214
+ "proc-macro2",
215
+ "quote",
216
+ "unicode-ident",
217
+ ]
218
+
219
+ [[package]]
220
+ name = "target-lexicon"
221
+ version = "0.13.6"
222
+ source = "registry+https://github.com/rust-lang/crates.io-index"
223
+ checksum = "8c447be3a2abfa84b2d91680c56600bd6b2873866de27b36ec911be705e291da"
224
+
225
+ [[package]]
226
+ name = "unicode-ident"
227
+ version = "1.0.27"
228
+ source = "registry+https://github.com/rust-lang/crates.io-index"
229
+ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
@@ -0,0 +1,21 @@
1
+ [package]
2
+ name = "sacremoses-rs"
3
+ version = "0.1.0"
4
+ edition = "2021"
5
+ description = "Rust-accelerated drop-in classes for sacremoses (Moses tokenizer, detokenizer, punctuation normalizer)"
6
+ license = "MIT"
7
+ readme = "README.md"
8
+
9
+ [lib]
10
+ name = "_core"
11
+ crate-type = ["cdylib"]
12
+
13
+ [dependencies]
14
+ pyo3 = { version = "0.30.1", features = ["extension-module", "abi3-py39"] }
15
+ rayon = "1.12.0"
16
+ regex = "1.13.1"
17
+
18
+ [profile.release]
19
+ opt-level = 3
20
+ lto = "fat"
21
+ codegen-units = 1
@@ -0,0 +1,22 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 markocerkez
4
+ Copyright (c) 2020 alvations (sacremoses, from which these algorithms are ported)
5
+
6
+ Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ of this software and associated documentation files (the "Software"), to deal
8
+ in the Software without restriction, including without limitation the rights
9
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ copies of the Software, and to permit persons to whom the Software is
11
+ furnished to do so, subject to the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be included in all
14
+ copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
+ SOFTWARE.
@@ -0,0 +1,106 @@
1
+ Metadata-Version: 2.4
2
+ Name: sacremoses-rs
3
+ Version: 0.1.0
4
+ Classifier: License :: OSI Approved :: MIT License
5
+ Classifier: Programming Language :: Rust
6
+ Classifier: Programming Language :: Python :: 3
7
+ Requires-Dist: sacremoses>=0.2.0
8
+ Requires-Dist: pytest ; extra == 'test'
9
+ Provides-Extra: test
10
+ License-File: LICENSE
11
+ Summary: Rust-accelerated drop-in classes for sacremoses (Moses tokenizer, detokenizer, punctuation normalizer)
12
+ Keywords: moses,tokenizer,nlp,machine-translation,rust
13
+ License: MIT
14
+ Requires-Python: >=3.9
15
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
16
+ Project-URL: Repository, https://github.com/markocerkez/sacremoses-rs
17
+ Project-URL: Upstream, https://github.com/hplt-project/sacremoses
18
+
19
+ # sacremoses-rs
20
+
21
+ Rust-accelerated drop-in classes for [sacremoses](https://github.com/hplt-project/sacremoses)
22
+ (the Python port of the Moses tokenizer, detokenizer and punctuation normalizer).
23
+ Same API, same output, byte for byte, much faster.
24
+
25
+ ```python
26
+ from sacremoses_rs import MosesTokenizer, MosesDetokenizer, MosesPunctNormalizer
27
+
28
+ tok = MosesTokenizer("en")
29
+ tok.tokenize("Hello, world... it's 5,300 dollars.", return_str=True)
30
+
31
+ # batch API (extra): one call, all CPU cores, GIL released, order preserved
32
+ tok.tokenize_many(list_of_sentences)
33
+ MosesPunctNormalizer("en").normalize_many(list_of_sentences)
34
+ MosesDetokenizer("en").detokenize_many(list_of_token_lists)
35
+
36
+ # or speed up everything that already uses sacremoses (CLI, truecaser, ...)
37
+ import sacremoses_rs; sacremoses_rs.patch()
38
+ ```
39
+
40
+ The classes subclass the originals, so every option and attribute keeps working.
41
+ Requires `sacremoses>=0.2.0`.
42
+
43
+ ## Speed
44
+
45
+ 20,000 lines / 1.7 MB of English prose, Python 3.13, three runs, one core unless noted
46
+ (the batch rows use both cores of the 2-thread test machine; more cores scale further):
47
+
48
+ | step | speedup vs. python |
49
+ |------------------------------|--------------------|
50
+ | tokenize (`return_str=True`) | 28-30x |
51
+ | tokenize (token list) | 18-19x |
52
+ | tokenize (aggressive dashes) | 27-30x |
53
+ | tokenize (batch API) | 50-55x |
54
+ | punctuation normalize | 4-6x |
55
+ | punctuation normalize (batch)| 6-7x |
56
+ | detokenize | 4-5x |
57
+ | detokenize (batch) | 5-6x |
58
+
59
+ Reproduce: `python -I bench/bench.py <sacremoses checkout> <corpus.txt> 20000`
60
+
61
+ ## How it was verified
62
+
63
+ * `tests/diff_test.py`: differential test, original vs. Rust, outputs must be identical.
64
+ About 1.2 million cases per run: a real corpus plus adversarial fuzz (unicode whitespace,
65
+ control characters, exotic scripts, the library's own marker strings, ...) across 19
66
+ languages, all option combinations of the normalizer, the detokenizer, and the batch APIs.
67
+ Result: 0 mismatches against both sacremoses 0.2.0 (PyPI) and the current git HEAD.
68
+ * `tests/test_basic.py`: 54 self-contained pytest tests (golden outputs, fallbacks,
69
+ batch order, thread safety, `patch()`).
70
+ * sacremoses' own test suite with `patch()` applied gives the same results as without
71
+ it (the truecaser tests that need network downloads fail identically either way).
72
+ * `cargo fmt` and `cargo clippy -D warnings` are clean.
73
+
74
+ sacremoses 0.1.x is not supported: its handling of inputs that literally contain the
75
+ text `DOTMULTI` differs from 0.2.0 (a quirk upstream fixed).
76
+
77
+ ## What is NOT accelerated (falls back to the original Python, same results)
78
+
79
+ * `tokenize(..., protected_patterns=...)` and `penn_tokenize`
80
+ * strings containing lone surrogates, non-str input
81
+ * Czech detokenization of `x - li` / `e-mail` style tokens (the original raises
82
+ a `TypeError` there; the fallback keeps that behaviour)
83
+
84
+ ## Known limits
85
+
86
+ * Changing `NONBREAKING_PREFIXES` / `NUMERIC_ONLY_PREFIXES` of an existing tokenizer is
87
+ noticed when the list object or its length changes, not when an item is edited in place.
88
+ * The normalizer uses the Rust `regex` crate for `\d` and `\p{C}`; for code points newly
89
+ assigned in a recent Unicode version, Python and Rust may disagree.
90
+ * Everything above was measured on Linux x86-64 with Python 3.13. The GitHub Actions
91
+ workflows (`.github/workflows`) that test and build Linux, macOS and Windows wheels
92
+ are written but have not been run yet.
93
+
94
+ ## Build
95
+
96
+ ```
97
+ pip install maturin "sacremoses>=0.2.0"
98
+ maturin build --release -o dist && pip install dist/*.whl
99
+ pytest tests/test_basic.py
100
+ ```
101
+
102
+ ## License / attribution
103
+
104
+ MIT. The algorithms are ported from sacremoses (MIT, (c) 2020 alvations), which in turn
105
+ ports the Moses scripts. Keep both copyright lines.
106
+
@@ -0,0 +1,87 @@
1
+ # sacremoses-rs
2
+
3
+ Rust-accelerated drop-in classes for [sacremoses](https://github.com/hplt-project/sacremoses)
4
+ (the Python port of the Moses tokenizer, detokenizer and punctuation normalizer).
5
+ Same API, same output, byte for byte, much faster.
6
+
7
+ ```python
8
+ from sacremoses_rs import MosesTokenizer, MosesDetokenizer, MosesPunctNormalizer
9
+
10
+ tok = MosesTokenizer("en")
11
+ tok.tokenize("Hello, world... it's 5,300 dollars.", return_str=True)
12
+
13
+ # batch API (extra): one call, all CPU cores, GIL released, order preserved
14
+ tok.tokenize_many(list_of_sentences)
15
+ MosesPunctNormalizer("en").normalize_many(list_of_sentences)
16
+ MosesDetokenizer("en").detokenize_many(list_of_token_lists)
17
+
18
+ # or speed up everything that already uses sacremoses (CLI, truecaser, ...)
19
+ import sacremoses_rs; sacremoses_rs.patch()
20
+ ```
21
+
22
+ The classes subclass the originals, so every option and attribute keeps working.
23
+ Requires `sacremoses>=0.2.0`.
24
+
25
+ ## Speed
26
+
27
+ 20,000 lines / 1.7 MB of English prose, Python 3.13, three runs, one core unless noted
28
+ (the batch rows use both cores of the 2-thread test machine; more cores scale further):
29
+
30
+ | step | speedup vs. python |
31
+ |------------------------------|--------------------|
32
+ | tokenize (`return_str=True`) | 28-30x |
33
+ | tokenize (token list) | 18-19x |
34
+ | tokenize (aggressive dashes) | 27-30x |
35
+ | tokenize (batch API) | 50-55x |
36
+ | punctuation normalize | 4-6x |
37
+ | punctuation normalize (batch)| 6-7x |
38
+ | detokenize | 4-5x |
39
+ | detokenize (batch) | 5-6x |
40
+
41
+ Reproduce: `python -I bench/bench.py <sacremoses checkout> <corpus.txt> 20000`
42
+
43
+ ## How it was verified
44
+
45
+ * `tests/diff_test.py`: differential test, original vs. Rust, outputs must be identical.
46
+ About 1.2 million cases per run: a real corpus plus adversarial fuzz (unicode whitespace,
47
+ control characters, exotic scripts, the library's own marker strings, ...) across 19
48
+ languages, all option combinations of the normalizer, the detokenizer, and the batch APIs.
49
+ Result: 0 mismatches against both sacremoses 0.2.0 (PyPI) and the current git HEAD.
50
+ * `tests/test_basic.py`: 54 self-contained pytest tests (golden outputs, fallbacks,
51
+ batch order, thread safety, `patch()`).
52
+ * sacremoses' own test suite with `patch()` applied gives the same results as without
53
+ it (the truecaser tests that need network downloads fail identically either way).
54
+ * `cargo fmt` and `cargo clippy -D warnings` are clean.
55
+
56
+ sacremoses 0.1.x is not supported: its handling of inputs that literally contain the
57
+ text `DOTMULTI` differs from 0.2.0 (a quirk upstream fixed).
58
+
59
+ ## What is NOT accelerated (falls back to the original Python, same results)
60
+
61
+ * `tokenize(..., protected_patterns=...)` and `penn_tokenize`
62
+ * strings containing lone surrogates, non-str input
63
+ * Czech detokenization of `x - li` / `e-mail` style tokens (the original raises
64
+ a `TypeError` there; the fallback keeps that behaviour)
65
+
66
+ ## Known limits
67
+
68
+ * Changing `NONBREAKING_PREFIXES` / `NUMERIC_ONLY_PREFIXES` of an existing tokenizer is
69
+ noticed when the list object or its length changes, not when an item is edited in place.
70
+ * The normalizer uses the Rust `regex` crate for `\d` and `\p{C}`; for code points newly
71
+ assigned in a recent Unicode version, Python and Rust may disagree.
72
+ * Everything above was measured on Linux x86-64 with Python 3.13. The GitHub Actions
73
+ workflows (`.github/workflows`) that test and build Linux, macOS and Windows wheels
74
+ are written but have not been run yet.
75
+
76
+ ## Build
77
+
78
+ ```
79
+ pip install maturin "sacremoses>=0.2.0"
80
+ maturin build --release -o dist && pip install dist/*.whl
81
+ pytest tests/test_basic.py
82
+ ```
83
+
84
+ ## License / attribution
85
+
86
+ MIT. The algorithms are ported from sacremoses (MIT, (c) 2020 alvations), which in turn
87
+ ports the Moses scripts. Keep both copyright lines.
@@ -0,0 +1,47 @@
1
+ """Benchmark: original sacremoses vs sacremoses_rs.
2
+ Usage: python3 -I bench.py <path-to-original-sacremoses-checkout> <corpus.txt> [n_lines]
3
+ """
4
+ import sys, time
5
+ orig, corpus = sys.argv[1], sys.argv[2]
6
+ n = int(sys.argv[3]) if len(sys.argv) > 3 else 20000
7
+ sys.path.insert(0, orig)
8
+ import sacremoses_rs as rs
9
+ from sacremoses.normalize import MosesPunctNormalizer as PyN
10
+ from sacremoses.tokenize import MosesDetokenizer as PyD, MosesTokenizer as PyT
11
+
12
+ lines = [l.rstrip("\n") for l in open(corpus, encoding="utf-8", errors="replace") if l.strip()][:n]
13
+ mb = sum(len(l.encode()) for l in lines) / 1e6
14
+
15
+ def best(fn, reps=3):
16
+ r = []
17
+ for _ in range(reps):
18
+ t = time.perf_counter(); out = fn(); r.append(time.perf_counter() - t)
19
+ return min(r), out
20
+
21
+ pn, rn = PyN(), rs.MosesPunctNormalizer()
22
+ pt, rt = PyT("en"), rs.MosesTokenizer("en")
23
+ pd, rd = PyD("en"), rs.MosesDetokenizer("en")
24
+
25
+ rows = []
26
+ a, normed = best(lambda: [pn.normalize(l) for l in lines]); b, got = best(lambda: [rn.normalize(l) for l in lines]); assert normed == got
27
+ rows.append(("punct normalize", a, b))
28
+ b2, got = best(lambda: rn.normalize_many(lines)); assert normed == got
29
+ rows.append(("punct normalize (batch)", a, b2))
30
+ a, toks = best(lambda: [pt.tokenize(l, return_str=True) for l in normed]); b, got = best(lambda: [rt.tokenize(l, return_str=True) for l in normed]); assert toks == got
31
+ rows.append(("tokenize (return_str)", a, b))
32
+ a, tl = best(lambda: [pt.tokenize(l) for l in normed]); b, got = best(lambda: [rt.tokenize(l) for l in normed]); assert tl == got
33
+ rows.append(("tokenize (token list)", a, b))
34
+ b2, got = best(lambda: rt.tokenize_many(normed)); assert toks == got
35
+ rows.append(("tokenize (batch)", a, b2))
36
+ a, det = best(lambda: [pd.detokenize(x) for x in tl]); b, got = best(lambda: [rd.detokenize(x) for x in tl]); assert det == got
37
+ rows.append(("detokenize", a, b))
38
+ b2, got = best(lambda: rd.detokenize_many(tl)); assert det == got
39
+ rows.append(("detokenize (batch)", a, b2))
40
+ a, _ = best(lambda: [pt.tokenize(l, aggressive_dash_splits=True, return_str=True) for l in normed]); b, _ = best(lambda: [rt.tokenize(l, aggressive_dash_splits=True, return_str=True) for l in normed])
41
+ rows.append(("tokenize (aggressive dashes)", a, b))
42
+
43
+ import os
44
+ print(f"{len(lines):,} lines, {mb:.2f} MB, Python {sys.version.split()[0]}, {os.cpu_count()} CPU threads (batch rows use all of them)\n")
45
+ print(f"{'step':32s} {'python':>9s} {'rust':>9s} {'speedup':>9s} {'MB/s rust':>9s}")
46
+ for name, a, b in rows:
47
+ print(f"{name:32s} {a:8.3f}s {b:8.3f}s {a/b:8.1f}x {mb/b:9.1f}")
@@ -0,0 +1,30 @@
1
+ [build-system]
2
+ requires = ["maturin>=1.5,<2"]
3
+ build-backend = "maturin"
4
+
5
+ [project]
6
+ name = "sacremoses-rs"
7
+ version = "0.1.0"
8
+ description = "Rust-accelerated drop-in classes for sacremoses (Moses tokenizer, detokenizer, punctuation normalizer)"
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ requires-python = ">=3.9"
12
+ dependencies = ["sacremoses>=0.2.0"]
13
+ keywords = ["moses", "tokenizer", "nlp", "machine-translation", "rust"]
14
+ classifiers = [
15
+ "License :: OSI Approved :: MIT License",
16
+ "Programming Language :: Rust",
17
+ "Programming Language :: Python :: 3",
18
+ ]
19
+
20
+ [project.optional-dependencies]
21
+ test = ["pytest"]
22
+
23
+ [project.urls]
24
+ Repository = "https://github.com/markocerkez/sacremoses-rs"
25
+ Upstream = "https://github.com/hplt-project/sacremoses"
26
+
27
+ [tool.maturin]
28
+ python-source = "python"
29
+ module-name = "sacremoses_rs._core"
30
+ features = ["pyo3/extension-module"]