sacremoses-rs 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sacremoses_rs-0.1.0/.github/workflows/ci.yml +48 -0
- sacremoses_rs-0.1.0/.github/workflows/release.yml +61 -0
- sacremoses_rs-0.1.0/.gitignore +4 -0
- sacremoses_rs-0.1.0/Cargo.lock +229 -0
- sacremoses_rs-0.1.0/Cargo.toml +21 -0
- sacremoses_rs-0.1.0/LICENSE +22 -0
- sacremoses_rs-0.1.0/PKG-INFO +106 -0
- sacremoses_rs-0.1.0/README.md +87 -0
- sacremoses_rs-0.1.0/bench/bench.py +47 -0
- sacremoses_rs-0.1.0/pyproject.toml +30 -0
- sacremoses_rs-0.1.0/python/sacremoses_rs/__init__.py +208 -0
- sacremoses_rs-0.1.0/python/sacremoses_rs/_core.pyi +20 -0
- sacremoses_rs-0.1.0/python/sacremoses_rs/py.typed +0 -0
- sacremoses_rs-0.1.0/src/lib.rs +978 -0
- sacremoses_rs-0.1.0/tests/diff_test.py +142 -0
- sacremoses_rs-0.1.0/tests/test_basic.py +115 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
lint:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v4
|
|
13
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
14
|
+
with:
|
|
15
|
+
components: rustfmt, clippy
|
|
16
|
+
- run: cargo fmt --check
|
|
17
|
+
- run: cargo clippy --release -- -D warnings
|
|
18
|
+
|
|
19
|
+
test:
|
|
20
|
+
runs-on: ${{ matrix.os }}
|
|
21
|
+
strategy:
|
|
22
|
+
fail-fast: false
|
|
23
|
+
matrix:
|
|
24
|
+
os: [ubuntu-24.04, macos-latest, windows-latest]
|
|
25
|
+
python: ["3.9", "3.13"]
|
|
26
|
+
env:
|
|
27
|
+
PYTHONUTF8: "1"
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@v4
|
|
30
|
+
- uses: actions/setup-python@v5
|
|
31
|
+
with:
|
|
32
|
+
python-version: ${{ matrix.python }}
|
|
33
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
34
|
+
- name: Build wheel and install
|
|
35
|
+
shell: bash
|
|
36
|
+
run: |
|
|
37
|
+
python -m pip install maturin pytest "sacremoses>=0.2.0"
|
|
38
|
+
maturin build --release -o dist
|
|
39
|
+
python -m pip install --no-index --find-links dist sacremoses-rs
|
|
40
|
+
- name: Unit tests
|
|
41
|
+
run: python -m pytest tests/test_basic.py -q
|
|
42
|
+
- name: Differential test against released sacremoses
|
|
43
|
+
shell: bash
|
|
44
|
+
run: |
|
|
45
|
+
python -m pip download "sacremoses==0.2.0" --no-deps --no-binary :all: -d upstream
|
|
46
|
+
mkdir upstream/src && tar xzf upstream/sacremoses-0.2.0.tar.gz -C upstream/src
|
|
47
|
+
curl -fsSL https://norvig.com/big.txt -o big.txt || cp README.md big.txt
|
|
48
|
+
python -I tests/diff_test.py upstream/src/sacremoses-0.2.0 big.txt 600 1500
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Builds wheels + sdist and publishes to PyPI when a tag like v0.1.0 is pushed.
|
|
2
|
+
# Nothing here runs until you push such a tag. One-time setup before the first
|
|
3
|
+
# release: on PyPI, add a "trusted publisher" for this repository (workflow:
|
|
4
|
+
# release.yml, environment: pypi). No API token needs to be stored.
|
|
5
|
+
name: Release
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
push:
|
|
9
|
+
tags: ["v*"]
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
wheels:
|
|
16
|
+
runs-on: ${{ matrix.os }}
|
|
17
|
+
strategy:
|
|
18
|
+
matrix:
|
|
19
|
+
include:
|
|
20
|
+
- { os: ubuntu-latest, target: x86_64 }
|
|
21
|
+
- { os: ubuntu-latest, target: aarch64 }
|
|
22
|
+
- { os: macos-latest, target: x86_64 }
|
|
23
|
+
- { os: macos-latest, target: aarch64 }
|
|
24
|
+
- { os: windows-latest, target: x64 }
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
- uses: PyO3/maturin-action@v1
|
|
28
|
+
with:
|
|
29
|
+
target: ${{ matrix.target }}
|
|
30
|
+
args: --release --out dist
|
|
31
|
+
manylinux: auto
|
|
32
|
+
- uses: actions/upload-artifact@v4
|
|
33
|
+
with:
|
|
34
|
+
name: wheels-${{ matrix.os }}-${{ matrix.target }}
|
|
35
|
+
path: dist
|
|
36
|
+
|
|
37
|
+
sdist:
|
|
38
|
+
runs-on: ubuntu-latest
|
|
39
|
+
steps:
|
|
40
|
+
- uses: actions/checkout@v4
|
|
41
|
+
- uses: PyO3/maturin-action@v1
|
|
42
|
+
with:
|
|
43
|
+
command: sdist
|
|
44
|
+
args: --out dist
|
|
45
|
+
- uses: actions/upload-artifact@v4
|
|
46
|
+
with:
|
|
47
|
+
name: sdist
|
|
48
|
+
path: dist
|
|
49
|
+
|
|
50
|
+
publish:
|
|
51
|
+
needs: [wheels, sdist]
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
environment: pypi
|
|
54
|
+
permissions:
|
|
55
|
+
id-token: write
|
|
56
|
+
steps:
|
|
57
|
+
- uses: actions/download-artifact@v4
|
|
58
|
+
with:
|
|
59
|
+
path: dist
|
|
60
|
+
merge-multiple: true
|
|
61
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "aho-corasick"
|
|
7
|
+
version = "1.1.5"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"memchr",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[[package]]
|
|
15
|
+
name = "crossbeam-deque"
|
|
16
|
+
version = "0.8.8"
|
|
17
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
+
checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
|
|
19
|
+
dependencies = [
|
|
20
|
+
"crossbeam-epoch",
|
|
21
|
+
"crossbeam-utils",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[[package]]
|
|
25
|
+
name = "crossbeam-epoch"
|
|
26
|
+
version = "0.9.21"
|
|
27
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
28
|
+
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
|
|
29
|
+
dependencies = [
|
|
30
|
+
"crossbeam-utils",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[[package]]
|
|
34
|
+
name = "crossbeam-utils"
|
|
35
|
+
version = "0.8.23"
|
|
36
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
37
|
+
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
|
38
|
+
|
|
39
|
+
[[package]]
|
|
40
|
+
name = "either"
|
|
41
|
+
version = "1.19.0"
|
|
42
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
43
|
+
checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
|
|
44
|
+
|
|
45
|
+
[[package]]
|
|
46
|
+
name = "heck"
|
|
47
|
+
version = "0.5.0"
|
|
48
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
49
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
50
|
+
|
|
51
|
+
[[package]]
|
|
52
|
+
name = "libc"
|
|
53
|
+
version = "0.2.191"
|
|
54
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
55
|
+
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
|
56
|
+
|
|
57
|
+
[[package]]
|
|
58
|
+
name = "memchr"
|
|
59
|
+
version = "2.8.3"
|
|
60
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
61
|
+
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
62
|
+
|
|
63
|
+
[[package]]
|
|
64
|
+
name = "once_cell"
|
|
65
|
+
version = "1.21.4"
|
|
66
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
67
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
68
|
+
|
|
69
|
+
[[package]]
|
|
70
|
+
name = "portable-atomic"
|
|
71
|
+
version = "1.15.0"
|
|
72
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
73
|
+
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
74
|
+
|
|
75
|
+
[[package]]
|
|
76
|
+
name = "proc-macro2"
|
|
77
|
+
version = "1.0.107"
|
|
78
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
79
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
80
|
+
dependencies = [
|
|
81
|
+
"unicode-ident",
|
|
82
|
+
]
|
|
83
|
+
|
|
84
|
+
[[package]]
|
|
85
|
+
name = "pyo3"
|
|
86
|
+
version = "0.30.1"
|
|
87
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
88
|
+
checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
|
|
89
|
+
dependencies = [
|
|
90
|
+
"libc",
|
|
91
|
+
"once_cell",
|
|
92
|
+
"portable-atomic",
|
|
93
|
+
"pyo3-build-config",
|
|
94
|
+
"pyo3-ffi",
|
|
95
|
+
"pyo3-macros",
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
[[package]]
|
|
99
|
+
name = "pyo3-build-config"
|
|
100
|
+
version = "0.30.1"
|
|
101
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
102
|
+
checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
|
|
103
|
+
dependencies = [
|
|
104
|
+
"target-lexicon",
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
[[package]]
|
|
108
|
+
name = "pyo3-ffi"
|
|
109
|
+
version = "0.30.1"
|
|
110
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
111
|
+
checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
|
|
112
|
+
dependencies = [
|
|
113
|
+
"libc",
|
|
114
|
+
"pyo3-build-config",
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
[[package]]
|
|
118
|
+
name = "pyo3-macros"
|
|
119
|
+
version = "0.30.1"
|
|
120
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
121
|
+
checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
|
|
122
|
+
dependencies = [
|
|
123
|
+
"proc-macro2",
|
|
124
|
+
"pyo3-macros-backend",
|
|
125
|
+
"quote",
|
|
126
|
+
"syn",
|
|
127
|
+
]
|
|
128
|
+
|
|
129
|
+
[[package]]
|
|
130
|
+
name = "pyo3-macros-backend"
|
|
131
|
+
version = "0.30.1"
|
|
132
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
133
|
+
checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
|
|
134
|
+
dependencies = [
|
|
135
|
+
"heck",
|
|
136
|
+
"proc-macro2",
|
|
137
|
+
"quote",
|
|
138
|
+
"syn",
|
|
139
|
+
]
|
|
140
|
+
|
|
141
|
+
[[package]]
|
|
142
|
+
name = "quote"
|
|
143
|
+
version = "1.0.47"
|
|
144
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
145
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
146
|
+
dependencies = [
|
|
147
|
+
"proc-macro2",
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
[[package]]
|
|
151
|
+
name = "rayon"
|
|
152
|
+
version = "1.12.0"
|
|
153
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
154
|
+
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
|
155
|
+
dependencies = [
|
|
156
|
+
"either",
|
|
157
|
+
"rayon-core",
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
[[package]]
|
|
161
|
+
name = "rayon-core"
|
|
162
|
+
version = "1.13.0"
|
|
163
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
164
|
+
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
|
165
|
+
dependencies = [
|
|
166
|
+
"crossbeam-deque",
|
|
167
|
+
"crossbeam-utils",
|
|
168
|
+
]
|
|
169
|
+
|
|
170
|
+
[[package]]
|
|
171
|
+
name = "regex"
|
|
172
|
+
version = "1.13.1"
|
|
173
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
174
|
+
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
|
175
|
+
dependencies = [
|
|
176
|
+
"aho-corasick",
|
|
177
|
+
"memchr",
|
|
178
|
+
"regex-automata",
|
|
179
|
+
"regex-syntax",
|
|
180
|
+
]
|
|
181
|
+
|
|
182
|
+
[[package]]
|
|
183
|
+
name = "regex-automata"
|
|
184
|
+
version = "0.4.18"
|
|
185
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
186
|
+
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
|
187
|
+
dependencies = [
|
|
188
|
+
"aho-corasick",
|
|
189
|
+
"memchr",
|
|
190
|
+
"regex-syntax",
|
|
191
|
+
]
|
|
192
|
+
|
|
193
|
+
[[package]]
|
|
194
|
+
name = "regex-syntax"
|
|
195
|
+
version = "0.8.11"
|
|
196
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
197
|
+
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
|
198
|
+
|
|
199
|
+
[[package]]
|
|
200
|
+
name = "sacremoses-rs"
|
|
201
|
+
version = "0.1.0"
|
|
202
|
+
dependencies = [
|
|
203
|
+
"pyo3",
|
|
204
|
+
"rayon",
|
|
205
|
+
"regex",
|
|
206
|
+
]
|
|
207
|
+
|
|
208
|
+
[[package]]
|
|
209
|
+
name = "syn"
|
|
210
|
+
version = "3.0.10"
|
|
211
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
212
|
+
checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"
|
|
213
|
+
dependencies = [
|
|
214
|
+
"proc-macro2",
|
|
215
|
+
"quote",
|
|
216
|
+
"unicode-ident",
|
|
217
|
+
]
|
|
218
|
+
|
|
219
|
+
[[package]]
|
|
220
|
+
name = "target-lexicon"
|
|
221
|
+
version = "0.13.6"
|
|
222
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
223
|
+
checksum = "8c447be3a2abfa84b2d91680c56600bd6b2873866de27b36ec911be705e291da"
|
|
224
|
+
|
|
225
|
+
[[package]]
|
|
226
|
+
name = "unicode-ident"
|
|
227
|
+
version = "1.0.27"
|
|
228
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
229
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "sacremoses-rs"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
description = "Rust-accelerated drop-in classes for sacremoses (Moses tokenizer, detokenizer, punctuation normalizer)"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
readme = "README.md"
|
|
8
|
+
|
|
9
|
+
[lib]
|
|
10
|
+
name = "_core"
|
|
11
|
+
crate-type = ["cdylib"]
|
|
12
|
+
|
|
13
|
+
[dependencies]
|
|
14
|
+
pyo3 = { version = "0.30.1", features = ["extension-module", "abi3-py39"] }
|
|
15
|
+
rayon = "1.12.0"
|
|
16
|
+
regex = "1.13.1"
|
|
17
|
+
|
|
18
|
+
[profile.release]
|
|
19
|
+
opt-level = 3
|
|
20
|
+
lto = "fat"
|
|
21
|
+
codegen-units = 1
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 markocerkez
|
|
4
|
+
Copyright (c) 2020 alvations (sacremoses, from which these algorithms are ported)
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sacremoses-rs
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
5
|
+
Classifier: Programming Language :: Rust
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Requires-Dist: sacremoses>=0.2.0
|
|
8
|
+
Requires-Dist: pytest ; extra == 'test'
|
|
9
|
+
Provides-Extra: test
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Summary: Rust-accelerated drop-in classes for sacremoses (Moses tokenizer, detokenizer, punctuation normalizer)
|
|
12
|
+
Keywords: moses,tokenizer,nlp,machine-translation,rust
|
|
13
|
+
License: MIT
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
16
|
+
Project-URL: Repository, https://github.com/markocerkez/sacremoses-rs
|
|
17
|
+
Project-URL: Upstream, https://github.com/hplt-project/sacremoses
|
|
18
|
+
|
|
19
|
+
# sacremoses-rs
|
|
20
|
+
|
|
21
|
+
Rust-accelerated drop-in classes for [sacremoses](https://github.com/hplt-project/sacremoses)
|
|
22
|
+
(the Python port of the Moses tokenizer, detokenizer and punctuation normalizer).
|
|
23
|
+
Same API, same output, byte for byte, much faster.
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
from sacremoses_rs import MosesTokenizer, MosesDetokenizer, MosesPunctNormalizer
|
|
27
|
+
|
|
28
|
+
tok = MosesTokenizer("en")
|
|
29
|
+
tok.tokenize("Hello, world... it's 5,300 dollars.", return_str=True)
|
|
30
|
+
|
|
31
|
+
# batch API (extra): one call, all CPU cores, GIL released, order preserved
|
|
32
|
+
tok.tokenize_many(list_of_sentences)
|
|
33
|
+
MosesPunctNormalizer("en").normalize_many(list_of_sentences)
|
|
34
|
+
MosesDetokenizer("en").detokenize_many(list_of_token_lists)
|
|
35
|
+
|
|
36
|
+
# or speed up everything that already uses sacremoses (CLI, truecaser, ...)
|
|
37
|
+
import sacremoses_rs; sacremoses_rs.patch()
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The classes subclass the originals, so every option and attribute keeps working.
|
|
41
|
+
Requires `sacremoses>=0.2.0`.
|
|
42
|
+
|
|
43
|
+
## Speed
|
|
44
|
+
|
|
45
|
+
20,000 lines / 1.7 MB of English prose, Python 3.13, three runs, one core unless noted
|
|
46
|
+
(the batch rows use both cores of the 2-thread test machine; more cores scale further):
|
|
47
|
+
|
|
48
|
+
| step | speedup vs. python |
|
|
49
|
+
|------------------------------|--------------------|
|
|
50
|
+
| tokenize (`return_str=True`) | 28-30x |
|
|
51
|
+
| tokenize (token list) | 18-19x |
|
|
52
|
+
| tokenize (aggressive dashes) | 27-30x |
|
|
53
|
+
| tokenize (batch API) | 50-55x |
|
|
54
|
+
| punctuation normalize | 4-6x |
|
|
55
|
+
| punctuation normalize (batch)| 6-7x |
|
|
56
|
+
| detokenize | 4-5x |
|
|
57
|
+
| detokenize (batch) | 5-6x |
|
|
58
|
+
|
|
59
|
+
Reproduce: `python -I bench/bench.py <sacremoses checkout> <corpus.txt> 20000`
|
|
60
|
+
|
|
61
|
+
## How it was verified
|
|
62
|
+
|
|
63
|
+
* `tests/diff_test.py`: differential test, original vs. Rust, outputs must be identical.
|
|
64
|
+
About 1.2 million cases per run: a real corpus plus adversarial fuzz (unicode whitespace,
|
|
65
|
+
control characters, exotic scripts, the library's own marker strings, ...) across 19
|
|
66
|
+
languages, all option combinations of the normalizer, the detokenizer, and the batch APIs.
|
|
67
|
+
Result: 0 mismatches against both sacremoses 0.2.0 (PyPI) and the current git HEAD.
|
|
68
|
+
* `tests/test_basic.py`: 54 self-contained pytest tests (golden outputs, fallbacks,
|
|
69
|
+
batch order, thread safety, `patch()`).
|
|
70
|
+
* sacremoses' own test suite with `patch()` applied gives the same results as without
|
|
71
|
+
it (the truecaser tests that need network downloads fail identically either way).
|
|
72
|
+
* `cargo fmt` and `cargo clippy -D warnings` are clean.
|
|
73
|
+
|
|
74
|
+
sacremoses 0.1.x is not supported: its handling of inputs that literally contain the
|
|
75
|
+
text `DOTMULTI` differs from 0.2.0 (a quirk upstream fixed).
|
|
76
|
+
|
|
77
|
+
## What is NOT accelerated (falls back to the original Python, same results)
|
|
78
|
+
|
|
79
|
+
* `tokenize(..., protected_patterns=...)` and `penn_tokenize`
|
|
80
|
+
* strings containing lone surrogates, non-str input
|
|
81
|
+
* Czech detokenization of `x - li` / `e-mail` style tokens (the original raises
|
|
82
|
+
a `TypeError` there; the fallback keeps that behaviour)
|
|
83
|
+
|
|
84
|
+
## Known limits
|
|
85
|
+
|
|
86
|
+
* Changing `NONBREAKING_PREFIXES` / `NUMERIC_ONLY_PREFIXES` of an existing tokenizer is
|
|
87
|
+
noticed when the list object or its length changes, not when an item is edited in place.
|
|
88
|
+
* The normalizer uses the Rust `regex` crate for `\d` and `\p{C}`; for code points newly
|
|
89
|
+
assigned in a recent Unicode version, Python and Rust may disagree.
|
|
90
|
+
* Everything above was measured on Linux x86-64 with Python 3.13. The GitHub Actions
|
|
91
|
+
workflows (`.github/workflows`) that test and build Linux, macOS and Windows wheels
|
|
92
|
+
are written but have not been run yet.
|
|
93
|
+
|
|
94
|
+
## Build
|
|
95
|
+
|
|
96
|
+
```
|
|
97
|
+
pip install maturin "sacremoses>=0.2.0"
|
|
98
|
+
maturin build --release -o dist && pip install dist/*.whl
|
|
99
|
+
pytest tests/test_basic.py
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## License / attribution
|
|
103
|
+
|
|
104
|
+
MIT. The algorithms are ported from sacremoses (MIT, (c) 2020 alvations), which in turn
|
|
105
|
+
ports the Moses scripts. Keep both copyright lines.
|
|
106
|
+
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# sacremoses-rs
|
|
2
|
+
|
|
3
|
+
Rust-accelerated drop-in classes for [sacremoses](https://github.com/hplt-project/sacremoses)
|
|
4
|
+
(the Python port of the Moses tokenizer, detokenizer and punctuation normalizer).
|
|
5
|
+
Same API, same output, byte for byte, much faster.
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from sacremoses_rs import MosesTokenizer, MosesDetokenizer, MosesPunctNormalizer
|
|
9
|
+
|
|
10
|
+
tok = MosesTokenizer("en")
|
|
11
|
+
tok.tokenize("Hello, world... it's 5,300 dollars.", return_str=True)
|
|
12
|
+
|
|
13
|
+
# batch API (extra): one call, all CPU cores, GIL released, order preserved
|
|
14
|
+
tok.tokenize_many(list_of_sentences)
|
|
15
|
+
MosesPunctNormalizer("en").normalize_many(list_of_sentences)
|
|
16
|
+
MosesDetokenizer("en").detokenize_many(list_of_token_lists)
|
|
17
|
+
|
|
18
|
+
# or speed up everything that already uses sacremoses (CLI, truecaser, ...)
|
|
19
|
+
import sacremoses_rs; sacremoses_rs.patch()
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The classes subclass the originals, so every option and attribute keeps working.
|
|
23
|
+
Requires `sacremoses>=0.2.0`.
|
|
24
|
+
|
|
25
|
+
## Speed
|
|
26
|
+
|
|
27
|
+
20,000 lines / 1.7 MB of English prose, Python 3.13, three runs, one core unless noted
|
|
28
|
+
(the batch rows use both cores of the 2-thread test machine; more cores scale further):
|
|
29
|
+
|
|
30
|
+
| step | speedup vs. python |
|
|
31
|
+
|------------------------------|--------------------|
|
|
32
|
+
| tokenize (`return_str=True`) | 28-30x |
|
|
33
|
+
| tokenize (token list) | 18-19x |
|
|
34
|
+
| tokenize (aggressive dashes) | 27-30x |
|
|
35
|
+
| tokenize (batch API) | 50-55x |
|
|
36
|
+
| punctuation normalize | 4-6x |
|
|
37
|
+
| punctuation normalize (batch)| 6-7x |
|
|
38
|
+
| detokenize | 4-5x |
|
|
39
|
+
| detokenize (batch) | 5-6x |
|
|
40
|
+
|
|
41
|
+
Reproduce: `python -I bench/bench.py <sacremoses checkout> <corpus.txt> 20000`
|
|
42
|
+
|
|
43
|
+
## How it was verified
|
|
44
|
+
|
|
45
|
+
* `tests/diff_test.py`: differential test, original vs. Rust, outputs must be identical.
|
|
46
|
+
About 1.2 million cases per run: a real corpus plus adversarial fuzz (unicode whitespace,
|
|
47
|
+
control characters, exotic scripts, the library's own marker strings, ...) across 19
|
|
48
|
+
languages, all option combinations of the normalizer, the detokenizer, and the batch APIs.
|
|
49
|
+
Result: 0 mismatches against both sacremoses 0.2.0 (PyPI) and the current git HEAD.
|
|
50
|
+
* `tests/test_basic.py`: 54 self-contained pytest tests (golden outputs, fallbacks,
|
|
51
|
+
batch order, thread safety, `patch()`).
|
|
52
|
+
* sacremoses' own test suite with `patch()` applied gives the same results as without
|
|
53
|
+
it (the truecaser tests that need network downloads fail identically either way).
|
|
54
|
+
* `cargo fmt` and `cargo clippy -D warnings` are clean.
|
|
55
|
+
|
|
56
|
+
sacremoses 0.1.x is not supported: its handling of inputs that literally contain the
|
|
57
|
+
text `DOTMULTI` differs from 0.2.0 (a quirk upstream fixed).
|
|
58
|
+
|
|
59
|
+
## What is NOT accelerated (falls back to the original Python, same results)
|
|
60
|
+
|
|
61
|
+
* `tokenize(..., protected_patterns=...)` and `penn_tokenize`
|
|
62
|
+
* strings containing lone surrogates, non-str input
|
|
63
|
+
* Czech detokenization of `x - li` / `e-mail` style tokens (the original raises
|
|
64
|
+
a `TypeError` there; the fallback keeps that behaviour)
|
|
65
|
+
|
|
66
|
+
## Known limits
|
|
67
|
+
|
|
68
|
+
* Changing `NONBREAKING_PREFIXES` / `NUMERIC_ONLY_PREFIXES` of an existing tokenizer is
|
|
69
|
+
noticed when the list object or its length changes, not when an item is edited in place.
|
|
70
|
+
* The normalizer uses the Rust `regex` crate for `\d` and `\p{C}`; for code points newly
|
|
71
|
+
assigned in a recent Unicode version, Python and Rust may disagree.
|
|
72
|
+
* Everything above was measured on Linux x86-64 with Python 3.13. The GitHub Actions
|
|
73
|
+
workflows (`.github/workflows`) that test and build Linux, macOS and Windows wheels
|
|
74
|
+
are written but have not been run yet.
|
|
75
|
+
|
|
76
|
+
## Build
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
pip install maturin "sacremoses>=0.2.0"
|
|
80
|
+
maturin build --release -o dist && pip install dist/*.whl
|
|
81
|
+
pytest tests/test_basic.py
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## License / attribution
|
|
85
|
+
|
|
86
|
+
MIT. The algorithms are ported from sacremoses (MIT, (c) 2020 alvations), which in turn
|
|
87
|
+
ports the Moses scripts. Keep both copyright lines.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Benchmark: original sacremoses vs sacremoses_rs.
|
|
2
|
+
Usage: python3 -I bench.py <path-to-original-sacremoses-checkout> <corpus.txt> [n_lines]
|
|
3
|
+
"""
|
|
4
|
+
import sys, time
|
|
5
|
+
orig, corpus = sys.argv[1], sys.argv[2]
|
|
6
|
+
n = int(sys.argv[3]) if len(sys.argv) > 3 else 20000
|
|
7
|
+
sys.path.insert(0, orig)
|
|
8
|
+
import sacremoses_rs as rs
|
|
9
|
+
from sacremoses.normalize import MosesPunctNormalizer as PyN
|
|
10
|
+
from sacremoses.tokenize import MosesDetokenizer as PyD, MosesTokenizer as PyT
|
|
11
|
+
|
|
12
|
+
lines = [l.rstrip("\n") for l in open(corpus, encoding="utf-8", errors="replace") if l.strip()][:n]
|
|
13
|
+
mb = sum(len(l.encode()) for l in lines) / 1e6
|
|
14
|
+
|
|
15
|
+
def best(fn, reps=3):
|
|
16
|
+
r = []
|
|
17
|
+
for _ in range(reps):
|
|
18
|
+
t = time.perf_counter(); out = fn(); r.append(time.perf_counter() - t)
|
|
19
|
+
return min(r), out
|
|
20
|
+
|
|
21
|
+
pn, rn = PyN(), rs.MosesPunctNormalizer()
|
|
22
|
+
pt, rt = PyT("en"), rs.MosesTokenizer("en")
|
|
23
|
+
pd, rd = PyD("en"), rs.MosesDetokenizer("en")
|
|
24
|
+
|
|
25
|
+
rows = []
|
|
26
|
+
a, normed = best(lambda: [pn.normalize(l) for l in lines]); b, got = best(lambda: [rn.normalize(l) for l in lines]); assert normed == got
|
|
27
|
+
rows.append(("punct normalize", a, b))
|
|
28
|
+
b2, got = best(lambda: rn.normalize_many(lines)); assert normed == got
|
|
29
|
+
rows.append(("punct normalize (batch)", a, b2))
|
|
30
|
+
a, toks = best(lambda: [pt.tokenize(l, return_str=True) for l in normed]); b, got = best(lambda: [rt.tokenize(l, return_str=True) for l in normed]); assert toks == got
|
|
31
|
+
rows.append(("tokenize (return_str)", a, b))
|
|
32
|
+
a, tl = best(lambda: [pt.tokenize(l) for l in normed]); b, got = best(lambda: [rt.tokenize(l) for l in normed]); assert tl == got
|
|
33
|
+
rows.append(("tokenize (token list)", a, b))
|
|
34
|
+
b2, got = best(lambda: rt.tokenize_many(normed)); assert toks == got
|
|
35
|
+
rows.append(("tokenize (batch)", a, b2))
|
|
36
|
+
a, det = best(lambda: [pd.detokenize(x) for x in tl]); b, got = best(lambda: [rd.detokenize(x) for x in tl]); assert det == got
|
|
37
|
+
rows.append(("detokenize", a, b))
|
|
38
|
+
b2, got = best(lambda: rd.detokenize_many(tl)); assert det == got
|
|
39
|
+
rows.append(("detokenize (batch)", a, b2))
|
|
40
|
+
a, _ = best(lambda: [pt.tokenize(l, aggressive_dash_splits=True, return_str=True) for l in normed]); b, _ = best(lambda: [rt.tokenize(l, aggressive_dash_splits=True, return_str=True) for l in normed])
|
|
41
|
+
rows.append(("tokenize (aggressive dashes)", a, b))
|
|
42
|
+
|
|
43
|
+
import os
|
|
44
|
+
print(f"{len(lines):,} lines, {mb:.2f} MB, Python {sys.version.split()[0]}, {os.cpu_count()} CPU threads (batch rows use all of them)\n")
|
|
45
|
+
print(f"{'step':32s} {'python':>9s} {'rust':>9s} {'speedup':>9s} {'MB/s rust':>9s}")
|
|
46
|
+
for name, a, b in rows:
|
|
47
|
+
print(f"{name:32s} {a:8.3f}s {b:8.3f}s {a/b:8.1f}x {mb/b:9.1f}")
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["maturin>=1.5,<2"]
|
|
3
|
+
build-backend = "maturin"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "sacremoses-rs"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Rust-accelerated drop-in classes for sacremoses (Moses tokenizer, detokenizer, punctuation normalizer)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
dependencies = ["sacremoses>=0.2.0"]
|
|
13
|
+
keywords = ["moses", "tokenizer", "nlp", "machine-translation", "rust"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"License :: OSI Approved :: MIT License",
|
|
16
|
+
"Programming Language :: Rust",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.optional-dependencies]
|
|
21
|
+
test = ["pytest"]
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Repository = "https://github.com/markocerkez/sacremoses-rs"
|
|
25
|
+
Upstream = "https://github.com/hplt-project/sacremoses"
|
|
26
|
+
|
|
27
|
+
[tool.maturin]
|
|
28
|
+
python-source = "python"
|
|
29
|
+
module-name = "sacremoses_rs._core"
|
|
30
|
+
features = ["pyo3/extension-module"]
|