unicode-segmentation-rs 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/actions/pre-commit-setup/action.yml +5 -4
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/release.yml +4 -5
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/workflows/ci.yml +54 -17
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/workflows/dependency-review.yml +1 -1
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/workflows/pre-commit.yml +1 -1
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.gitignore +0 -1
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.pre-commit-config.yaml +6 -4
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/Cargo.lock +19 -19
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/Cargo.toml +5 -2
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/PKG-INFO +8 -1
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/REUSE.toml +1 -1
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/pyproject.toml +9 -2
- unicode_segmentation_rs-0.3.2/src/lib.rs +303 -0
- unicode_segmentation_rs-0.3.0/src/lib.rs +0 -272
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.editorconfig +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/FUNDING.yml +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/PULL_REQUEST_TEMPLATE.md.license +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/renovate.json +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/renovate.json.license +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/workflows/pull_requests.yaml +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.yamllint.yml +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/LICENSE +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/LICENSES/Apache-2.0.txt +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/LICENSES/CC0-1.0.txt +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/README.md +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/SECURITY.md +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/tests/test_gettext_wrap.py +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/tests/test_readme_examples.py +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/tests/test_unicode_segmentation.py +0 -0
- {unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/unicode_segmentation_rs.pyi +0 -0
|
@@ -17,9 +17,10 @@ runs:
|
|
|
17
17
|
restore-keys: |
|
|
18
18
|
${{ runner.os }}-${{ runner.arch }}-prek-
|
|
19
19
|
- name: Setup Python
|
|
20
|
-
uses: actions/setup-python@
|
|
20
|
+
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
21
21
|
with:
|
|
22
|
-
python-version: '3.
|
|
23
|
-
|
|
22
|
+
python-version: '3.15'
|
|
23
|
+
allow-prereleases: true
|
|
24
|
+
- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
|
24
25
|
with:
|
|
25
|
-
version: 0.
|
|
26
|
+
version: 0.12.1
|
|
@@ -5,6 +5,10 @@
|
|
|
5
5
|
# This file is maintained in https://github.com/WeblateOrg/meta/
|
|
6
6
|
|
|
7
7
|
changelog:
|
|
8
|
+
exclude:
|
|
9
|
+
authors:
|
|
10
|
+
- weblate
|
|
11
|
+
- hosted-weblate[bot]
|
|
8
12
|
categories:
|
|
9
13
|
- title: Changes and improvements
|
|
10
14
|
labels:
|
|
@@ -12,11 +16,6 @@ changelog:
|
|
|
12
16
|
exclude:
|
|
13
17
|
labels:
|
|
14
18
|
- dependencies
|
|
15
|
-
authors:
|
|
16
|
-
- weblate
|
|
17
19
|
- title: Dependency updates
|
|
18
20
|
labels:
|
|
19
21
|
- dependencies
|
|
20
|
-
- title: Translation updates
|
|
21
|
-
authors:
|
|
22
|
-
- weblate
|
|
@@ -29,29 +29,31 @@ jobs:
|
|
|
29
29
|
- '3.12'
|
|
30
30
|
- '3.13'
|
|
31
31
|
- '3.14'
|
|
32
|
+
- '3.15'
|
|
32
33
|
include:
|
|
33
34
|
- os: windows-latest
|
|
34
|
-
python-version: '3.
|
|
35
|
+
python-version: '3.15'
|
|
35
36
|
- os: macos-latest
|
|
36
|
-
python-version: '3.
|
|
37
|
+
python-version: '3.15'
|
|
37
38
|
- os: ubuntu-24.04-arm
|
|
38
|
-
python-version: '3.
|
|
39
|
+
python-version: '3.15'
|
|
39
40
|
name: Tests, Python ${{ matrix.python-version }}, ${{ matrix.os }}
|
|
40
41
|
steps:
|
|
41
|
-
- uses: actions/checkout@
|
|
42
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
42
43
|
with:
|
|
43
44
|
persist-credentials: false
|
|
44
45
|
- name: Set up Python ${{ matrix.python-version }}
|
|
45
|
-
uses: actions/setup-python@
|
|
46
|
+
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
46
47
|
id: setup_python
|
|
47
48
|
with:
|
|
48
49
|
python-version: ${{ matrix.python-version }}
|
|
49
|
-
|
|
50
|
+
allow-prereleases: true
|
|
51
|
+
- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 # zizmor: ignore[cache-poisoning]
|
|
50
52
|
# The cache is stored only on main branch and not used for publishing
|
|
51
53
|
with:
|
|
52
54
|
save-cache: ${{ github.ref == 'refs/heads/main' }}
|
|
53
55
|
cache-suffix: ${{ steps.setup_python.outputs.python-version }}
|
|
54
|
-
version: 0.
|
|
56
|
+
version: 0.12.1
|
|
55
57
|
activate-environment: 'true'
|
|
56
58
|
- run: uv sync
|
|
57
59
|
- name: Run tests
|
|
@@ -77,7 +79,7 @@ jobs:
|
|
|
77
79
|
- runner: ubuntu-latest
|
|
78
80
|
target: ppc64le
|
|
79
81
|
steps:
|
|
80
|
-
- uses: actions/checkout@
|
|
82
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
81
83
|
with:
|
|
82
84
|
persist-credentials: false
|
|
83
85
|
- name: Build wheels
|
|
@@ -95,6 +97,13 @@ jobs:
|
|
|
95
97
|
args: --release --out dist -i python3.14t
|
|
96
98
|
sccache: false
|
|
97
99
|
manylinux: auto
|
|
100
|
+
- name: Build Python 3.15 free-threaded wheels
|
|
101
|
+
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
|
|
102
|
+
with:
|
|
103
|
+
target: ${{ matrix.platform.target }}
|
|
104
|
+
args: --release --out dist -i python3.15t --features abi3t
|
|
105
|
+
sccache: false
|
|
106
|
+
manylinux: auto
|
|
98
107
|
- name: Upload wheels
|
|
99
108
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
100
109
|
with:
|
|
@@ -116,7 +125,7 @@ jobs:
|
|
|
116
125
|
- runner: ubuntu-latest
|
|
117
126
|
target: armv7
|
|
118
127
|
steps:
|
|
119
|
-
- uses: actions/checkout@
|
|
128
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
120
129
|
with:
|
|
121
130
|
persist-credentials: false
|
|
122
131
|
- name: Build wheels
|
|
@@ -132,6 +141,13 @@ jobs:
|
|
|
132
141
|
target: ${{ matrix.platform.target }}
|
|
133
142
|
args: --release --out dist -i python3.14t
|
|
134
143
|
|
|
144
|
+
sccache: false
|
|
145
|
+
manylinux: musllinux_1_2
|
|
146
|
+
- name: Build Python 3.15 free-threaded wheels
|
|
147
|
+
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
|
|
148
|
+
with:
|
|
149
|
+
target: ${{ matrix.platform.target }}
|
|
150
|
+
args: --release --out dist -i python3.15t --features abi3t
|
|
135
151
|
sccache: false
|
|
136
152
|
manylinux: musllinux_1_2
|
|
137
153
|
- name: Upload wheels
|
|
@@ -151,10 +167,10 @@ jobs:
|
|
|
151
167
|
- runner: windows-latest
|
|
152
168
|
target: x86
|
|
153
169
|
steps:
|
|
154
|
-
- uses: actions/checkout@
|
|
170
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
155
171
|
with:
|
|
156
172
|
persist-credentials: false
|
|
157
|
-
- uses: actions/setup-python@
|
|
173
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
158
174
|
with:
|
|
159
175
|
python-version: '3.14'
|
|
160
176
|
architecture: ${{ matrix.platform.target }}
|
|
@@ -164,7 +180,7 @@ jobs:
|
|
|
164
180
|
target: ${{ matrix.platform.target }}
|
|
165
181
|
args: --release --out dist -i python
|
|
166
182
|
sccache: false
|
|
167
|
-
- uses: actions/setup-python@
|
|
183
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
168
184
|
with:
|
|
169
185
|
python-version: 3.14t
|
|
170
186
|
architecture: ${{ matrix.platform.target }}
|
|
@@ -174,6 +190,17 @@ jobs:
|
|
|
174
190
|
target: ${{ matrix.platform.target }}
|
|
175
191
|
args: --release --out dist -i python3.14t
|
|
176
192
|
sccache: false
|
|
193
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
194
|
+
with:
|
|
195
|
+
python-version: 3.15t
|
|
196
|
+
allow-prereleases: true
|
|
197
|
+
architecture: ${{ matrix.platform.target }}
|
|
198
|
+
- name: Build Python 3.15 free-threaded wheels
|
|
199
|
+
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
|
|
200
|
+
with:
|
|
201
|
+
target: ${{ matrix.platform.target }}
|
|
202
|
+
args: --release --out dist -i python3.15t --features abi3t
|
|
203
|
+
sccache: false
|
|
177
204
|
- name: Upload wheels
|
|
178
205
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
179
206
|
with:
|
|
@@ -191,10 +218,10 @@ jobs:
|
|
|
191
218
|
- runner: macos-15
|
|
192
219
|
target: aarch64
|
|
193
220
|
steps:
|
|
194
|
-
- uses: actions/checkout@
|
|
221
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
195
222
|
with:
|
|
196
223
|
persist-credentials: false
|
|
197
|
-
- uses: actions/setup-python@
|
|
224
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
198
225
|
with:
|
|
199
226
|
python-version: '3.14'
|
|
200
227
|
- name: Build wheels
|
|
@@ -209,6 +236,16 @@ jobs:
|
|
|
209
236
|
target: ${{ matrix.platform.target }}
|
|
210
237
|
args: --release --out dist -i python3.14t
|
|
211
238
|
|
|
239
|
+
sccache: false
|
|
240
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
241
|
+
with:
|
|
242
|
+
python-version: 3.15t
|
|
243
|
+
allow-prereleases: true
|
|
244
|
+
- name: Build Python 3.15 free-threaded wheels
|
|
245
|
+
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
|
|
246
|
+
with:
|
|
247
|
+
target: ${{ matrix.platform.target }}
|
|
248
|
+
args: --release --out dist -i python3.15t --features abi3t
|
|
212
249
|
sccache: false
|
|
213
250
|
- name: Upload wheels
|
|
214
251
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
@@ -220,7 +257,7 @@ jobs:
|
|
|
220
257
|
runs-on: ubuntu-24.04
|
|
221
258
|
name: Package sdist
|
|
222
259
|
steps:
|
|
223
|
-
- uses: actions/checkout@
|
|
260
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
224
261
|
with:
|
|
225
262
|
persist-credentials: false
|
|
226
263
|
- name: Build sdist
|
|
@@ -273,7 +310,7 @@ jobs:
|
|
|
273
310
|
# Used to generate artifact attestation
|
|
274
311
|
attestations: write
|
|
275
312
|
steps:
|
|
276
|
-
- uses: actions/checkout@
|
|
313
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
277
314
|
with:
|
|
278
315
|
persist-credentials: false
|
|
279
316
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
@@ -296,7 +333,7 @@ jobs:
|
|
|
296
333
|
needs: all-tests
|
|
297
334
|
if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
|
|
298
335
|
steps:
|
|
299
|
-
- uses: actions/checkout@
|
|
336
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
300
337
|
with:
|
|
301
338
|
persist-credentials: false
|
|
302
339
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
@@ -15,7 +15,7 @@ jobs:
|
|
|
15
15
|
runs-on: ubuntu-slim
|
|
16
16
|
steps:
|
|
17
17
|
- name: Checkout Repository
|
|
18
|
-
uses: actions/checkout@
|
|
18
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
19
19
|
with:
|
|
20
20
|
persist-credentials: false
|
|
21
21
|
- name: Dependency Review
|
{unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/workflows/pre-commit.yml
RENAMED
|
@@ -23,7 +23,7 @@ jobs:
|
|
|
23
23
|
runs-on: ubuntu-24.04-arm
|
|
24
24
|
|
|
25
25
|
steps:
|
|
26
|
-
- uses: actions/checkout@
|
|
26
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
27
27
|
with:
|
|
28
28
|
persist-credentials: false
|
|
29
29
|
- uses: ./.github/actions/pre-commit-setup
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# See https://pre-commit.com for more information
|
|
2
2
|
# See https://pre-commit.com/hooks.html for more hooks
|
|
3
|
+
exclude: ^Cargo\.lock$
|
|
4
|
+
|
|
3
5
|
repos:
|
|
4
6
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
5
7
|
rev: v6.0.0
|
|
@@ -19,7 +21,7 @@ repos:
|
|
|
19
21
|
hooks:
|
|
20
22
|
- id: yamllint
|
|
21
23
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
22
|
-
rev: v0.
|
|
24
|
+
rev: v0.16.1
|
|
23
25
|
hooks:
|
|
24
26
|
- id: ruff-check
|
|
25
27
|
args:
|
|
@@ -46,15 +48,15 @@ repos:
|
|
|
46
48
|
hooks:
|
|
47
49
|
- id: actionlint
|
|
48
50
|
- repo: https://github.com/zizmorcore/zizmor-pre-commit
|
|
49
|
-
rev: v1.
|
|
51
|
+
rev: v1.28.0
|
|
50
52
|
hooks:
|
|
51
53
|
- id: zizmor
|
|
52
54
|
- repo: https://github.com/mongodb/kingfisher
|
|
53
|
-
rev: v1.
|
|
55
|
+
rev: v1.110.0
|
|
54
56
|
hooks:
|
|
55
57
|
- id: kingfisher-auto
|
|
56
58
|
- repo: https://github.com/rvben/rumdl-pre-commit
|
|
57
|
-
rev: v0.2.
|
|
59
|
+
rev: v0.2.48
|
|
58
60
|
hooks:
|
|
59
61
|
- id: rumdl
|
|
60
62
|
- id: rumdl-fmt
|
|
@@ -10,9 +10,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
|
10
10
|
|
|
11
11
|
[[package]]
|
|
12
12
|
name = "libc"
|
|
13
|
-
version = "0.2.
|
|
13
|
+
version = "0.2.189"
|
|
14
14
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
15
|
-
checksum = "
|
|
15
|
+
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
|
16
16
|
|
|
17
17
|
[[package]]
|
|
18
18
|
name = "once_cell"
|
|
@@ -22,24 +22,24 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
|
22
22
|
|
|
23
23
|
[[package]]
|
|
24
24
|
name = "portable-atomic"
|
|
25
|
-
version = "1.
|
|
25
|
+
version = "1.14.0"
|
|
26
26
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
27
|
-
checksum = "
|
|
27
|
+
checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
|
|
28
28
|
|
|
29
29
|
[[package]]
|
|
30
30
|
name = "proc-macro2"
|
|
31
|
-
version = "1.0.
|
|
31
|
+
version = "1.0.107"
|
|
32
32
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
33
|
-
checksum = "
|
|
33
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
34
34
|
dependencies = [
|
|
35
35
|
"unicode-ident",
|
|
36
36
|
]
|
|
37
37
|
|
|
38
38
|
[[package]]
|
|
39
39
|
name = "pyo3"
|
|
40
|
-
version = "0.29.
|
|
40
|
+
version = "0.29.1"
|
|
41
41
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
42
|
-
checksum = "
|
|
42
|
+
checksum = "fc153f5fd745cc038b5eed86622125969f8a39834a57bc96beaaf2512b1da729"
|
|
43
43
|
dependencies = [
|
|
44
44
|
"libc",
|
|
45
45
|
"once_cell",
|
|
@@ -51,18 +51,18 @@ dependencies = [
|
|
|
51
51
|
|
|
52
52
|
[[package]]
|
|
53
53
|
name = "pyo3-build-config"
|
|
54
|
-
version = "0.29.
|
|
54
|
+
version = "0.29.1"
|
|
55
55
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
56
|
-
checksum = "
|
|
56
|
+
checksum = "bb77d9aa6d647507b55c69ee714d266d84c526c78ff0bc6dd8757f58591e64d1"
|
|
57
57
|
dependencies = [
|
|
58
58
|
"target-lexicon",
|
|
59
59
|
]
|
|
60
60
|
|
|
61
61
|
[[package]]
|
|
62
62
|
name = "pyo3-ffi"
|
|
63
|
-
version = "0.29.
|
|
63
|
+
version = "0.29.1"
|
|
64
64
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
65
|
-
checksum = "
|
|
65
|
+
checksum = "3160087aa5733bce7d9a729f8c55f85a2a53b74742a09613178eeebff0722253"
|
|
66
66
|
dependencies = [
|
|
67
67
|
"libc",
|
|
68
68
|
"pyo3-build-config",
|
|
@@ -70,9 +70,9 @@ dependencies = [
|
|
|
70
70
|
|
|
71
71
|
[[package]]
|
|
72
72
|
name = "pyo3-macros"
|
|
73
|
-
version = "0.29.
|
|
73
|
+
version = "0.29.1"
|
|
74
74
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
75
|
-
checksum = "
|
|
75
|
+
checksum = "91f9d455db760a9a0b0ddeaac25f1390b8a36ba73dfbda9f127cac6fc340d4d5"
|
|
76
76
|
dependencies = [
|
|
77
77
|
"proc-macro2",
|
|
78
78
|
"pyo3-macros-backend",
|
|
@@ -82,9 +82,9 @@ dependencies = [
|
|
|
82
82
|
|
|
83
83
|
[[package]]
|
|
84
84
|
name = "pyo3-macros-backend"
|
|
85
|
-
version = "0.29.
|
|
85
|
+
version = "0.29.1"
|
|
86
86
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
87
|
-
checksum = "
|
|
87
|
+
checksum = "e343bcec300ff262f5806a33a4e51b6d097a8a46435f512fcb83e95592581625"
|
|
88
88
|
dependencies = [
|
|
89
89
|
"heck",
|
|
90
90
|
"proc-macro2",
|
|
@@ -94,9 +94,9 @@ dependencies = [
|
|
|
94
94
|
|
|
95
95
|
[[package]]
|
|
96
96
|
name = "quote"
|
|
97
|
-
version = "1.0.
|
|
97
|
+
version = "1.0.47"
|
|
98
98
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
99
|
-
checksum = "
|
|
99
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
100
100
|
dependencies = [
|
|
101
101
|
"proc-macro2",
|
|
102
102
|
]
|
|
@@ -138,7 +138,7 @@ checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
|
|
138
138
|
|
|
139
139
|
[[package]]
|
|
140
140
|
name = "unicode-segmentation-rs"
|
|
141
|
-
version = "0.3.
|
|
141
|
+
version = "0.3.2"
|
|
142
142
|
dependencies = [
|
|
143
143
|
"pyo3",
|
|
144
144
|
"unicode-linebreak",
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
[dependencies]
|
|
2
|
-
pyo3 = {version = "0.29.
|
|
2
|
+
pyo3 = {version = "0.29.1", features = ["extension-module", "abi3-py310"]}
|
|
3
3
|
unicode-linebreak = "0.1.5"
|
|
4
4
|
unicode-segmentation = "1.13.3"
|
|
5
5
|
unicode-width = "0.2.2"
|
|
6
6
|
|
|
7
|
+
[features]
|
|
8
|
+
abi3t = ["pyo3/abi3t-py315"]
|
|
9
|
+
|
|
7
10
|
[lib]
|
|
8
11
|
crate-type = ["cdylib"]
|
|
9
12
|
name = "unicode_segmentation_rs"
|
|
@@ -16,4 +19,4 @@ license = "Apache-2.0"
|
|
|
16
19
|
name = "unicode-segmentation-rs"
|
|
17
20
|
readme = "README.md"
|
|
18
21
|
repository = "https://github.com/WeblateOrg/unicode-segmentation-rs"
|
|
19
|
-
version = "0.3.
|
|
22
|
+
version = "0.3.2"
|
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: unicode-segmentation-rs
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Classifier: Programming Language :: Rust
|
|
5
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
6
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
11
|
+
Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
|
|
5
12
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
6
13
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
7
14
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
@@ -6,7 +6,7 @@ version = 1
|
|
|
6
6
|
[[annotations]]
|
|
7
7
|
SPDX-FileCopyrightText = "Michal Čihař <michal@weblate.org>"
|
|
8
8
|
SPDX-License-Identifier = "Apache-2.0"
|
|
9
|
-
path = [".coveragerc", ".dockerignore", ".gitignore", ".isort.cfg", ".pre-commit-config.yaml", "MANIFEST.in", "codecov.yml", "requirements**.txt", "setup.cfg", "pyproject.toml", ".editorconfig", ".well-known/**", "Cargo.toml"]
|
|
9
|
+
path = [".coveragerc", ".dockerignore", ".gitignore", ".isort.cfg", ".pre-commit-config.yaml", "MANIFEST.in", "codecov.yml", "requirements**.txt", "setup.cfg", "pyproject.toml", ".editorconfig", ".well-known/**", "Cargo.toml", "Cargo.lock"]
|
|
10
10
|
precedence = "aggregate"
|
|
11
11
|
|
|
12
12
|
[[annotations]]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[build-system]
|
|
2
2
|
build-backend = "maturin"
|
|
3
|
-
requires = ["maturin>=1.
|
|
3
|
+
requires = ["maturin>=1.14.0,<2.0"]
|
|
4
4
|
|
|
5
5
|
[dependency-groups]
|
|
6
6
|
build = [
|
|
@@ -12,7 +12,7 @@ dev = [
|
|
|
12
12
|
{include-group = "pre-commit"}
|
|
13
13
|
]
|
|
14
14
|
pre-commit = [
|
|
15
|
-
"prek==0.4.
|
|
15
|
+
"prek==0.4.12"
|
|
16
16
|
]
|
|
17
17
|
test = [
|
|
18
18
|
"pytest-github-actions-annotate-failures==0.4.2",
|
|
@@ -22,6 +22,13 @@ test = [
|
|
|
22
22
|
[project]
|
|
23
23
|
classifiers = [
|
|
24
24
|
"Programming Language :: Rust",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Programming Language :: Python :: 3.13",
|
|
29
|
+
"Programming Language :: Python :: 3.14",
|
|
30
|
+
"Programming Language :: Python :: 3.15",
|
|
31
|
+
"Programming Language :: Python :: Free Threading :: 3 - Stable",
|
|
25
32
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
26
33
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
27
34
|
"Topic :: Software Development :: Libraries :: Python Modules"
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
// Copyright © Michal Čihař <michal@weblate.org>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
#[pyo3::pymodule(gil_used = false)]
|
|
6
|
+
mod unicode_segmentation_rs {
|
|
7
|
+
use pyo3::prelude::*;
|
|
8
|
+
use pyo3::types::PyList;
|
|
9
|
+
use unicode_linebreak::{BreakOpportunity, linebreaks as unicode_linebreaks};
|
|
10
|
+
use unicode_segmentation::{UWordBoundIndices, UnicodeSegmentation};
|
|
11
|
+
use unicode_width::UnicodeWidthStr;
|
|
12
|
+
|
|
13
|
+
/// Split a string into grapheme clusters.
|
|
14
|
+
#[pyfunction]
|
|
15
|
+
fn graphemes<'py>(
|
|
16
|
+
py: Python<'py>,
|
|
17
|
+
text: &str,
|
|
18
|
+
is_extended: bool,
|
|
19
|
+
) -> PyResult<Bound<'py, PyList>> {
|
|
20
|
+
PyList::new(py, text.graphemes(is_extended))
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/// Split a string into grapheme cluster indices
|
|
24
|
+
#[pyfunction]
|
|
25
|
+
fn grapheme_indices<'py>(
|
|
26
|
+
py: Python<'py>,
|
|
27
|
+
text: &str,
|
|
28
|
+
is_extended: bool,
|
|
29
|
+
) -> PyResult<Bound<'py, PyList>> {
|
|
30
|
+
PyList::new(py, text.grapheme_indices(is_extended))
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/// Split a string at word boundaries (includes punctuation and whitespace).
|
|
34
|
+
#[pyfunction]
|
|
35
|
+
fn split_word_bounds<'py>(py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyList>> {
|
|
36
|
+
PyList::new(py, text.split_word_bounds())
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/// Split a string at word boundaries with indices.
|
|
40
|
+
#[pyfunction]
|
|
41
|
+
fn split_word_bound_indices<'py>(py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyList>> {
|
|
42
|
+
PyList::new(py, text.split_word_bound_indices())
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/// Get Unicode words from a string (excludes punctuation and whitespace).
|
|
46
|
+
#[pyfunction]
|
|
47
|
+
fn unicode_words<'py>(py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyList>> {
|
|
48
|
+
PyList::new(py, text.unicode_words())
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/// Split a string at word boundaries (includes punctuation and whitespace).
|
|
52
|
+
#[pyfunction]
|
|
53
|
+
fn unicode_sentences<'py>(py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyList>> {
|
|
54
|
+
let mut sentences = text.unicode_sentences();
|
|
55
|
+
PyList::new(py, std::iter::from_fn(move || sentences.next()))
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/// Get Unicode line-break opportunities as UTF-8 byte offsets.
|
|
59
|
+
#[pyfunction]
|
|
60
|
+
fn line_breaks(text: &str) -> PyResult<Vec<(usize, bool)>> {
|
|
61
|
+
if text.is_empty() {
|
|
62
|
+
return Ok(vec![]);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
Ok(unicode_linebreaks(text)
|
|
66
|
+
.map(|(offset, opportunity)| (offset, opportunity == BreakOpportunity::Mandatory))
|
|
67
|
+
.collect())
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/// Split a string at every Unicode line-break opportunity.
|
|
71
|
+
#[pyfunction]
|
|
72
|
+
fn line_break_units<'py>(py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyList>> {
|
|
73
|
+
let mut previous_offset = 0;
|
|
74
|
+
|
|
75
|
+
PyList::new(
|
|
76
|
+
py,
|
|
77
|
+
unicode_linebreaks(text).filter_map(|(offset, _)| {
|
|
78
|
+
if offset == previous_offset {
|
|
79
|
+
None
|
|
80
|
+
} else {
|
|
81
|
+
let unit = &text[previous_offset..offset];
|
|
82
|
+
previous_offset = offset;
|
|
83
|
+
Some(unit)
|
|
84
|
+
}
|
|
85
|
+
}),
|
|
86
|
+
)
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/// Get the display width of a string (as it would appear in a terminal)
|
|
90
|
+
#[pyfunction]
|
|
91
|
+
fn text_width(text: &str) -> PyResult<usize> {
|
|
92
|
+
Ok(UnicodeWidthStr::width(text))
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/// Wrap text for gettext PO files
|
|
96
|
+
///
|
|
97
|
+
/// This implementation follows gettext's wrapping behavior:
|
|
98
|
+
/// - Never breaks escape sequences (\\n, \\", etc.)
|
|
99
|
+
/// - Prefers breaking after spaces
|
|
100
|
+
/// - Handles CJK characters with proper width calculation
|
|
101
|
+
/// - Breaks long words only when necessary
|
|
102
|
+
#[pyfunction]
|
|
103
|
+
fn gettext_wrap(text: &str, width: usize) -> PyResult<Vec<String>> {
|
|
104
|
+
if text.is_empty() || width == 0 {
|
|
105
|
+
return if text.is_empty() {
|
|
106
|
+
Ok(vec![])
|
|
107
|
+
} else {
|
|
108
|
+
Ok(vec![text.to_string()])
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
Ok(wrap_po_chunks(PoChunks::new(text), width))
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/// Iterator over borrowed chunks split using word boundaries and PO-specific rules.
|
|
116
|
+
struct PoChunks<'a> {
|
|
117
|
+
text: &'a str,
|
|
118
|
+
word_bounds: UWordBoundIndices<'a>,
|
|
119
|
+
chunk_start: Option<usize>,
|
|
120
|
+
last_char: Option<char>,
|
|
121
|
+
second_last_char: Option<char>,
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
impl<'a> PoChunks<'a> {
|
|
125
|
+
fn new(text: &'a str) -> Self {
|
|
126
|
+
Self {
|
|
127
|
+
text,
|
|
128
|
+
word_bounds: text.split_word_bound_indices(),
|
|
129
|
+
chunk_start: None,
|
|
130
|
+
last_char: None,
|
|
131
|
+
second_last_char: None,
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/// Add a word-boundary segment and return the completed chunk's byte range, if any.
|
|
136
|
+
fn add_segment(
|
|
137
|
+
&mut self,
|
|
138
|
+
segment_start: usize,
|
|
139
|
+
segment: &str,
|
|
140
|
+
) -> Option<std::ops::Range<usize>> {
|
|
141
|
+
let first_char = segment.chars().next().unwrap();
|
|
142
|
+
let should_merge = self.last_char.is_some_and(|last_char| {
|
|
143
|
+
(self.second_last_char.is_none()
|
|
144
|
+
|| !matches!(last_char, '\\' | 'n')
|
|
145
|
+
|| self.second_last_char != Some('\\'))
|
|
146
|
+
&& (is_mergeable(segment)
|
|
147
|
+
|| (!is_open_parenthesis(&first_char)
|
|
148
|
+
&& !is_line_break(&last_char)
|
|
149
|
+
&& (is_punctuation(&last_char)
|
|
150
|
+
|| (is_punctuation(&first_char) && !last_char.is_whitespace()))))
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
let completed = if should_merge {
|
|
154
|
+
if self.chunk_start.is_none() {
|
|
155
|
+
self.chunk_start = Some(segment_start);
|
|
156
|
+
}
|
|
157
|
+
None
|
|
158
|
+
} else {
|
|
159
|
+
self.chunk_start
|
|
160
|
+
.replace(segment_start)
|
|
161
|
+
.map(|start| start..segment_start)
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
let second_fallback = if should_merge { self.last_char } else { None };
|
|
165
|
+
let mut chars = segment.chars().rev();
|
|
166
|
+
let final_char = chars.next().unwrap();
|
|
167
|
+
if let Some(penultimate_char) = chars.next() {
|
|
168
|
+
self.last_char = Some(penultimate_char);
|
|
169
|
+
self.second_last_char = Some(final_char);
|
|
170
|
+
} else {
|
|
171
|
+
self.last_char = Some(final_char);
|
|
172
|
+
self.second_last_char = second_fallback;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
completed
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
impl<'a> Iterator for PoChunks<'a> {
|
|
180
|
+
type Item = &'a str;
|
|
181
|
+
|
|
182
|
+
fn next(&mut self) -> Option<Self::Item> {
|
|
183
|
+
while let Some((segment_start, segment)) = self.word_bounds.next() {
|
|
184
|
+
// Keep an escape sequence in the chunk before it, then emit that chunk.
|
|
185
|
+
if self.last_char == Some('\\') && segment.len() > 1 {
|
|
186
|
+
let first_char_len = segment.chars().next().unwrap().len_utf8();
|
|
187
|
+
let segment_end = segment_start + first_char_len;
|
|
188
|
+
let completed_start = self.chunk_start.take().unwrap_or(segment_start);
|
|
189
|
+
let remainder = &segment[first_char_len..];
|
|
190
|
+
|
|
191
|
+
if !remainder.is_empty() {
|
|
192
|
+
let pending = self.add_segment(segment_end, remainder);
|
|
193
|
+
debug_assert!(
|
|
194
|
+
pending.is_none(),
|
|
195
|
+
"an emitted escape sequence should leave no pending chunk"
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
return Some(&self.text[completed_start..segment_end]);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if let Some(completed) = self.add_segment(segment_start, segment) {
|
|
203
|
+
return Some(&self.text[completed]);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
self.chunk_start
|
|
208
|
+
.take()
|
|
209
|
+
.map(|start| &self.text[start..self.text.len()])
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/// Wrap borrowed chunks into lines respecting the width limit.
|
|
214
|
+
fn wrap_po_chunks<'a>(chunks: impl IntoIterator<Item = &'a str>, width: usize) -> Vec<String> {
|
|
215
|
+
let mut lines = Vec::new();
|
|
216
|
+
let mut current_line = String::new();
|
|
217
|
+
let mut current_width = 0;
|
|
218
|
+
|
|
219
|
+
for chunk in chunks {
|
|
220
|
+
let chunk_width: usize = chunk.width();
|
|
221
|
+
|
|
222
|
+
if current_width + chunk_width > width && !current_line.is_empty() {
|
|
223
|
+
lines.push(std::mem::take(&mut current_line));
|
|
224
|
+
current_width = 0;
|
|
225
|
+
}
|
|
226
|
+
current_line.push_str(chunk);
|
|
227
|
+
current_width += chunk_width;
|
|
228
|
+
|
|
229
|
+
// Force break on \n
|
|
230
|
+
if chunk.ends_with("\\n") {
|
|
231
|
+
lines.push(std::mem::take(&mut current_line));
|
|
232
|
+
current_width = 0;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if !current_line.is_empty() {
|
|
237
|
+
lines.push(current_line);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
lines
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/// Check if a string contains only mergeable characters
|
|
244
|
+
#[inline]
|
|
245
|
+
fn is_mergeable(s: &str) -> bool {
|
|
246
|
+
s.len() == 1
|
|
247
|
+
&& matches!(
|
|
248
|
+
&s.chars().next().unwrap(),
|
|
249
|
+
'/' | '}' | ')' | '>' | '-' | ' ' | '\t'
|
|
250
|
+
)
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/// Check if a string starts with an open parenthesis character
|
|
254
|
+
#[inline]
|
|
255
|
+
fn is_open_parenthesis(c: &char) -> bool {
|
|
256
|
+
matches!(c, '{' | '(')
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/// Check if a string should trigger line break
|
|
260
|
+
#[inline]
|
|
261
|
+
fn is_line_break(c: &char) -> bool {
|
|
262
|
+
matches!(c, '/' | '}' | ')' | '>' | '-')
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/// Check if a string contains punctuation characters
|
|
266
|
+
#[inline]
|
|
267
|
+
fn is_punctuation(c: &char) -> bool {
|
|
268
|
+
matches!(
|
|
269
|
+
c,
|
|
270
|
+
'!' | '"'
|
|
271
|
+
| '#'
|
|
272
|
+
| '$'
|
|
273
|
+
| '%'
|
|
274
|
+
| '&'
|
|
275
|
+
| '\''
|
|
276
|
+
| '('
|
|
277
|
+
| ')'
|
|
278
|
+
| '*'
|
|
279
|
+
| '+'
|
|
280
|
+
| ','
|
|
281
|
+
| '-'
|
|
282
|
+
| '.'
|
|
283
|
+
| '/'
|
|
284
|
+
| ':'
|
|
285
|
+
| ';'
|
|
286
|
+
| '<'
|
|
287
|
+
| '='
|
|
288
|
+
| '>'
|
|
289
|
+
| '?'
|
|
290
|
+
| '@'
|
|
291
|
+
| '['
|
|
292
|
+
| '\\'
|
|
293
|
+
| ']'
|
|
294
|
+
| '^'
|
|
295
|
+
| '_'
|
|
296
|
+
| '`'
|
|
297
|
+
| '{'
|
|
298
|
+
| '|'
|
|
299
|
+
| '}'
|
|
300
|
+
| '~'
|
|
301
|
+
)
|
|
302
|
+
}
|
|
303
|
+
}
|
|
@@ -1,272 +0,0 @@
|
|
|
1
|
-
// Copyright © Michal Čihař <michal@weblate.org>
|
|
2
|
-
//
|
|
3
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
|
|
5
|
-
#[pyo3::pymodule(gil_used = false)]
|
|
6
|
-
mod unicode_segmentation_rs {
|
|
7
|
-
use pyo3::prelude::*;
|
|
8
|
-
use unicode_linebreak::{BreakOpportunity, linebreaks as unicode_linebreaks};
|
|
9
|
-
use unicode_segmentation::UnicodeSegmentation;
|
|
10
|
-
use unicode_width::UnicodeWidthStr;
|
|
11
|
-
|
|
12
|
-
/// Split a string into grapheme clusters.
|
|
13
|
-
#[pyfunction]
|
|
14
|
-
fn graphemes(text: &str, is_extended: bool) -> PyResult<Vec<String>> {
|
|
15
|
-
Ok(text.graphemes(is_extended).map(|s| s.to_string()).collect())
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
/// Split a string into grapheme cluster indices
|
|
19
|
-
#[pyfunction]
|
|
20
|
-
fn grapheme_indices(text: &str, is_extended: bool) -> PyResult<Vec<(usize, String)>> {
|
|
21
|
-
Ok(text
|
|
22
|
-
.grapheme_indices(is_extended)
|
|
23
|
-
.map(|(i, s)| (i, s.to_string()))
|
|
24
|
-
.collect())
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
/// Split a string at word boundaries (includes punctuation and whitespace).
|
|
28
|
-
#[pyfunction]
|
|
29
|
-
fn split_word_bounds(text: &str) -> PyResult<Vec<String>> {
|
|
30
|
-
Ok(text.split_word_bounds().map(|s| s.to_string()).collect())
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
/// Split a string at word boundaries with indices.
|
|
34
|
-
#[pyfunction]
|
|
35
|
-
fn split_word_bound_indices(text: &str) -> PyResult<Vec<(usize, String)>> {
|
|
36
|
-
Ok(text
|
|
37
|
-
.split_word_bound_indices()
|
|
38
|
-
.map(|(i, s)| (i, s.to_string()))
|
|
39
|
-
.collect())
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
/// Get Unicode words from a string (excludes punctuation and whitespace).
|
|
43
|
-
#[pyfunction]
|
|
44
|
-
fn unicode_words(text: &str) -> PyResult<Vec<String>> {
|
|
45
|
-
Ok(text.unicode_words().map(|s| s.to_string()).collect())
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/// Split a string at word boundaries (includes punctuation and whitespace).
|
|
49
|
-
#[pyfunction]
|
|
50
|
-
fn unicode_sentences(text: &str) -> PyResult<Vec<String>> {
|
|
51
|
-
Ok(text.unicode_sentences().map(|s| s.to_string()).collect())
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/// Get Unicode line-break opportunities as UTF-8 byte offsets.
|
|
55
|
-
#[pyfunction]
|
|
56
|
-
fn line_breaks(text: &str) -> PyResult<Vec<(usize, bool)>> {
|
|
57
|
-
if text.is_empty() {
|
|
58
|
-
return Ok(vec![]);
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
Ok(unicode_linebreaks(text)
|
|
62
|
-
.map(|(offset, opportunity)| (offset, opportunity == BreakOpportunity::Mandatory))
|
|
63
|
-
.collect())
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
/// Split a string at every Unicode line-break opportunity.
|
|
67
|
-
#[pyfunction]
|
|
68
|
-
fn line_break_units(text: &str) -> PyResult<Vec<String>> {
|
|
69
|
-
let mut previous_offset = 0;
|
|
70
|
-
|
|
71
|
-
Ok(unicode_linebreaks(text)
|
|
72
|
-
.filter_map(|(offset, _)| {
|
|
73
|
-
if offset == previous_offset {
|
|
74
|
-
None
|
|
75
|
-
} else {
|
|
76
|
-
let unit = text[previous_offset..offset].to_string();
|
|
77
|
-
previous_offset = offset;
|
|
78
|
-
Some(unit)
|
|
79
|
-
}
|
|
80
|
-
})
|
|
81
|
-
.collect())
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
/// Get the display width of a string (as it would appear in a terminal)
|
|
85
|
-
#[pyfunction]
|
|
86
|
-
fn text_width(text: &str) -> PyResult<usize> {
|
|
87
|
-
Ok(UnicodeWidthStr::width(text))
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/// Wrap text for gettext PO files
|
|
91
|
-
///
|
|
92
|
-
/// This implementation follows gettext's wrapping behavior:
|
|
93
|
-
/// - Never breaks escape sequences (\\n, \\", etc.)
|
|
94
|
-
/// - Prefers breaking after spaces
|
|
95
|
-
/// - Handles CJK characters with proper width calculation
|
|
96
|
-
/// - Breaks long words only when necessary
|
|
97
|
-
#[pyfunction]
|
|
98
|
-
fn gettext_wrap(text: &str, width: usize) -> PyResult<Vec<String>> {
|
|
99
|
-
if text.is_empty() || width == 0 {
|
|
100
|
-
return if text.is_empty() {
|
|
101
|
-
Ok(vec![])
|
|
102
|
-
} else {
|
|
103
|
-
Ok(vec![text.to_string()])
|
|
104
|
-
};
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
// Split text into chunks at word boundaries
|
|
108
|
-
let chunks = split_po_chunks(text);
|
|
109
|
-
|
|
110
|
-
// Wrap chunks into lines
|
|
111
|
-
Ok(wrap_po_chunks(&chunks, width))
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
/// Split text into chunks using word boundaries with PO-specific rules
|
|
115
|
-
fn split_po_chunks(text: &str) -> Vec<String> {
|
|
116
|
-
let mut chunks: Vec<String> = Vec::new();
|
|
117
|
-
let mut last_char: Option<char> = None;
|
|
118
|
-
let mut second_last_char: Option<char> = None;
|
|
119
|
-
let mut second_fallback: Option<char>;
|
|
120
|
-
let mut last_chunk = String::new();
|
|
121
|
-
|
|
122
|
-
for chunk in text.split_word_bounds() {
|
|
123
|
-
let mut chunk_str = chunk.to_string();
|
|
124
|
-
|
|
125
|
-
// Detect escape sequences and emit them
|
|
126
|
-
if last_char.is_some() && last_char.unwrap() == '\\' && chunk_str.len() > 1 {
|
|
127
|
-
last_chunk.push(chunk_str.remove(0));
|
|
128
|
-
chunks.push(last_chunk.clone());
|
|
129
|
-
last_chunk.clear();
|
|
130
|
-
if chunk_str.is_empty() {
|
|
131
|
-
continue;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
let should_merge = last_char.is_some()
|
|
136
|
-
&& (second_last_char.is_none()
|
|
137
|
-
|| !matches!(last_char.unwrap(), '\\' | 'n')
|
|
138
|
-
|| second_last_char.unwrap() != '\\')
|
|
139
|
-
&& (is_mergeable(&chunk_str)
|
|
140
|
-
|| (!is_open_parenthesis(&chunk_str.chars().next().unwrap())
|
|
141
|
-
&& !is_line_break(&last_char.unwrap())
|
|
142
|
-
&& (is_punctuation(&last_char.unwrap())
|
|
143
|
-
|| (is_punctuation(&chunk_str.chars().next().unwrap())
|
|
144
|
-
&& !last_char.unwrap().is_whitespace()))));
|
|
145
|
-
|
|
146
|
-
if !should_merge {
|
|
147
|
-
if !last_chunk.is_empty() {
|
|
148
|
-
chunks.push(last_chunk.clone())
|
|
149
|
-
}
|
|
150
|
-
last_chunk.clear();
|
|
151
|
-
second_fallback = None;
|
|
152
|
-
} else {
|
|
153
|
-
second_fallback = Some(last_char.unwrap());
|
|
154
|
-
}
|
|
155
|
-
last_chunk.push_str(chunk_str.as_str());
|
|
156
|
-
|
|
157
|
-
// Update last_char and second_last_char
|
|
158
|
-
let chars: Vec<char> = chunk_str.chars().collect();
|
|
159
|
-
if chars.len() >= 2 {
|
|
160
|
-
let len = chars.len();
|
|
161
|
-
last_char = Some(chars[len - 2]);
|
|
162
|
-
second_last_char = Some(chars[len - 1]);
|
|
163
|
-
} else {
|
|
164
|
-
second_last_char = second_fallback;
|
|
165
|
-
last_char = Some(chars[0]);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
if !last_chunk.is_empty() {
|
|
169
|
-
chunks.push(last_chunk.clone())
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
chunks
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
/// Wrap chunks into lines respecting the width limit
|
|
176
|
-
fn wrap_po_chunks(chunks: &Vec<String>, width: usize) -> Vec<String> {
|
|
177
|
-
let mut lines = Vec::new();
|
|
178
|
-
let mut current_line = String::new();
|
|
179
|
-
let mut current_width = 0;
|
|
180
|
-
|
|
181
|
-
for chunk in chunks {
|
|
182
|
-
let chunk_width: usize = chunk.width();
|
|
183
|
-
|
|
184
|
-
if current_width + chunk_width <= width {
|
|
185
|
-
current_line.push_str(chunk.as_str());
|
|
186
|
-
current_width += chunk_width;
|
|
187
|
-
} else {
|
|
188
|
-
if !current_line.is_empty() {
|
|
189
|
-
lines.push(current_line.clone());
|
|
190
|
-
current_line.clear();
|
|
191
|
-
current_width = 0;
|
|
192
|
-
}
|
|
193
|
-
current_line.push_str(chunk.as_str());
|
|
194
|
-
current_width += chunk_width;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
// Force break on \n
|
|
198
|
-
if chunk.ends_with("\\n") {
|
|
199
|
-
lines.push(current_line.clone());
|
|
200
|
-
current_line.clear();
|
|
201
|
-
current_width = 0;
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
if !current_line.is_empty() {
|
|
206
|
-
lines.push(current_line.clone());
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
lines
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
/// Check if a string contains only mergeable characters
|
|
213
|
-
#[inline]
|
|
214
|
-
fn is_mergeable(s: &str) -> bool {
|
|
215
|
-
s.len() == 1
|
|
216
|
-
&& matches!(
|
|
217
|
-
&s.chars().next().unwrap(),
|
|
218
|
-
'/' | '}' | ')' | '>' | '-' | ' ' | '\t'
|
|
219
|
-
)
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
/// Check if a string starts with an open parenthesis character
|
|
223
|
-
#[inline]
|
|
224
|
-
fn is_open_parenthesis(c: &char) -> bool {
|
|
225
|
-
matches!(c, '{' | '(')
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
/// Check if a string should trigger line break
|
|
229
|
-
#[inline]
|
|
230
|
-
fn is_line_break(c: &char) -> bool {
|
|
231
|
-
matches!(c, '/' | '}' | ')' | '>' | '-')
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
/// Check if a string contains punctuation characters
|
|
235
|
-
#[inline]
|
|
236
|
-
fn is_punctuation(c: &char) -> bool {
|
|
237
|
-
matches!(
|
|
238
|
-
c,
|
|
239
|
-
'!' | '"'
|
|
240
|
-
| '#'
|
|
241
|
-
| '$'
|
|
242
|
-
| '%'
|
|
243
|
-
| '&'
|
|
244
|
-
| '\''
|
|
245
|
-
| '('
|
|
246
|
-
| ')'
|
|
247
|
-
| '*'
|
|
248
|
-
| '+'
|
|
249
|
-
| ','
|
|
250
|
-
| '-'
|
|
251
|
-
| '.'
|
|
252
|
-
| '/'
|
|
253
|
-
| ':'
|
|
254
|
-
| ';'
|
|
255
|
-
| '<'
|
|
256
|
-
| '='
|
|
257
|
-
| '>'
|
|
258
|
-
| '?'
|
|
259
|
-
| '@'
|
|
260
|
-
| '['
|
|
261
|
-
| '\\'
|
|
262
|
-
| ']'
|
|
263
|
-
| '^'
|
|
264
|
-
| '_'
|
|
265
|
-
| '`'
|
|
266
|
-
| '{'
|
|
267
|
-
| '|'
|
|
268
|
-
| '}'
|
|
269
|
-
| '~'
|
|
270
|
-
)
|
|
271
|
-
}
|
|
272
|
-
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/ISSUE_TEMPLATE/config.yml
RENAMED
|
File without changes
|
|
File without changes
|
{unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/PULL_REQUEST_TEMPLATE.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/renovate.json.license
RENAMED
|
File without changes
|
{unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/.github/workflows/pull_requests.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/tests/test_readme_examples.py
RENAMED
|
File without changes
|
{unicode_segmentation_rs-0.3.0 → unicode_segmentation_rs-0.3.2}/tests/test_unicode_segmentation.py
RENAMED
|
File without changes
|
|
File without changes
|