laghu-rs 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- laghu_rs-0.1.0/.gitignore +1 -0
- laghu_rs-0.1.0/Cargo.lock +804 -0
- laghu_rs-0.1.0/Cargo.toml +21 -0
- laghu_rs-0.1.0/PKG-INFO +34 -0
- laghu_rs-0.1.0/README.md +21 -0
- laghu_rs-0.1.0/pyproject.toml +22 -0
- laghu_rs-0.1.0/smoke.py +26 -0
- laghu_rs-0.1.0/src/lib.rs +313 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
target/
|
|
@@ -0,0 +1,804 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "ahash"
|
|
7
|
+
version = "0.8.12"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"cfg-if",
|
|
12
|
+
"getrandom",
|
|
13
|
+
"once_cell",
|
|
14
|
+
"serde",
|
|
15
|
+
"version_check",
|
|
16
|
+
"zerocopy",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[[package]]
|
|
20
|
+
name = "aho-corasick"
|
|
21
|
+
version = "1.1.5"
|
|
22
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
23
|
+
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
|
24
|
+
dependencies = [
|
|
25
|
+
"memchr",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[[package]]
|
|
29
|
+
name = "base64"
|
|
30
|
+
version = "0.13.1"
|
|
31
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
32
|
+
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
|
|
33
|
+
|
|
34
|
+
[[package]]
|
|
35
|
+
name = "bitflags"
|
|
36
|
+
version = "2.13.2"
|
|
37
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
38
|
+
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
|
39
|
+
|
|
40
|
+
[[package]]
|
|
41
|
+
name = "castaway"
|
|
42
|
+
version = "0.2.4"
|
|
43
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
44
|
+
checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
|
|
45
|
+
dependencies = [
|
|
46
|
+
"rustversion",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
[[package]]
|
|
50
|
+
name = "cc"
|
|
51
|
+
version = "1.9.0"
|
|
52
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
53
|
+
checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df"
|
|
54
|
+
dependencies = [
|
|
55
|
+
"find-msvc-tools",
|
|
56
|
+
"shlex",
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
[[package]]
|
|
60
|
+
name = "cfg-if"
|
|
61
|
+
version = "1.0.5"
|
|
62
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
63
|
+
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
|
64
|
+
|
|
65
|
+
[[package]]
|
|
66
|
+
name = "compact_str"
|
|
67
|
+
version = "0.9.1"
|
|
68
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
69
|
+
checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab"
|
|
70
|
+
dependencies = [
|
|
71
|
+
"castaway",
|
|
72
|
+
"cfg-if",
|
|
73
|
+
"itoa",
|
|
74
|
+
"rustversion",
|
|
75
|
+
"ryu",
|
|
76
|
+
"serde",
|
|
77
|
+
"static_assertions",
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
[[package]]
|
|
81
|
+
name = "crossbeam-deque"
|
|
82
|
+
version = "0.8.8"
|
|
83
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
84
|
+
checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
|
|
85
|
+
dependencies = [
|
|
86
|
+
"crossbeam-epoch",
|
|
87
|
+
"crossbeam-utils",
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
[[package]]
|
|
91
|
+
name = "crossbeam-epoch"
|
|
92
|
+
version = "0.9.21"
|
|
93
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
94
|
+
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
|
|
95
|
+
dependencies = [
|
|
96
|
+
"crossbeam-utils",
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
[[package]]
|
|
100
|
+
name = "crossbeam-utils"
|
|
101
|
+
version = "0.8.23"
|
|
102
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
103
|
+
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
|
104
|
+
|
|
105
|
+
[[package]]
|
|
106
|
+
name = "daachorse"
|
|
107
|
+
version = "3.0.3"
|
|
108
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
109
|
+
checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d"
|
|
110
|
+
|
|
111
|
+
[[package]]
|
|
112
|
+
name = "darling"
|
|
113
|
+
version = "0.20.11"
|
|
114
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
115
|
+
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
|
|
116
|
+
dependencies = [
|
|
117
|
+
"darling_core",
|
|
118
|
+
"darling_macro",
|
|
119
|
+
]
|
|
120
|
+
|
|
121
|
+
[[package]]
|
|
122
|
+
name = "darling_core"
|
|
123
|
+
version = "0.20.11"
|
|
124
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
125
|
+
checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
|
|
126
|
+
dependencies = [
|
|
127
|
+
"fnv",
|
|
128
|
+
"ident_case",
|
|
129
|
+
"proc-macro2",
|
|
130
|
+
"quote",
|
|
131
|
+
"strsim",
|
|
132
|
+
"syn 2.0.119",
|
|
133
|
+
]
|
|
134
|
+
|
|
135
|
+
[[package]]
|
|
136
|
+
name = "darling_macro"
|
|
137
|
+
version = "0.20.11"
|
|
138
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
139
|
+
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
|
|
140
|
+
dependencies = [
|
|
141
|
+
"darling_core",
|
|
142
|
+
"quote",
|
|
143
|
+
"syn 2.0.119",
|
|
144
|
+
]
|
|
145
|
+
|
|
146
|
+
[[package]]
|
|
147
|
+
name = "dary_heap"
|
|
148
|
+
version = "0.3.9"
|
|
149
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
150
|
+
checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
|
|
151
|
+
dependencies = [
|
|
152
|
+
"serde",
|
|
153
|
+
]
|
|
154
|
+
|
|
155
|
+
[[package]]
|
|
156
|
+
name = "derive_builder"
|
|
157
|
+
version = "0.20.2"
|
|
158
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
159
|
+
checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
|
|
160
|
+
dependencies = [
|
|
161
|
+
"derive_builder_macro",
|
|
162
|
+
]
|
|
163
|
+
|
|
164
|
+
[[package]]
|
|
165
|
+
name = "derive_builder_core"
|
|
166
|
+
version = "0.20.2"
|
|
167
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
168
|
+
checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
|
|
169
|
+
dependencies = [
|
|
170
|
+
"darling",
|
|
171
|
+
"proc-macro2",
|
|
172
|
+
"quote",
|
|
173
|
+
"syn 2.0.119",
|
|
174
|
+
]
|
|
175
|
+
|
|
176
|
+
[[package]]
|
|
177
|
+
name = "derive_builder_macro"
|
|
178
|
+
version = "0.20.2"
|
|
179
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
180
|
+
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
|
|
181
|
+
dependencies = [
|
|
182
|
+
"derive_builder_core",
|
|
183
|
+
"syn 2.0.119",
|
|
184
|
+
]
|
|
185
|
+
|
|
186
|
+
[[package]]
|
|
187
|
+
name = "either"
|
|
188
|
+
version = "1.19.0"
|
|
189
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
190
|
+
checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
|
|
191
|
+
|
|
192
|
+
[[package]]
|
|
193
|
+
name = "esaxx-rs"
|
|
194
|
+
version = "0.1.10"
|
|
195
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
196
|
+
checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
|
|
197
|
+
|
|
198
|
+
[[package]]
|
|
199
|
+
name = "find-msvc-tools"
|
|
200
|
+
version = "0.1.15"
|
|
201
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
202
|
+
checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"
|
|
203
|
+
|
|
204
|
+
[[package]]
|
|
205
|
+
name = "fnv"
|
|
206
|
+
version = "1.0.7"
|
|
207
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
208
|
+
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
|
209
|
+
|
|
210
|
+
[[package]]
|
|
211
|
+
name = "getrandom"
|
|
212
|
+
version = "0.3.4"
|
|
213
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
214
|
+
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
|
215
|
+
dependencies = [
|
|
216
|
+
"cfg-if",
|
|
217
|
+
"libc",
|
|
218
|
+
"r-efi",
|
|
219
|
+
"wasip2",
|
|
220
|
+
]
|
|
221
|
+
|
|
222
|
+
[[package]]
|
|
223
|
+
name = "heck"
|
|
224
|
+
version = "0.5.0"
|
|
225
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
226
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
227
|
+
|
|
228
|
+
[[package]]
|
|
229
|
+
name = "ident_case"
|
|
230
|
+
version = "1.0.1"
|
|
231
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
232
|
+
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
|
|
233
|
+
|
|
234
|
+
[[package]]
|
|
235
|
+
name = "itertools"
|
|
236
|
+
version = "0.14.0"
|
|
237
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
238
|
+
checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
|
|
239
|
+
dependencies = [
|
|
240
|
+
"either",
|
|
241
|
+
]
|
|
242
|
+
|
|
243
|
+
[[package]]
|
|
244
|
+
name = "itoa"
|
|
245
|
+
version = "1.0.18"
|
|
246
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
247
|
+
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|
248
|
+
|
|
249
|
+
[[package]]
|
|
250
|
+
name = "laghu_rs"
|
|
251
|
+
version = "0.1.0"
|
|
252
|
+
dependencies = [
|
|
253
|
+
"pyo3",
|
|
254
|
+
"tokenizers",
|
|
255
|
+
]
|
|
256
|
+
|
|
257
|
+
[[package]]
|
|
258
|
+
name = "libc"
|
|
259
|
+
version = "0.2.186"
|
|
260
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
261
|
+
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
|
262
|
+
|
|
263
|
+
[[package]]
|
|
264
|
+
name = "log"
|
|
265
|
+
version = "0.4.34"
|
|
266
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
267
|
+
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
|
268
|
+
|
|
269
|
+
[[package]]
|
|
270
|
+
name = "macro_rules_attribute"
|
|
271
|
+
version = "0.2.3"
|
|
272
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
273
|
+
checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c"
|
|
274
|
+
dependencies = [
|
|
275
|
+
"macro_rules_attribute-proc_macro",
|
|
276
|
+
"pastey",
|
|
277
|
+
]
|
|
278
|
+
|
|
279
|
+
[[package]]
|
|
280
|
+
name = "macro_rules_attribute-proc_macro"
|
|
281
|
+
version = "0.2.3"
|
|
282
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
283
|
+
checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c"
|
|
284
|
+
|
|
285
|
+
[[package]]
|
|
286
|
+
name = "memchr"
|
|
287
|
+
version = "2.8.3"
|
|
288
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
289
|
+
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
290
|
+
|
|
291
|
+
[[package]]
|
|
292
|
+
name = "minimal-lexical"
|
|
293
|
+
version = "0.2.1"
|
|
294
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
295
|
+
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
|
296
|
+
|
|
297
|
+
[[package]]
|
|
298
|
+
name = "monostate"
|
|
299
|
+
version = "0.1.18"
|
|
300
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
301
|
+
checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67"
|
|
302
|
+
dependencies = [
|
|
303
|
+
"monostate-impl",
|
|
304
|
+
"serde",
|
|
305
|
+
"serde_core",
|
|
306
|
+
]
|
|
307
|
+
|
|
308
|
+
[[package]]
|
|
309
|
+
name = "monostate-impl"
|
|
310
|
+
version = "0.1.18"
|
|
311
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
312
|
+
checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9"
|
|
313
|
+
dependencies = [
|
|
314
|
+
"proc-macro2",
|
|
315
|
+
"quote",
|
|
316
|
+
"syn 2.0.119",
|
|
317
|
+
]
|
|
318
|
+
|
|
319
|
+
[[package]]
|
|
320
|
+
name = "nom"
|
|
321
|
+
version = "7.1.3"
|
|
322
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
323
|
+
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
|
324
|
+
dependencies = [
|
|
325
|
+
"memchr",
|
|
326
|
+
"minimal-lexical",
|
|
327
|
+
]
|
|
328
|
+
|
|
329
|
+
[[package]]
|
|
330
|
+
name = "once_cell"
|
|
331
|
+
version = "1.21.4"
|
|
332
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
333
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
334
|
+
|
|
335
|
+
[[package]]
|
|
336
|
+
name = "onig"
|
|
337
|
+
version = "6.5.3"
|
|
338
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
339
|
+
checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2"
|
|
340
|
+
dependencies = [
|
|
341
|
+
"bitflags",
|
|
342
|
+
"libc",
|
|
343
|
+
"once_cell",
|
|
344
|
+
"onig_sys",
|
|
345
|
+
]
|
|
346
|
+
|
|
347
|
+
[[package]]
|
|
348
|
+
name = "onig_sys"
|
|
349
|
+
version = "69.9.3"
|
|
350
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
351
|
+
checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7"
|
|
352
|
+
dependencies = [
|
|
353
|
+
"cc",
|
|
354
|
+
"pkg-config",
|
|
355
|
+
]
|
|
356
|
+
|
|
357
|
+
[[package]]
|
|
358
|
+
name = "paste"
|
|
359
|
+
version = "1.0.15"
|
|
360
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
361
|
+
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
|
|
362
|
+
|
|
363
|
+
[[package]]
|
|
364
|
+
name = "pastey"
|
|
365
|
+
version = "0.2.3"
|
|
366
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
367
|
+
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
|
368
|
+
|
|
369
|
+
[[package]]
|
|
370
|
+
name = "pkg-config"
|
|
371
|
+
version = "0.3.34"
|
|
372
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
373
|
+
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
|
374
|
+
|
|
375
|
+
[[package]]
|
|
376
|
+
name = "portable-atomic"
|
|
377
|
+
version = "1.14.0"
|
|
378
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
379
|
+
checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
|
|
380
|
+
|
|
381
|
+
[[package]]
|
|
382
|
+
name = "ppv-lite86"
|
|
383
|
+
version = "0.2.21"
|
|
384
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
385
|
+
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
|
|
386
|
+
dependencies = [
|
|
387
|
+
"zerocopy",
|
|
388
|
+
]
|
|
389
|
+
|
|
390
|
+
[[package]]
|
|
391
|
+
name = "proc-macro2"
|
|
392
|
+
version = "1.0.107"
|
|
393
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
394
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
395
|
+
dependencies = [
|
|
396
|
+
"unicode-ident",
|
|
397
|
+
]
|
|
398
|
+
|
|
399
|
+
[[package]]
|
|
400
|
+
name = "pyo3"
|
|
401
|
+
version = "0.29.0"
|
|
402
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
403
|
+
checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c"
|
|
404
|
+
dependencies = [
|
|
405
|
+
"libc",
|
|
406
|
+
"once_cell",
|
|
407
|
+
"portable-atomic",
|
|
408
|
+
"pyo3-build-config",
|
|
409
|
+
"pyo3-ffi",
|
|
410
|
+
"pyo3-macros",
|
|
411
|
+
]
|
|
412
|
+
|
|
413
|
+
[[package]]
|
|
414
|
+
name = "pyo3-build-config"
|
|
415
|
+
version = "0.29.0"
|
|
416
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
417
|
+
checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
|
|
418
|
+
dependencies = [
|
|
419
|
+
"target-lexicon",
|
|
420
|
+
]
|
|
421
|
+
|
|
422
|
+
[[package]]
|
|
423
|
+
name = "pyo3-ffi"
|
|
424
|
+
version = "0.29.0"
|
|
425
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
426
|
+
checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b"
|
|
427
|
+
dependencies = [
|
|
428
|
+
"libc",
|
|
429
|
+
"pyo3-build-config",
|
|
430
|
+
]
|
|
431
|
+
|
|
432
|
+
[[package]]
|
|
433
|
+
name = "pyo3-macros"
|
|
434
|
+
version = "0.29.0"
|
|
435
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
436
|
+
checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771"
|
|
437
|
+
dependencies = [
|
|
438
|
+
"proc-macro2",
|
|
439
|
+
"pyo3-macros-backend",
|
|
440
|
+
"quote",
|
|
441
|
+
"syn 2.0.119",
|
|
442
|
+
]
|
|
443
|
+
|
|
444
|
+
[[package]]
|
|
445
|
+
name = "pyo3-macros-backend"
|
|
446
|
+
version = "0.29.0"
|
|
447
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
448
|
+
checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362"
|
|
449
|
+
dependencies = [
|
|
450
|
+
"heck",
|
|
451
|
+
"proc-macro2",
|
|
452
|
+
"quote",
|
|
453
|
+
"syn 2.0.119",
|
|
454
|
+
]
|
|
455
|
+
|
|
456
|
+
[[package]]
|
|
457
|
+
name = "quote"
|
|
458
|
+
version = "1.0.47"
|
|
459
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
460
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
461
|
+
dependencies = [
|
|
462
|
+
"proc-macro2",
|
|
463
|
+
]
|
|
464
|
+
|
|
465
|
+
[[package]]
|
|
466
|
+
name = "r-efi"
|
|
467
|
+
version = "5.3.0"
|
|
468
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
469
|
+
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
|
470
|
+
|
|
471
|
+
[[package]]
|
|
472
|
+
name = "rand"
|
|
473
|
+
version = "0.9.5"
|
|
474
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
475
|
+
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
|
|
476
|
+
dependencies = [
|
|
477
|
+
"rand_chacha",
|
|
478
|
+
"rand_core",
|
|
479
|
+
]
|
|
480
|
+
|
|
481
|
+
[[package]]
|
|
482
|
+
name = "rand_chacha"
|
|
483
|
+
version = "0.9.0"
|
|
484
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
485
|
+
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
|
486
|
+
dependencies = [
|
|
487
|
+
"ppv-lite86",
|
|
488
|
+
"rand_core",
|
|
489
|
+
]
|
|
490
|
+
|
|
491
|
+
[[package]]
|
|
492
|
+
name = "rand_core"
|
|
493
|
+
version = "0.9.5"
|
|
494
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
495
|
+
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
|
|
496
|
+
dependencies = [
|
|
497
|
+
"getrandom",
|
|
498
|
+
]
|
|
499
|
+
|
|
500
|
+
[[package]]
|
|
501
|
+
name = "rayon"
|
|
502
|
+
version = "1.12.0"
|
|
503
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
504
|
+
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
|
505
|
+
dependencies = [
|
|
506
|
+
"either",
|
|
507
|
+
"rayon-core",
|
|
508
|
+
]
|
|
509
|
+
|
|
510
|
+
[[package]]
|
|
511
|
+
name = "rayon-cond"
|
|
512
|
+
version = "0.4.0"
|
|
513
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
514
|
+
checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"
|
|
515
|
+
dependencies = [
|
|
516
|
+
"either",
|
|
517
|
+
"itertools",
|
|
518
|
+
"rayon",
|
|
519
|
+
]
|
|
520
|
+
|
|
521
|
+
[[package]]
|
|
522
|
+
name = "rayon-core"
|
|
523
|
+
version = "1.13.0"
|
|
524
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
525
|
+
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
|
526
|
+
dependencies = [
|
|
527
|
+
"crossbeam-deque",
|
|
528
|
+
"crossbeam-utils",
|
|
529
|
+
]
|
|
530
|
+
|
|
531
|
+
[[package]]
|
|
532
|
+
name = "regex"
|
|
533
|
+
version = "1.13.1"
|
|
534
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
535
|
+
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
|
536
|
+
dependencies = [
|
|
537
|
+
"aho-corasick",
|
|
538
|
+
"memchr",
|
|
539
|
+
"regex-automata",
|
|
540
|
+
"regex-syntax",
|
|
541
|
+
]
|
|
542
|
+
|
|
543
|
+
[[package]]
|
|
544
|
+
name = "regex-automata"
|
|
545
|
+
version = "0.4.18"
|
|
546
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
547
|
+
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
|
548
|
+
dependencies = [
|
|
549
|
+
"aho-corasick",
|
|
550
|
+
"memchr",
|
|
551
|
+
"regex-syntax",
|
|
552
|
+
]
|
|
553
|
+
|
|
554
|
+
[[package]]
|
|
555
|
+
name = "regex-syntax"
|
|
556
|
+
version = "0.8.11"
|
|
557
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
558
|
+
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
|
559
|
+
|
|
560
|
+
[[package]]
|
|
561
|
+
name = "rustversion"
|
|
562
|
+
version = "1.0.23"
|
|
563
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
564
|
+
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
|
565
|
+
|
|
566
|
+
[[package]]
|
|
567
|
+
name = "ryu"
|
|
568
|
+
version = "1.0.23"
|
|
569
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
570
|
+
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
|
571
|
+
|
|
572
|
+
[[package]]
|
|
573
|
+
name = "serde"
|
|
574
|
+
version = "1.0.229"
|
|
575
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
576
|
+
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
|
577
|
+
dependencies = [
|
|
578
|
+
"serde_core",
|
|
579
|
+
"serde_derive",
|
|
580
|
+
]
|
|
581
|
+
|
|
582
|
+
[[package]]
|
|
583
|
+
name = "serde_core"
|
|
584
|
+
version = "1.0.229"
|
|
585
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
586
|
+
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
|
587
|
+
dependencies = [
|
|
588
|
+
"serde_derive",
|
|
589
|
+
]
|
|
590
|
+
|
|
591
|
+
[[package]]
|
|
592
|
+
name = "serde_derive"
|
|
593
|
+
version = "1.0.229"
|
|
594
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
595
|
+
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
596
|
+
dependencies = [
|
|
597
|
+
"proc-macro2",
|
|
598
|
+
"quote",
|
|
599
|
+
"syn 3.0.9",
|
|
600
|
+
]
|
|
601
|
+
|
|
602
|
+
[[package]]
|
|
603
|
+
name = "serde_json"
|
|
604
|
+
version = "1.0.154"
|
|
605
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
606
|
+
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
|
607
|
+
dependencies = [
|
|
608
|
+
"itoa",
|
|
609
|
+
"memchr",
|
|
610
|
+
"serde",
|
|
611
|
+
"serde_core",
|
|
612
|
+
"zmij",
|
|
613
|
+
]
|
|
614
|
+
|
|
615
|
+
[[package]]
|
|
616
|
+
name = "shlex"
|
|
617
|
+
version = "2.0.1"
|
|
618
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
619
|
+
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
|
620
|
+
|
|
621
|
+
[[package]]
|
|
622
|
+
name = "smallvec"
|
|
623
|
+
version = "1.16.3"
|
|
624
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
625
|
+
checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
|
|
626
|
+
|
|
627
|
+
[[package]]
|
|
628
|
+
name = "spm_precompiled"
|
|
629
|
+
version = "0.1.4"
|
|
630
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
631
|
+
checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
|
|
632
|
+
dependencies = [
|
|
633
|
+
"base64",
|
|
634
|
+
"nom",
|
|
635
|
+
"serde",
|
|
636
|
+
"unicode-segmentation",
|
|
637
|
+
]
|
|
638
|
+
|
|
639
|
+
[[package]]
|
|
640
|
+
name = "static_assertions"
|
|
641
|
+
version = "1.1.0"
|
|
642
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
643
|
+
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
|
644
|
+
|
|
645
|
+
[[package]]
|
|
646
|
+
name = "strsim"
|
|
647
|
+
version = "0.11.1"
|
|
648
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
649
|
+
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
|
650
|
+
|
|
651
|
+
[[package]]
|
|
652
|
+
name = "syn"
|
|
653
|
+
version = "2.0.119"
|
|
654
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
655
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
656
|
+
dependencies = [
|
|
657
|
+
"proc-macro2",
|
|
658
|
+
"quote",
|
|
659
|
+
"unicode-ident",
|
|
660
|
+
]
|
|
661
|
+
|
|
662
|
+
[[package]]
|
|
663
|
+
name = "syn"
|
|
664
|
+
version = "3.0.9"
|
|
665
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
666
|
+
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
667
|
+
dependencies = [
|
|
668
|
+
"proc-macro2",
|
|
669
|
+
"quote",
|
|
670
|
+
"unicode-ident",
|
|
671
|
+
]
|
|
672
|
+
|
|
673
|
+
[[package]]
|
|
674
|
+
name = "target-lexicon"
|
|
675
|
+
version = "0.13.5"
|
|
676
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
677
|
+
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
678
|
+
|
|
679
|
+
[[package]]
|
|
680
|
+
name = "thiserror"
|
|
681
|
+
version = "2.0.21"
|
|
682
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
683
|
+
checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
|
|
684
|
+
dependencies = [
|
|
685
|
+
"thiserror-impl",
|
|
686
|
+
]
|
|
687
|
+
|
|
688
|
+
[[package]]
|
|
689
|
+
name = "thiserror-impl"
|
|
690
|
+
version = "2.0.21"
|
|
691
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
692
|
+
checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
|
|
693
|
+
dependencies = [
|
|
694
|
+
"proc-macro2",
|
|
695
|
+
"quote",
|
|
696
|
+
"syn 3.0.9",
|
|
697
|
+
]
|
|
698
|
+
|
|
699
|
+
[[package]]
|
|
700
|
+
name = "tokenizers"
|
|
701
|
+
version = "0.23.2"
|
|
702
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
703
|
+
checksum = "7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc"
|
|
704
|
+
dependencies = [
|
|
705
|
+
"ahash",
|
|
706
|
+
"compact_str",
|
|
707
|
+
"daachorse",
|
|
708
|
+
"dary_heap",
|
|
709
|
+
"derive_builder",
|
|
710
|
+
"esaxx-rs",
|
|
711
|
+
"getrandom",
|
|
712
|
+
"itertools",
|
|
713
|
+
"log",
|
|
714
|
+
"macro_rules_attribute",
|
|
715
|
+
"monostate",
|
|
716
|
+
"onig",
|
|
717
|
+
"paste",
|
|
718
|
+
"rand",
|
|
719
|
+
"rayon",
|
|
720
|
+
"rayon-cond",
|
|
721
|
+
"regex",
|
|
722
|
+
"regex-syntax",
|
|
723
|
+
"serde",
|
|
724
|
+
"serde_json",
|
|
725
|
+
"spm_precompiled",
|
|
726
|
+
"thiserror",
|
|
727
|
+
"unicode-normalization-alignments",
|
|
728
|
+
"unicode-segmentation",
|
|
729
|
+
"unicode_categories",
|
|
730
|
+
]
|
|
731
|
+
|
|
732
|
+
[[package]]
|
|
733
|
+
name = "unicode-ident"
|
|
734
|
+
version = "1.0.24"
|
|
735
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
736
|
+
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
|
737
|
+
|
|
738
|
+
[[package]]
|
|
739
|
+
name = "unicode-normalization-alignments"
|
|
740
|
+
version = "0.1.12"
|
|
741
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
742
|
+
checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
|
|
743
|
+
dependencies = [
|
|
744
|
+
"smallvec",
|
|
745
|
+
]
|
|
746
|
+
|
|
747
|
+
[[package]]
|
|
748
|
+
name = "unicode-segmentation"
|
|
749
|
+
version = "1.13.3"
|
|
750
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
751
|
+
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
|
752
|
+
|
|
753
|
+
[[package]]
|
|
754
|
+
name = "unicode_categories"
|
|
755
|
+
version = "0.1.1"
|
|
756
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
757
|
+
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
|
|
758
|
+
|
|
759
|
+
[[package]]
|
|
760
|
+
name = "version_check"
|
|
761
|
+
version = "0.9.5"
|
|
762
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
763
|
+
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
|
764
|
+
|
|
765
|
+
[[package]]
|
|
766
|
+
name = "wasip2"
|
|
767
|
+
version = "1.0.4+wasi-0.2.12"
|
|
768
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
769
|
+
checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
|
|
770
|
+
dependencies = [
|
|
771
|
+
"wit-bindgen",
|
|
772
|
+
]
|
|
773
|
+
|
|
774
|
+
[[package]]
|
|
775
|
+
name = "wit-bindgen"
|
|
776
|
+
version = "0.57.1"
|
|
777
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
778
|
+
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
|
|
779
|
+
|
|
780
|
+
[[package]]
|
|
781
|
+
name = "zerocopy"
|
|
782
|
+
version = "0.8.63"
|
|
783
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
784
|
+
checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8"
|
|
785
|
+
dependencies = [
|
|
786
|
+
"zerocopy-derive",
|
|
787
|
+
]
|
|
788
|
+
|
|
789
|
+
[[package]]
|
|
790
|
+
name = "zerocopy-derive"
|
|
791
|
+
version = "0.8.63"
|
|
792
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
793
|
+
checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f"
|
|
794
|
+
dependencies = [
|
|
795
|
+
"proc-macro2",
|
|
796
|
+
"quote",
|
|
797
|
+
"syn 2.0.119",
|
|
798
|
+
]
|
|
799
|
+
|
|
800
|
+
[[package]]
|
|
801
|
+
name = "zmij"
|
|
802
|
+
version = "1.0.23"
|
|
803
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
804
|
+
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "laghu_rs"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
description = "The optional Rust core of laghu: normalization, pre-tokenization, Unigram Viterbi and mean pooling over mapped files"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
repository = "https://github.com/vedicreader/laghu"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
|
|
10
|
+
[lib]
|
|
11
|
+
name = "laghu_rs"
|
|
12
|
+
crate-type = ["cdylib"]
|
|
13
|
+
|
|
14
|
+
[dependencies]
|
|
15
|
+
pyo3 = { version = "=0.29.0", features = ["abi3-py311", "extension-module"] }
|
|
16
|
+
tokenizers = { version = "=0.23.2", default-features = false, features = ["onig"] } # the version of Python `tokenizers` laghu matches, with its regex engine
|
|
17
|
+
|
|
18
|
+
[profile.release]
|
|
19
|
+
opt-level = 3
|
|
20
|
+
lto = true
|
|
21
|
+
codegen-units = 1
|
laghu_rs-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: laghu-rs
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Classifier: Programming Language :: Python :: 3
|
|
5
|
+
Classifier: Programming Language :: Rust
|
|
6
|
+
Summary: The optional Rust core of laghu: normalization, pre-tokenization, Unigram Viterbi and mean pooling over mapped files
|
|
7
|
+
Author-email: Karthik <karthik.rajgopal@hotmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
11
|
+
Project-URL: Repository, https://github.com/vedicreader/laghu
|
|
12
|
+
|
|
13
|
+
# laghu-rs
|
|
14
|
+
|
|
15
|
+
The optional Rust core of [laghu](https://github.com/vedicreader/laghu). laghu loads model2vec static embedding models with far less memory than model2vec and gives the same vectors. With laghu-rs installed, laghu runs normalization, pre-tokenization, the Unigram Viterbi search and mean pooling in Rust threads, over the same memory-mapped files.
|
|
16
|
+
|
|
17
|
+
``` sh
|
|
18
|
+
pip install laghu-rs # or: pip install 'laghu[rust]'
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Nothing else changes: laghu finds laghu-rs and uses it for the models it supports. Output is identical, bit for bit, to laghu's pure-Python path and to model2vec 0.9.
|
|
22
|
+
|
|
23
|
+
On potion-multilingual-128M, 500-character chunks on 14 cores of an M3 Max:
|
|
24
|
+
|
|
25
|
+
| path | chunks/s | private memory |
|
|
26
|
+
|---|---|---|
|
|
27
|
+
| laghu with laghu-rs | 39,750 | 64 MB |
|
|
28
|
+
| laghu, pure Python | 1,736 | 67 MB |
|
|
29
|
+
| model2vec 0.9 | 5,276 | 1,360 MB |
|
|
30
|
+
|
|
31
|
+
laghu falls back to pure Python when laghu-rs is not installed or `LAGHU_RUST=0` is set. It keeps Python's normalization and pre-tokenization when the `tokenizers` crate inside laghu-rs is not the same version as the installed Python `tokenizers`, or when the two disagree on a set of probe texts.
|
|
32
|
+
|
|
33
|
+
laghu-rs needs Python 3.11 or later. Building it from source needs Rust and a C compiler.
|
|
34
|
+
|
laghu_rs-0.1.0/README.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# laghu-rs
|
|
2
|
+
|
|
3
|
+
The optional Rust core of [laghu](https://github.com/vedicreader/laghu). laghu loads model2vec static embedding models with far less memory than model2vec and gives the same vectors. With laghu-rs installed, laghu runs normalization, pre-tokenization, the Unigram Viterbi search and mean pooling in Rust threads, over the same memory-mapped files.
|
|
4
|
+
|
|
5
|
+
``` sh
|
|
6
|
+
pip install laghu-rs # or: pip install 'laghu[rust]'
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Nothing else changes: laghu finds laghu-rs and uses it for the models it supports. Output is identical, bit for bit, to laghu's pure-Python path and to model2vec 0.9.
|
|
10
|
+
|
|
11
|
+
On potion-multilingual-128M, 500-character chunks on 14 cores of an M3 Max:
|
|
12
|
+
|
|
13
|
+
| path | chunks/s | private memory |
|
|
14
|
+
|---|---|---|
|
|
15
|
+
| laghu with laghu-rs | 39,750 | 64 MB |
|
|
16
|
+
| laghu, pure Python | 1,736 | 67 MB |
|
|
17
|
+
| model2vec 0.9 | 5,276 | 1,360 MB |
|
|
18
|
+
|
|
19
|
+
laghu falls back to pure Python when laghu-rs is not installed or `LAGHU_RUST=0` is set. It keeps Python's normalization and pre-tokenization when the `tokenizers` crate inside laghu-rs is not the same version as the installed Python `tokenizers`, or when the two disagree on a set of probe texts.
|
|
20
|
+
|
|
21
|
+
laghu-rs needs Python 3.11 or later. Building it from source needs Rust and a C compiler.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["maturin>=1.9,<2"]
|
|
3
|
+
build-backend = "maturin"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "laghu-rs"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "The optional Rust core of laghu: normalization, pre-tokenization, Unigram Viterbi and mean pooling over mapped files"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{name = "Karthik", email = "karthik.rajgopal@hotmail.com"}]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Programming Language :: Rust",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
[project.urls]
|
|
19
|
+
Repository = "https://github.com/vedicreader/laghu"
|
|
20
|
+
|
|
21
|
+
[tool.maturin]
|
|
22
|
+
module-name = "laghu_rs"
|
laghu_rs-0.1.0/smoke.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"Check an installed laghu-rs wheel: its `tokenizers` version, and laghu's Rust ids against `tokenizers` on a tiny Unigram model."
|
|
2
|
+
import os, sys, tempfile
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import laghu_rs, tokenizers
|
|
5
|
+
from tokenizers import Tokenizer, AddedToken, models, normalizers, pre_tokenizers
|
|
6
|
+
|
|
7
|
+
assert laghu_rs.TOKENIZERS == tokenizers.__version__, (laghu_rs.TOKENIZERS, tokenizers.__version__)
|
|
8
|
+
|
|
9
|
+
pieces = ['<unk>', '▁', '▁the', '▁cat', '▁sat', 'the', 'cat', 's', 'a', 't', 'c', 'h', 'e', '▁on', 'on', 'o', 'n', 'm']
|
|
10
|
+
vocab = [(p, 0.0 if i == 0 else -1.0 - len(p) / 10 - i / 100) for i, p in enumerate(pieces)]
|
|
11
|
+
tok = Tokenizer(models.Unigram(vocab, unk_id=0, byte_fallback=False))
|
|
12
|
+
tok.normalizer = normalizers.NFKC()
|
|
13
|
+
tok.pre_tokenizer = pre_tokenizers.Metaspace()
|
|
14
|
+
tok.add_special_tokens([AddedToken('<mask>', normalized=False)])
|
|
15
|
+
texts = ['the cat sat on the mat', 'Thé cät <mask> sat', 'fine spaces\tand\nlines', '', '<mask><mask> x', 'cats on mats ' * 20]
|
|
16
|
+
|
|
17
|
+
with tempfile.TemporaryDirectory() as d:
|
|
18
|
+
os.environ['LAGHU_CACHE'] = str(Path(d) / 'cache')
|
|
19
|
+
fn = Path(d) / 'tokenizer.json'; tok.save(str(fn))
|
|
20
|
+
from laghu.unigram import Unigram
|
|
21
|
+
u = Unigram(fn)
|
|
22
|
+
assert u.rs is not None and u.rs_pre is not None, (u.rs, u.rs_pre)
|
|
23
|
+
got, want = u(texts), [e.ids for e in tok.encode_batch(texts, add_special_tokens=False)]
|
|
24
|
+
assert got == want, (got, want)
|
|
25
|
+
del u
|
|
26
|
+
print(f'laghu_rs ok: tokenizers {tokenizers.__version__}, {len(texts)} texts match on {sys.platform}')
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
//! The optional Rust core of laghu: Unigram Viterbi over byte offsets, searching a CSR byte trie mapped from laghu's
|
|
2
|
+
//! vocabulary file, and mean pooling over a mapped float32 table, batched over threads with the GIL released.
|
|
3
|
+
//! `Pre` normalizes, splits out added tokens and pre-tokenizes with the `tokenizers` crate, the version Python `tokenizers` has,
|
|
4
|
+
//! so whole texts can go to the threads; every result must equal laghu's Python path, and model2vec's, bit for bit.
|
|
5
|
+
use pyo3::buffer::{Element, PyBuffer};
|
|
6
|
+
use pyo3::exceptions::PyValueError;
|
|
7
|
+
use pyo3::prelude::*;
|
|
8
|
+
use pyo3::types::{PyList, PyString};
|
|
9
|
+
use std::collections::HashMap;
|
|
10
|
+
use std::str::FromStr;
|
|
11
|
+
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
12
|
+
use tokenizers::{OffsetReferential, OffsetType, PreTokenizer, Tokenizer};
|
|
13
|
+
|
|
14
|
+
/// The `tokenizers` version pinned in Cargo.toml, which Python `tokenizers` must also be for `Pre` to be used.
|
|
15
|
+
const TOKENIZERS: &str = "0.23.2";
|
|
16
|
+
|
|
17
|
+
const NONE: u32 = u32::MAX;
|
|
18
|
+
|
|
19
|
+
/// A one-dimensional C-contiguous buffer held for as long as the slice over it is used.
|
|
20
|
+
struct Arr<T: Element> { _buf: PyBuffer<T>, ptr: usize, len: usize }
|
|
21
|
+
|
|
22
|
+
impl<T: Element> Arr<T> {
|
|
23
|
+
fn get(o: &Bound<'_, PyAny>, name: &str) -> PyResult<Arr<T>> {
|
|
24
|
+
let b = PyBuffer::<T>::get(o)?;
|
|
25
|
+
if b.dimensions() != 1 || !b.is_c_contiguous() { return Err(PyValueError::new_err(format!("{name}: not a 1-D contiguous buffer"))); }
|
|
26
|
+
let (ptr, len) = (b.buf_ptr() as usize, b.item_count());
|
|
27
|
+
Ok(Arr { _buf: b, ptr, len })
|
|
28
|
+
}
|
|
29
|
+
#[inline(always)]
|
|
30
|
+
fn s(&self) -> &[T] { unsafe { std::slice::from_raw_parts(self.ptr as *const T, self.len) } }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
#[derive(IntoPyObject)]
|
|
34
|
+
enum Item { W(String), I(i64) }
|
|
35
|
+
|
|
36
|
+
enum Part<'a> { W(&'a str), I(i64) }
|
|
37
|
+
|
|
38
|
+
/// A text's normalization, added-token split and pre-tokenization, done as `TokenizerImpl::encode_single_sequence` does them,
|
|
39
|
+
/// by a `Tokenizer` whose model is a stub: `real` takes the stub's added-token ids to the ids in tokenizer.json.
|
|
40
|
+
#[pyclass(frozen)]
|
|
41
|
+
struct Pre { tok: Tokenizer, real: HashMap<u32, i64> }
|
|
42
|
+
|
|
43
|
+
impl Pre {
|
|
44
|
+
/// `f` on each pre-token and each added token's id of `s`, in order.
|
|
45
|
+
fn each(&self, s: &str, mut f: impl FnMut(Part) -> Result<(), String>) -> Result<(), String> {
|
|
46
|
+
let mut pts = self.tok.get_added_vocabulary().extract_and_normalize(self.tok.get_normalizer(), s);
|
|
47
|
+
if let Some(p) = self.tok.get_pre_tokenizer() { p.pre_tokenize(&mut pts).map_err(|e| e.to_string())?; }
|
|
48
|
+
for (w, _, toks) in pts.get_splits(OffsetReferential::Normalized, OffsetType::None) {
|
|
49
|
+
match toks {
|
|
50
|
+
Some(ts) => for t in ts { f(Part::I(*self.real.get(&t.id).ok_or_else(|| format!("unknown added token id {}", t.id))?))?; },
|
|
51
|
+
None => f(Part::W(w))?,
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
Ok(())
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
#[pymethods]
|
|
59
|
+
impl Pre {
|
|
60
|
+
/// Build from tokenizer JSON `json` whose model is a stub, with `added` mapping each added token's content to its id.
|
|
61
|
+
#[new]
|
|
62
|
+
fn new(json: &str, added: HashMap<String, i64>) -> PyResult<Pre> {
|
|
63
|
+
let tok = Tokenizer::from_str(json).map_err(|e| PyValueError::new_err(e.to_string()))?;
|
|
64
|
+
let mut real = HashMap::new();
|
|
65
|
+
for (c, k) in added {
|
|
66
|
+
let s = tok.token_to_id(&c).ok_or_else(|| PyValueError::new_err(format!("added token {c:?} missing")))?;
|
|
67
|
+
real.insert(s, k);
|
|
68
|
+
}
|
|
69
|
+
Ok(Pre { tok, real })
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/// Text `s` as the Viterbi search receives it, as laghu's `Unigram.split`: pre-tokens (str) and added-token ids (int).
|
|
73
|
+
fn split(&self, py: Python<'_>, s: &str) -> PyResult<Vec<Item>> {
|
|
74
|
+
let mut out = Vec::new();
|
|
75
|
+
py.detach(|| self.each(s, |p| { out.push(match p { Part::W(w) => Item::W(w.to_owned()), Part::I(k) => Item::I(k) }); Ok(()) }))
|
|
76
|
+
.map_err(PyValueError::new_err)?;
|
|
77
|
+
Ok(out)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// The texts of a batch: each already split by Python, or whole, for `Pre` to split in the thread that segments it.
|
|
82
|
+
enum Texts<'a> { Split(Vec<Vec<Item>>), Whole(Vec<String>, &'a Pre) }
|
|
83
|
+
|
|
84
|
+
impl Texts<'_> {
|
|
85
|
+
fn len(&self) -> usize { match self { Texts::Split(v) => v.len(), Texts::Whole(v, _) => v.len() } }
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
#[inline(always)]
|
|
89
|
+
fn utf8_len(b: u8) -> usize { if b < 0x80 { 1 } else if b < 0xE0 { 2 } else if b < 0xF0 { 3 } else { 4 } }
|
|
90
|
+
|
|
91
|
+
/// A Unigram segmenter over the mapped vocabulary: the byte trie (edge starts per node, edge bytes sorted within a node,
|
|
92
|
+
/// child nodes, node values) whose values index the ids and float64 scores.
|
|
93
|
+
#[pyclass(frozen)]
|
|
94
|
+
struct Seg {
|
|
95
|
+
start: Arr<u32>, ebyte: Arr<u8>, echild: Arr<u32>, val: Arr<u32>, ids: Arr<i32>, scores: Arr<f64>,
|
|
96
|
+
unk_id: i64, unk_score: f64, byte_fallback: bool, byte_ids: [i64; 256],
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
impl Seg {
|
|
100
|
+
#[inline(always)]
|
|
101
|
+
fn child(&self, node: u32, b: u8) -> u32 {
|
|
102
|
+
let st = self.start.s();
|
|
103
|
+
let (lo, hi) = (st[node as usize] as usize, st[node as usize + 1] as usize);
|
|
104
|
+
let s = &self.ebyte.s()[lo..hi];
|
|
105
|
+
if s.len() <= 8 {
|
|
106
|
+
for (i, &x) in s.iter().enumerate() { if x == b { return self.echild.s()[lo + i]; } }
|
|
107
|
+
NONE
|
|
108
|
+
} else {
|
|
109
|
+
match s.binary_search(&b) { Ok(i) => self.echild.s()[lo + i], Err(_) => NONE }
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
fn get(&self, key: &[u8]) -> u32 {
|
|
114
|
+
let mut node = 0u32;
|
|
115
|
+
for &b in key { node = self.child(node, b); if node == NONE { return NONE; } }
|
|
116
|
+
self.val.s()[node as usize]
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/// The ids of an unknown run, as laghu's `_unk`: a piece if it spells one, else its bytes under byte fallback, else the unknown id.
|
|
120
|
+
fn unk(&self, p: &str) -> Result<Vec<i64>, String> {
|
|
121
|
+
let k = self.get(p.as_bytes());
|
|
122
|
+
if k != NONE { return Ok(vec![self.ids.s()[k as usize] as i64]); }
|
|
123
|
+
if self.byte_fallback {
|
|
124
|
+
let v: Option<Vec<i64>> = p.bytes().map(|x| { let q = self.byte_ids[x as usize]; if q >= 0 { Some(q) } else { None } }).collect();
|
|
125
|
+
if let Some(v) = v { return Ok(v); }
|
|
126
|
+
}
|
|
127
|
+
if self.unk_id < 0 { return Err(format!("Encountered an unknown token but `unk_id` is missing: {:?}", p)); }
|
|
128
|
+
Ok(vec![self.unk_id])
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/// laghu's `viterbi_trie` and `_path` on byte offsets: candidates by start ascending, then length ascending, kept on a strict `>`.
|
|
132
|
+
fn viterbi(&self, s: &str, out: &mut Vec<i64>) -> Result<(), String> {
|
|
133
|
+
let b = s.as_bytes();
|
|
134
|
+
let n = b.len();
|
|
135
|
+
if n == 0 { return Ok(()); }
|
|
136
|
+
let (ids, scores, val) = (self.ids.s(), self.scores.s(), self.val.s());
|
|
137
|
+
let mut best = vec![0f64; n + 1];
|
|
138
|
+
let mut set = vec![false; n + 1];
|
|
139
|
+
let mut start = vec![0usize; n + 1];
|
|
140
|
+
let mut tid = vec![0i64; n + 1];
|
|
141
|
+
set[0] = true;
|
|
142
|
+
let mut i = 0;
|
|
143
|
+
while i < n {
|
|
144
|
+
let cl = utf8_len(b[i]);
|
|
145
|
+
let bi = best[i];
|
|
146
|
+
let mut single = false;
|
|
147
|
+
let (mut node, mut j) = (0u32, i);
|
|
148
|
+
while j < n {
|
|
149
|
+
node = self.child(node, b[j]);
|
|
150
|
+
if node == NONE { break; }
|
|
151
|
+
j += 1;
|
|
152
|
+
let v = val[node as usize];
|
|
153
|
+
if v != NONE {
|
|
154
|
+
let c = scores[v as usize] + bi;
|
|
155
|
+
if !set[j] || c > best[j] { best[j] = c; set[j] = true; start[j] = i; tid[j] = ids[v as usize] as i64; }
|
|
156
|
+
if j == i + cl { single = true; }
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if !single {
|
|
160
|
+
let j = i + cl;
|
|
161
|
+
let c = self.unk_score + bi;
|
|
162
|
+
if !set[j] || c > best[j] { best[j] = c; set[j] = true; start[j] = i; tid[j] = self.unk_id; }
|
|
163
|
+
}
|
|
164
|
+
i += cl;
|
|
165
|
+
}
|
|
166
|
+
let mut rev: Vec<i64> = Vec::new();
|
|
167
|
+
let (mut e, mut run) = (n, 0usize);
|
|
168
|
+
while e > 0 {
|
|
169
|
+
if tid[e] == self.unk_id { if run == 0 { run = e; } }
|
|
170
|
+
else {
|
|
171
|
+
if run != 0 { let u = self.unk(&s[e..run])?; rev.extend(u.iter().rev()); run = 0; }
|
|
172
|
+
rev.push(tid[e]);
|
|
173
|
+
}
|
|
174
|
+
e = start[e];
|
|
175
|
+
if run != 0 && e == 0 { let u = self.unk(&s[..run])?; rev.extend(u.iter().rev()); }
|
|
176
|
+
}
|
|
177
|
+
out.extend(rev.iter().rev());
|
|
178
|
+
Ok(())
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/// The ids of text `i` of `texts`, id `drop` removed and cut to `max_len`.
|
|
182
|
+
fn text(&self, texts: &Texts, i: usize, drop: Option<i64>, max_len: Option<usize>) -> Result<Vec<i64>, String> {
|
|
183
|
+
let mut out = Vec::new();
|
|
184
|
+
match texts {
|
|
185
|
+
Texts::Split(v) => for it in &v[i] { match it { Item::W(s) => self.viterbi(s, &mut out)?, Item::I(k) => out.push(*k) } },
|
|
186
|
+
Texts::Whole(v, pre) => pre.each(&v[i], |p| match p { Part::W(w) => self.viterbi(w, &mut out), Part::I(k) => { out.push(k); Ok(()) } })?,
|
|
187
|
+
}
|
|
188
|
+
if let Some(d) = drop { out.retain(|&k| k != d); }
|
|
189
|
+
if let Some(m) = max_len { out.truncate(m); }
|
|
190
|
+
Ok(out)
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/// `texts` as whole strings when `pre` is given, else as lists of pre-tokens (str) and added-token ids (int).
|
|
195
|
+
fn extract<'a>(texts: &Bound<'_, PyList>, pre: Option<&'a Pre>) -> PyResult<Texts<'a>> {
|
|
196
|
+
if let Some(p) = pre { return Ok(Texts::Whole(texts.extract()?, p)); }
|
|
197
|
+
let mut all = Vec::with_capacity(texts.len());
|
|
198
|
+
for t in texts.iter() {
|
|
199
|
+
let t = t.cast_into::<PyList>()?;
|
|
200
|
+
let mut v = Vec::with_capacity(t.len());
|
|
201
|
+
for x in t.iter() {
|
|
202
|
+
if x.is_instance_of::<PyString>() { v.push(Item::W(x.extract::<String>()?)); } else { v.push(Item::I(x.extract::<i64>()?)); }
|
|
203
|
+
}
|
|
204
|
+
all.push(v);
|
|
205
|
+
}
|
|
206
|
+
Ok(Texts::Split(all))
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/// Map `f` over `n` indices on `threads` threads, results in index order.
|
|
210
|
+
fn par_map<R: Send>(n: usize, threads: usize, f: impl Fn(usize) -> R + Sync) -> Vec<R> {
|
|
211
|
+
if threads <= 1 || n < 2 { return (0..n).map(f).collect(); }
|
|
212
|
+
let next = AtomicUsize::new(0);
|
|
213
|
+
let mut parts: Vec<Vec<(usize, R)>> = Vec::new();
|
|
214
|
+
std::thread::scope(|sc| {
|
|
215
|
+
let hs: Vec<_> = (0..threads.min(n)).map(|_| sc.spawn(|| {
|
|
216
|
+
let mut got = Vec::new();
|
|
217
|
+
loop {
|
|
218
|
+
let a = next.fetch_add(8, Ordering::Relaxed);
|
|
219
|
+
if a >= n { break; }
|
|
220
|
+
for i in a..(a + 8).min(n) { got.push((i, f(i))); }
|
|
221
|
+
}
|
|
222
|
+
got
|
|
223
|
+
})).collect();
|
|
224
|
+
for h in hs { parts.push(h.join().unwrap()); }
|
|
225
|
+
});
|
|
226
|
+
let mut out: Vec<Option<R>> = (0..n).map(|_| None).collect();
|
|
227
|
+
for p in parts { for (i, r) in p { out[i] = Some(r); } }
|
|
228
|
+
out.into_iter().map(|o| o.unwrap()).collect()
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
#[pymethods]
|
|
232
|
+
impl Seg {
|
|
233
|
+
/// Segment with the byte trie in buffers `start` (u32), `ebyte` (u8), `echild` (u32), `val` (u32) over `ids` (i32) and `scores` (f64).
|
|
234
|
+
#[new]
|
|
235
|
+
#[pyo3(signature = (start, ebyte, echild, val, ids, scores, unk_id, unk_score, byte_fallback))]
|
|
236
|
+
fn new(
|
|
237
|
+
start: &Bound<'_, PyAny>, ebyte: &Bound<'_, PyAny>, echild: &Bound<'_, PyAny>, val: &Bound<'_, PyAny>,
|
|
238
|
+
ids: &Bound<'_, PyAny>, scores: &Bound<'_, PyAny>, unk_id: i64, unk_score: f64, byte_fallback: bool,
|
|
239
|
+
) -> PyResult<Seg> {
|
|
240
|
+
let mut s = Seg {
|
|
241
|
+
start: Arr::get(start, "start")?, ebyte: Arr::get(ebyte, "ebyte")?, echild: Arr::get(echild, "echild")?,
|
|
242
|
+
val: Arr::get(val, "val")?, ids: Arr::get(ids, "ids")?, scores: Arr::get(scores, "scores")?,
|
|
243
|
+
unk_id, unk_score, byte_fallback, byte_ids: [-1; 256],
|
|
244
|
+
};
|
|
245
|
+
let (nodes, edges, nk) = (s.val.len, s.ebyte.len, s.ids.len);
|
|
246
|
+
let bad = |m: &str| Err(PyValueError::new_err(format!("malformed byte trie: {m}")));
|
|
247
|
+
if nodes == 0 || s.start.len != nodes + 1 || s.echild.len != edges || s.scores.len != nk { return bad("array lengths"); }
|
|
248
|
+
let st = s.start.s();
|
|
249
|
+
if st[0] != 0 || st[nodes] as usize != edges || st.windows(2).any(|w| w[0] > w[1]) { return bad("edge starts"); }
|
|
250
|
+
if s.echild.s().iter().any(|&c| c == 0 || c as usize >= nodes) { return bad("child nodes"); }
|
|
251
|
+
if s.val.s().iter().any(|&v| v != NONE && v as usize >= nk) { return bad("node values"); }
|
|
252
|
+
for b in 0..256usize {
|
|
253
|
+
let k = s.get(format!("<0x{:02X}>", b).as_bytes());
|
|
254
|
+
if k != NONE { s.byte_ids[b] = s.ids.s()[k as usize] as i64; }
|
|
255
|
+
}
|
|
256
|
+
Ok(s)
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/// The ids of one pre-token, as laghu's `viterbi_trie`.
|
|
260
|
+
fn word(&self, py: Python<'_>, s: &str) -> PyResult<Vec<i64>> {
|
|
261
|
+
let mut out = Vec::new();
|
|
262
|
+
py.detach(|| self.viterbi(s, &mut out)).map_err(PyValueError::new_err)?;
|
|
263
|
+
Ok(out)
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/// The ids of each text, id `drop` removed and cut to `max_len`. A text is a str split by `pre` when it is given,
|
|
267
|
+
/// else a list of pre-tokens (str) and added-token ids (int).
|
|
268
|
+
#[pyo3(signature = (texts, drop=None, max_len=None, threads=1, pre=None))]
|
|
269
|
+
fn batch(&self, py: Python<'_>, texts: &Bound<'_, PyList>, drop: Option<i64>, max_len: Option<usize>, threads: usize,
|
|
270
|
+
pre: Option<Bound<'_, Pre>>) -> PyResult<Vec<Vec<i64>>> {
|
|
271
|
+
let items = extract(texts, pre.as_ref().map(|p| p.get()))?;
|
|
272
|
+
let r = py.detach(|| par_map(items.len(), threads, |i| self.text(&items, i, drop, max_len)));
|
|
273
|
+
r.into_iter().collect::<Result<Vec<_>, _>>().map_err(PyValueError::new_err)
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/// `batch`, then the mean of each text's rows of float32 `table` (rows x dim, C order) written into float32 `out`
|
|
277
|
+
/// (texts x dim), as numpy's `mean(axis=0)`: the first row copied, the rest added in order, divided by the count. Returns the counts.
|
|
278
|
+
#[pyo3(signature = (texts, table, out, drop=None, max_len=None, threads=1, pre=None))]
|
|
279
|
+
fn encode(&self, py: Python<'_>, texts: &Bound<'_, PyList>, table: &Bound<'_, PyAny>, out: &Bound<'_, PyAny>,
|
|
280
|
+
drop: Option<i64>, max_len: Option<usize>, threads: usize, pre: Option<Bound<'_, Pre>>) -> PyResult<Vec<usize>> {
|
|
281
|
+
let items = extract(texts, pre.as_ref().map(|p| p.get()))?;
|
|
282
|
+
let tb = PyBuffer::<f32>::get(table)?;
|
|
283
|
+
let ob = PyBuffer::<f32>::get(out)?;
|
|
284
|
+
if tb.dimensions() != 2 || !tb.is_c_contiguous() { return Err(PyValueError::new_err("table must be 2-D C-contiguous float32")); }
|
|
285
|
+
let (rows, dim) = (tb.shape()[0], tb.shape()[1]);
|
|
286
|
+
if ob.readonly() || !ob.is_c_contiguous() || ob.item_count() != items.len() * dim { return Err(PyValueError::new_err("out must be writable float32 (texts, dim)")); }
|
|
287
|
+
let (tp, op) = (tb.buf_ptr() as usize, ob.buf_ptr() as usize);
|
|
288
|
+
let r = py.detach(|| par_map(items.len(), threads, |i| -> Result<usize, String> {
|
|
289
|
+
let ids = self.text(&items, i, drop, max_len)?;
|
|
290
|
+
let table = unsafe { std::slice::from_raw_parts(tp as *const f32, rows * dim) };
|
|
291
|
+
let o = unsafe { std::slice::from_raw_parts_mut((op as *mut f32).add(i * dim), dim) };
|
|
292
|
+
if ids.is_empty() { o.fill(0.0); return Ok(0); }
|
|
293
|
+
for &k in &ids { if k < 0 || k as usize >= rows { return Err(format!("id {} out of range", k)); } }
|
|
294
|
+
o.copy_from_slice(&table[ids[0] as usize * dim..(ids[0] as usize + 1) * dim]);
|
|
295
|
+
for &k in &ids[1..] {
|
|
296
|
+
let row = &table[k as usize * dim..(k as usize + 1) * dim];
|
|
297
|
+
for d in 0..dim { o[d] += row[d]; }
|
|
298
|
+
}
|
|
299
|
+
let nf = ids.len() as f32;
|
|
300
|
+
for d in 0..dim { o[d] /= nf; }
|
|
301
|
+
Ok(ids.len())
|
|
302
|
+
}));
|
|
303
|
+
r.into_iter().collect::<Result<Vec<_>, _>>().map_err(PyValueError::new_err)
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
#[pymodule]
|
|
308
|
+
fn laghu_rs(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
309
|
+
m.add_class::<Seg>()?;
|
|
310
|
+
m.add_class::<Pre>()?;
|
|
311
|
+
m.add("TOKENIZERS", TOKENIZERS)?;
|
|
312
|
+
Ok(())
|
|
313
|
+
}
|