dimq-load-task 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dimq_load_task-0.1.1/Cargo.lock +710 -0
- dimq_load_task-0.1.1/Cargo.toml +18 -0
- dimq_load_task-0.1.1/PKG-INFO +243 -0
- dimq_load_task-0.1.1/README.md +227 -0
- dimq_load_task-0.1.1/pyproject.toml +25 -0
- dimq_load_task-0.1.1/src/lib.rs +104 -0
|
@@ -0,0 +1,710 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "anyhow"
|
|
7
|
+
version = "1.0.102"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
|
|
10
|
+
|
|
11
|
+
[[package]]
|
|
12
|
+
name = "autocfg"
|
|
13
|
+
version = "1.5.0"
|
|
14
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
15
|
+
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
|
16
|
+
|
|
17
|
+
[[package]]
|
|
18
|
+
name = "bitflags"
|
|
19
|
+
version = "2.11.0"
|
|
20
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
21
|
+
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
|
|
22
|
+
|
|
23
|
+
[[package]]
|
|
24
|
+
name = "block-buffer"
|
|
25
|
+
version = "0.10.4"
|
|
26
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
27
|
+
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
|
28
|
+
dependencies = [
|
|
29
|
+
"generic-array",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[[package]]
|
|
33
|
+
name = "cc"
|
|
34
|
+
version = "1.2.56"
|
|
35
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
36
|
+
checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
|
|
37
|
+
dependencies = [
|
|
38
|
+
"find-msvc-tools",
|
|
39
|
+
"shlex",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
[[package]]
|
|
43
|
+
name = "cfg-if"
|
|
44
|
+
version = "1.0.4"
|
|
45
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
46
|
+
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
|
47
|
+
|
|
48
|
+
[[package]]
|
|
49
|
+
name = "cpufeatures"
|
|
50
|
+
version = "0.2.17"
|
|
51
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
52
|
+
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
|
53
|
+
dependencies = [
|
|
54
|
+
"libc",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[[package]]
|
|
58
|
+
name = "crossbeam-deque"
|
|
59
|
+
version = "0.8.6"
|
|
60
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
61
|
+
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
|
|
62
|
+
dependencies = [
|
|
63
|
+
"crossbeam-epoch",
|
|
64
|
+
"crossbeam-utils",
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
[[package]]
|
|
68
|
+
name = "crossbeam-epoch"
|
|
69
|
+
version = "0.9.18"
|
|
70
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
71
|
+
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
|
72
|
+
dependencies = [
|
|
73
|
+
"crossbeam-utils",
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
[[package]]
|
|
77
|
+
name = "crossbeam-utils"
|
|
78
|
+
version = "0.8.21"
|
|
79
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
80
|
+
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
|
81
|
+
|
|
82
|
+
[[package]]
|
|
83
|
+
name = "crypto-common"
|
|
84
|
+
version = "0.1.7"
|
|
85
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
86
|
+
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
|
87
|
+
dependencies = [
|
|
88
|
+
"generic-array",
|
|
89
|
+
"typenum",
|
|
90
|
+
]
|
|
91
|
+
|
|
92
|
+
[[package]]
|
|
93
|
+
name = "digest"
|
|
94
|
+
version = "0.10.7"
|
|
95
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
96
|
+
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|
97
|
+
dependencies = [
|
|
98
|
+
"block-buffer",
|
|
99
|
+
"crypto-common",
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
[[package]]
|
|
103
|
+
name = "dimq_load_task"
|
|
104
|
+
version = "0.1.1"
|
|
105
|
+
dependencies = [
|
|
106
|
+
"pyo3",
|
|
107
|
+
"rayon",
|
|
108
|
+
"sha2",
|
|
109
|
+
"tempfile",
|
|
110
|
+
]
|
|
111
|
+
|
|
112
|
+
[[package]]
|
|
113
|
+
name = "either"
|
|
114
|
+
version = "1.15.0"
|
|
115
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
116
|
+
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
|
117
|
+
|
|
118
|
+
[[package]]
|
|
119
|
+
name = "equivalent"
|
|
120
|
+
version = "1.0.2"
|
|
121
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
122
|
+
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
|
123
|
+
|
|
124
|
+
[[package]]
|
|
125
|
+
name = "errno"
|
|
126
|
+
version = "0.3.14"
|
|
127
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
128
|
+
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
|
129
|
+
dependencies = [
|
|
130
|
+
"libc",
|
|
131
|
+
"windows-sys",
|
|
132
|
+
]
|
|
133
|
+
|
|
134
|
+
[[package]]
|
|
135
|
+
name = "fastrand"
|
|
136
|
+
version = "2.3.0"
|
|
137
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
138
|
+
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
|
139
|
+
|
|
140
|
+
[[package]]
|
|
141
|
+
name = "find-msvc-tools"
|
|
142
|
+
version = "0.1.9"
|
|
143
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
144
|
+
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
|
145
|
+
|
|
146
|
+
[[package]]
|
|
147
|
+
name = "foldhash"
|
|
148
|
+
version = "0.1.5"
|
|
149
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
150
|
+
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
|
151
|
+
|
|
152
|
+
[[package]]
|
|
153
|
+
name = "generic-array"
|
|
154
|
+
version = "0.14.7"
|
|
155
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
156
|
+
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
|
157
|
+
dependencies = [
|
|
158
|
+
"typenum",
|
|
159
|
+
"version_check",
|
|
160
|
+
]
|
|
161
|
+
|
|
162
|
+
[[package]]
|
|
163
|
+
name = "getrandom"
|
|
164
|
+
version = "0.4.2"
|
|
165
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
166
|
+
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
|
|
167
|
+
dependencies = [
|
|
168
|
+
"cfg-if",
|
|
169
|
+
"libc",
|
|
170
|
+
"r-efi",
|
|
171
|
+
"wasip2",
|
|
172
|
+
"wasip3",
|
|
173
|
+
]
|
|
174
|
+
|
|
175
|
+
[[package]]
|
|
176
|
+
name = "hashbrown"
|
|
177
|
+
version = "0.15.5"
|
|
178
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
179
|
+
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
|
|
180
|
+
dependencies = [
|
|
181
|
+
"foldhash",
|
|
182
|
+
]
|
|
183
|
+
|
|
184
|
+
[[package]]
|
|
185
|
+
name = "hashbrown"
|
|
186
|
+
version = "0.16.1"
|
|
187
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
188
|
+
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
|
189
|
+
|
|
190
|
+
[[package]]
|
|
191
|
+
name = "heck"
|
|
192
|
+
version = "0.5.0"
|
|
193
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
194
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
195
|
+
|
|
196
|
+
[[package]]
|
|
197
|
+
name = "id-arena"
|
|
198
|
+
version = "2.3.0"
|
|
199
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
200
|
+
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
|
|
201
|
+
|
|
202
|
+
[[package]]
|
|
203
|
+
name = "indexmap"
|
|
204
|
+
version = "2.13.0"
|
|
205
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
206
|
+
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
|
|
207
|
+
dependencies = [
|
|
208
|
+
"equivalent",
|
|
209
|
+
"hashbrown 0.16.1",
|
|
210
|
+
"serde",
|
|
211
|
+
"serde_core",
|
|
212
|
+
]
|
|
213
|
+
|
|
214
|
+
[[package]]
|
|
215
|
+
name = "indoc"
|
|
216
|
+
version = "2.0.7"
|
|
217
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
218
|
+
checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
|
|
219
|
+
dependencies = [
|
|
220
|
+
"rustversion",
|
|
221
|
+
]
|
|
222
|
+
|
|
223
|
+
[[package]]
|
|
224
|
+
name = "itoa"
|
|
225
|
+
version = "1.0.17"
|
|
226
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
227
|
+
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
|
|
228
|
+
|
|
229
|
+
[[package]]
|
|
230
|
+
name = "leb128fmt"
|
|
231
|
+
version = "0.1.0"
|
|
232
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
233
|
+
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
|
234
|
+
|
|
235
|
+
[[package]]
|
|
236
|
+
name = "libc"
|
|
237
|
+
version = "0.2.182"
|
|
238
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
239
|
+
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
|
|
240
|
+
|
|
241
|
+
[[package]]
|
|
242
|
+
name = "linux-raw-sys"
|
|
243
|
+
version = "0.12.1"
|
|
244
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
245
|
+
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
|
246
|
+
|
|
247
|
+
[[package]]
|
|
248
|
+
name = "log"
|
|
249
|
+
version = "0.4.29"
|
|
250
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
251
|
+
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
|
252
|
+
|
|
253
|
+
[[package]]
|
|
254
|
+
name = "memchr"
|
|
255
|
+
version = "2.8.0"
|
|
256
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
257
|
+
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
|
258
|
+
|
|
259
|
+
[[package]]
|
|
260
|
+
name = "memoffset"
|
|
261
|
+
version = "0.9.1"
|
|
262
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
263
|
+
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
|
|
264
|
+
dependencies = [
|
|
265
|
+
"autocfg",
|
|
266
|
+
]
|
|
267
|
+
|
|
268
|
+
[[package]]
|
|
269
|
+
name = "once_cell"
|
|
270
|
+
version = "1.21.3"
|
|
271
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
272
|
+
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
|
273
|
+
|
|
274
|
+
[[package]]
|
|
275
|
+
name = "portable-atomic"
|
|
276
|
+
version = "1.13.1"
|
|
277
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
278
|
+
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
|
|
279
|
+
|
|
280
|
+
[[package]]
|
|
281
|
+
name = "prettyplease"
|
|
282
|
+
version = "0.2.37"
|
|
283
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
284
|
+
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
|
|
285
|
+
dependencies = [
|
|
286
|
+
"proc-macro2",
|
|
287
|
+
"syn",
|
|
288
|
+
]
|
|
289
|
+
|
|
290
|
+
[[package]]
|
|
291
|
+
name = "proc-macro2"
|
|
292
|
+
version = "1.0.106"
|
|
293
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
294
|
+
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
|
295
|
+
dependencies = [
|
|
296
|
+
"unicode-ident",
|
|
297
|
+
]
|
|
298
|
+
|
|
299
|
+
[[package]]
|
|
300
|
+
name = "pyo3"
|
|
301
|
+
version = "0.22.6"
|
|
302
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
303
|
+
checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
|
|
304
|
+
dependencies = [
|
|
305
|
+
"cfg-if",
|
|
306
|
+
"indoc",
|
|
307
|
+
"libc",
|
|
308
|
+
"memoffset",
|
|
309
|
+
"once_cell",
|
|
310
|
+
"portable-atomic",
|
|
311
|
+
"pyo3-build-config",
|
|
312
|
+
"pyo3-ffi",
|
|
313
|
+
"pyo3-macros",
|
|
314
|
+
"unindent",
|
|
315
|
+
]
|
|
316
|
+
|
|
317
|
+
[[package]]
|
|
318
|
+
name = "pyo3-build-config"
|
|
319
|
+
version = "0.22.6"
|
|
320
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
321
|
+
checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38"
|
|
322
|
+
dependencies = [
|
|
323
|
+
"once_cell",
|
|
324
|
+
"python3-dll-a",
|
|
325
|
+
"target-lexicon",
|
|
326
|
+
]
|
|
327
|
+
|
|
328
|
+
[[package]]
|
|
329
|
+
name = "pyo3-ffi"
|
|
330
|
+
version = "0.22.6"
|
|
331
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
332
|
+
checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636"
|
|
333
|
+
dependencies = [
|
|
334
|
+
"libc",
|
|
335
|
+
"pyo3-build-config",
|
|
336
|
+
]
|
|
337
|
+
|
|
338
|
+
[[package]]
|
|
339
|
+
name = "pyo3-macros"
|
|
340
|
+
version = "0.22.6"
|
|
341
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
342
|
+
checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453"
|
|
343
|
+
dependencies = [
|
|
344
|
+
"proc-macro2",
|
|
345
|
+
"pyo3-macros-backend",
|
|
346
|
+
"quote",
|
|
347
|
+
"syn",
|
|
348
|
+
]
|
|
349
|
+
|
|
350
|
+
[[package]]
|
|
351
|
+
name = "pyo3-macros-backend"
|
|
352
|
+
version = "0.22.6"
|
|
353
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
354
|
+
checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe"
|
|
355
|
+
dependencies = [
|
|
356
|
+
"heck",
|
|
357
|
+
"proc-macro2",
|
|
358
|
+
"pyo3-build-config",
|
|
359
|
+
"quote",
|
|
360
|
+
"syn",
|
|
361
|
+
]
|
|
362
|
+
|
|
363
|
+
[[package]]
|
|
364
|
+
name = "python3-dll-a"
|
|
365
|
+
version = "0.2.14"
|
|
366
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
367
|
+
checksum = "d381ef313ae70b4da5f95f8a4de773c6aa5cd28f73adec4b4a31df70b66780d8"
|
|
368
|
+
dependencies = [
|
|
369
|
+
"cc",
|
|
370
|
+
]
|
|
371
|
+
|
|
372
|
+
[[package]]
|
|
373
|
+
name = "quote"
|
|
374
|
+
version = "1.0.45"
|
|
375
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
376
|
+
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
|
377
|
+
dependencies = [
|
|
378
|
+
"proc-macro2",
|
|
379
|
+
]
|
|
380
|
+
|
|
381
|
+
[[package]]
|
|
382
|
+
name = "r-efi"
|
|
383
|
+
version = "6.0.0"
|
|
384
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
385
|
+
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
|
386
|
+
|
|
387
|
+
[[package]]
|
|
388
|
+
name = "rayon"
|
|
389
|
+
version = "1.11.0"
|
|
390
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
391
|
+
checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
|
|
392
|
+
dependencies = [
|
|
393
|
+
"either",
|
|
394
|
+
"rayon-core",
|
|
395
|
+
]
|
|
396
|
+
|
|
397
|
+
[[package]]
|
|
398
|
+
name = "rayon-core"
|
|
399
|
+
version = "1.13.0"
|
|
400
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
401
|
+
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
|
402
|
+
dependencies = [
|
|
403
|
+
"crossbeam-deque",
|
|
404
|
+
"crossbeam-utils",
|
|
405
|
+
]
|
|
406
|
+
|
|
407
|
+
[[package]]
|
|
408
|
+
name = "rustix"
|
|
409
|
+
version = "1.1.4"
|
|
410
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
411
|
+
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
|
412
|
+
dependencies = [
|
|
413
|
+
"bitflags",
|
|
414
|
+
"errno",
|
|
415
|
+
"libc",
|
|
416
|
+
"linux-raw-sys",
|
|
417
|
+
"windows-sys",
|
|
418
|
+
]
|
|
419
|
+
|
|
420
|
+
[[package]]
|
|
421
|
+
name = "rustversion"
|
|
422
|
+
version = "1.0.22"
|
|
423
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
424
|
+
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
|
425
|
+
|
|
426
|
+
[[package]]
|
|
427
|
+
name = "semver"
|
|
428
|
+
version = "1.0.27"
|
|
429
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
430
|
+
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
|
|
431
|
+
|
|
432
|
+
[[package]]
|
|
433
|
+
name = "serde"
|
|
434
|
+
version = "1.0.228"
|
|
435
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
436
|
+
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
|
437
|
+
dependencies = [
|
|
438
|
+
"serde_core",
|
|
439
|
+
]
|
|
440
|
+
|
|
441
|
+
[[package]]
|
|
442
|
+
name = "serde_core"
|
|
443
|
+
version = "1.0.228"
|
|
444
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
445
|
+
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
|
446
|
+
dependencies = [
|
|
447
|
+
"serde_derive",
|
|
448
|
+
]
|
|
449
|
+
|
|
450
|
+
[[package]]
|
|
451
|
+
name = "serde_derive"
|
|
452
|
+
version = "1.0.228"
|
|
453
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
454
|
+
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
|
455
|
+
dependencies = [
|
|
456
|
+
"proc-macro2",
|
|
457
|
+
"quote",
|
|
458
|
+
"syn",
|
|
459
|
+
]
|
|
460
|
+
|
|
461
|
+
[[package]]
|
|
462
|
+
name = "serde_json"
|
|
463
|
+
version = "1.0.149"
|
|
464
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
465
|
+
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
|
466
|
+
dependencies = [
|
|
467
|
+
"itoa",
|
|
468
|
+
"memchr",
|
|
469
|
+
"serde",
|
|
470
|
+
"serde_core",
|
|
471
|
+
"zmij",
|
|
472
|
+
]
|
|
473
|
+
|
|
474
|
+
[[package]]
|
|
475
|
+
name = "sha2"
|
|
476
|
+
version = "0.10.9"
|
|
477
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
478
|
+
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
|
479
|
+
dependencies = [
|
|
480
|
+
"cfg-if",
|
|
481
|
+
"cpufeatures",
|
|
482
|
+
"digest",
|
|
483
|
+
]
|
|
484
|
+
|
|
485
|
+
[[package]]
|
|
486
|
+
name = "shlex"
|
|
487
|
+
version = "1.3.0"
|
|
488
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
489
|
+
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
|
490
|
+
|
|
491
|
+
[[package]]
|
|
492
|
+
name = "syn"
|
|
493
|
+
version = "2.0.117"
|
|
494
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
495
|
+
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
|
496
|
+
dependencies = [
|
|
497
|
+
"proc-macro2",
|
|
498
|
+
"quote",
|
|
499
|
+
"unicode-ident",
|
|
500
|
+
]
|
|
501
|
+
|
|
502
|
+
[[package]]
|
|
503
|
+
name = "target-lexicon"
|
|
504
|
+
version = "0.12.16"
|
|
505
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
506
|
+
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
|
|
507
|
+
|
|
508
|
+
[[package]]
|
|
509
|
+
name = "tempfile"
|
|
510
|
+
version = "3.26.0"
|
|
511
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
512
|
+
checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
|
|
513
|
+
dependencies = [
|
|
514
|
+
"fastrand",
|
|
515
|
+
"getrandom",
|
|
516
|
+
"once_cell",
|
|
517
|
+
"rustix",
|
|
518
|
+
"windows-sys",
|
|
519
|
+
]
|
|
520
|
+
|
|
521
|
+
[[package]]
|
|
522
|
+
name = "typenum"
|
|
523
|
+
version = "1.19.0"
|
|
524
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
525
|
+
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
|
|
526
|
+
|
|
527
|
+
[[package]]
|
|
528
|
+
name = "unicode-ident"
|
|
529
|
+
version = "1.0.24"
|
|
530
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
531
|
+
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
|
532
|
+
|
|
533
|
+
[[package]]
|
|
534
|
+
name = "unicode-xid"
|
|
535
|
+
version = "0.2.6"
|
|
536
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
537
|
+
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
|
538
|
+
|
|
539
|
+
[[package]]
|
|
540
|
+
name = "unindent"
|
|
541
|
+
version = "0.2.4"
|
|
542
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
543
|
+
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
|
|
544
|
+
|
|
545
|
+
[[package]]
|
|
546
|
+
name = "version_check"
|
|
547
|
+
version = "0.9.5"
|
|
548
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
549
|
+
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
|
550
|
+
|
|
551
|
+
[[package]]
|
|
552
|
+
name = "wasip2"
|
|
553
|
+
version = "1.0.2+wasi-0.2.9"
|
|
554
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
555
|
+
checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
|
|
556
|
+
dependencies = [
|
|
557
|
+
"wit-bindgen",
|
|
558
|
+
]
|
|
559
|
+
|
|
560
|
+
[[package]]
|
|
561
|
+
name = "wasip3"
|
|
562
|
+
version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
|
|
563
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
564
|
+
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
|
|
565
|
+
dependencies = [
|
|
566
|
+
"wit-bindgen",
|
|
567
|
+
]
|
|
568
|
+
|
|
569
|
+
[[package]]
|
|
570
|
+
name = "wasm-encoder"
|
|
571
|
+
version = "0.244.0"
|
|
572
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
573
|
+
checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
|
|
574
|
+
dependencies = [
|
|
575
|
+
"leb128fmt",
|
|
576
|
+
"wasmparser",
|
|
577
|
+
]
|
|
578
|
+
|
|
579
|
+
[[package]]
|
|
580
|
+
name = "wasm-metadata"
|
|
581
|
+
version = "0.244.0"
|
|
582
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
583
|
+
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
|
|
584
|
+
dependencies = [
|
|
585
|
+
"anyhow",
|
|
586
|
+
"indexmap",
|
|
587
|
+
"wasm-encoder",
|
|
588
|
+
"wasmparser",
|
|
589
|
+
]
|
|
590
|
+
|
|
591
|
+
[[package]]
|
|
592
|
+
name = "wasmparser"
|
|
593
|
+
version = "0.244.0"
|
|
594
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
595
|
+
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
|
|
596
|
+
dependencies = [
|
|
597
|
+
"bitflags",
|
|
598
|
+
"hashbrown 0.15.5",
|
|
599
|
+
"indexmap",
|
|
600
|
+
"semver",
|
|
601
|
+
]
|
|
602
|
+
|
|
603
|
+
[[package]]
|
|
604
|
+
name = "windows-link"
|
|
605
|
+
version = "0.2.1"
|
|
606
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
607
|
+
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
|
608
|
+
|
|
609
|
+
[[package]]
|
|
610
|
+
name = "windows-sys"
|
|
611
|
+
version = "0.61.2"
|
|
612
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
613
|
+
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
|
614
|
+
dependencies = [
|
|
615
|
+
"windows-link",
|
|
616
|
+
]
|
|
617
|
+
|
|
618
|
+
[[package]]
|
|
619
|
+
name = "wit-bindgen"
|
|
620
|
+
version = "0.51.0"
|
|
621
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
622
|
+
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
|
|
623
|
+
dependencies = [
|
|
624
|
+
"wit-bindgen-rust-macro",
|
|
625
|
+
]
|
|
626
|
+
|
|
627
|
+
[[package]]
|
|
628
|
+
name = "wit-bindgen-core"
|
|
629
|
+
version = "0.51.0"
|
|
630
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
631
|
+
checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
|
|
632
|
+
dependencies = [
|
|
633
|
+
"anyhow",
|
|
634
|
+
"heck",
|
|
635
|
+
"wit-parser",
|
|
636
|
+
]
|
|
637
|
+
|
|
638
|
+
[[package]]
|
|
639
|
+
name = "wit-bindgen-rust"
|
|
640
|
+
version = "0.51.0"
|
|
641
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
642
|
+
checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
|
|
643
|
+
dependencies = [
|
|
644
|
+
"anyhow",
|
|
645
|
+
"heck",
|
|
646
|
+
"indexmap",
|
|
647
|
+
"prettyplease",
|
|
648
|
+
"syn",
|
|
649
|
+
"wasm-metadata",
|
|
650
|
+
"wit-bindgen-core",
|
|
651
|
+
"wit-component",
|
|
652
|
+
]
|
|
653
|
+
|
|
654
|
+
[[package]]
|
|
655
|
+
name = "wit-bindgen-rust-macro"
|
|
656
|
+
version = "0.51.0"
|
|
657
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
658
|
+
checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
|
|
659
|
+
dependencies = [
|
|
660
|
+
"anyhow",
|
|
661
|
+
"prettyplease",
|
|
662
|
+
"proc-macro2",
|
|
663
|
+
"quote",
|
|
664
|
+
"syn",
|
|
665
|
+
"wit-bindgen-core",
|
|
666
|
+
"wit-bindgen-rust",
|
|
667
|
+
]
|
|
668
|
+
|
|
669
|
+
[[package]]
|
|
670
|
+
name = "wit-component"
|
|
671
|
+
version = "0.244.0"
|
|
672
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
673
|
+
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
|
|
674
|
+
dependencies = [
|
|
675
|
+
"anyhow",
|
|
676
|
+
"bitflags",
|
|
677
|
+
"indexmap",
|
|
678
|
+
"log",
|
|
679
|
+
"serde",
|
|
680
|
+
"serde_derive",
|
|
681
|
+
"serde_json",
|
|
682
|
+
"wasm-encoder",
|
|
683
|
+
"wasm-metadata",
|
|
684
|
+
"wasmparser",
|
|
685
|
+
"wit-parser",
|
|
686
|
+
]
|
|
687
|
+
|
|
688
|
+
[[package]]
|
|
689
|
+
name = "wit-parser"
|
|
690
|
+
version = "0.244.0"
|
|
691
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
692
|
+
checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
|
|
693
|
+
dependencies = [
|
|
694
|
+
"anyhow",
|
|
695
|
+
"id-arena",
|
|
696
|
+
"indexmap",
|
|
697
|
+
"log",
|
|
698
|
+
"semver",
|
|
699
|
+
"serde",
|
|
700
|
+
"serde_derive",
|
|
701
|
+
"serde_json",
|
|
702
|
+
"unicode-xid",
|
|
703
|
+
"wasmparser",
|
|
704
|
+
]
|
|
705
|
+
|
|
706
|
+
[[package]]
|
|
707
|
+
name = "zmij"
|
|
708
|
+
version = "1.0.21"
|
|
709
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
710
|
+
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "dimq_load_task"
|
|
3
|
+
version = "0.1.1"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
description = "Rust-powered load generation task for DIMQ"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
repository = "https://github.com/walnutgeek/dimq"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
|
|
10
|
+
[lib]
|
|
11
|
+
name = "dimq_load_task"
|
|
12
|
+
crate-type = ["cdylib"]
|
|
13
|
+
|
|
14
|
+
[dependencies]
|
|
15
|
+
pyo3 = { version = "0.22", features = ["extension-module", "abi3-py39", "generate-import-lib"] }
|
|
16
|
+
rayon = "1.10"
|
|
17
|
+
sha2 = "0.10"
|
|
18
|
+
tempfile = "3"
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dimq-load-task
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Classifier: Development Status :: 3 - Alpha
|
|
5
|
+
Classifier: Programming Language :: Python :: 3
|
|
6
|
+
Classifier: Programming Language :: Rust
|
|
7
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
8
|
+
Summary: Rust-powered load generation task for DIMQ
|
|
9
|
+
Author-email: Walnut Geek <wg@walnutgeek.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
13
|
+
Project-URL: Homepage, https://github.com/walnutgeek/dimq
|
|
14
|
+
Project-URL: Repository, https://github.com/walnutgeek/dimq
|
|
15
|
+
|
|
16
|
+
# DIMQ - Distributed In-Memory Queue
|
|
17
|
+
|
|
18
|
+
A general-purpose distributed task processing framework built on ZeroMQ. Workers connect to a central orchestrator, receive tasks, and return results. The orchestrator adaptively tunes per-worker parallelization to maximize throughput.
|
|
19
|
+
|
|
20
|
+
## Architecture
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
+-------------------+
|
|
24
|
+
Clients -------->| Orchestrator |<-------- Workers
|
|
25
|
+
(DEALER) | (2x ROUTER) | (DEALER)
|
|
26
|
+
| |
|
|
27
|
+
SUBMIT/STATUS/ | - FIFO task queue | READY/HEARTBEAT/
|
|
28
|
+
RESULT queries | - Result storage | RESULT messages
|
|
29
|
+
| - Retry logic |
|
|
30
|
+
| - Adaptive tuning |
|
|
31
|
+
+-------------------+
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
**Orchestrator** runs two ZMQ ROUTER sockets: one for workers (registration, heartbeats, task dispatch/results) and one for clients (task submission, status queries, result retrieval).
|
|
35
|
+
|
|
36
|
+
**Workers** connect via ZMQ DEALER sockets, register with their CPU count, and pull tasks. They run tasks concurrently (sync tasks in a thread pool, async tasks natively) and report results back. Workers handle timeout cancellation locally; the orchestrator owns all retry decisions.
|
|
37
|
+
|
|
38
|
+
**Adaptive parallelization** starts each worker's parallel task limit at its CPU count, then probes upward. If throughput plateaus or drops, it scales back by one and enters steady mode, re-probing periodically.
|
|
39
|
+
|
|
40
|
+
**Tasks** are plain Python functions with Pydantic-typed input and output. No base class or decorator needed -- the framework introspects types via `inspect`.
|
|
41
|
+
|
|
42
|
+
## Prerequisites
|
|
43
|
+
|
|
44
|
+
- Python 3.9+
|
|
45
|
+
- [uv](https://docs.astral.sh/uv/)
|
|
46
|
+
- Rust toolchain (for the LoadTask extension only)
|
|
47
|
+
|
|
48
|
+
## Setup
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
# Install Python dependencies
|
|
52
|
+
uv sync
|
|
53
|
+
|
|
54
|
+
# (Optional) Build the Rust LoadTask extension
|
|
55
|
+
cd dimq_load_task
|
|
56
|
+
uv tool run maturin develop --uv
|
|
57
|
+
cd ..
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Running
|
|
61
|
+
|
|
62
|
+
Create a config file:
|
|
63
|
+
|
|
64
|
+
```yaml
|
|
65
|
+
# config.yaml
|
|
66
|
+
endpoint: "tcp://0.0.0.0:5555"
|
|
67
|
+
client_endpoint: "tcp://0.0.0.0:5556"
|
|
68
|
+
heartbeat_interval_seconds: 5
|
|
69
|
+
heartbeat_timeout_missed: 3
|
|
70
|
+
|
|
71
|
+
tasks:
|
|
72
|
+
- name: "my_app.tasks:process"
|
|
73
|
+
max_retries: 3
|
|
74
|
+
timeout_seconds: 30
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Start the orchestrator and one or more workers:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
# Terminal 1: start orchestrator
|
|
81
|
+
uv run dimq orchestrator --config config.yaml
|
|
82
|
+
|
|
83
|
+
# Terminal 2: start a worker (same machine)
|
|
84
|
+
uv run dimq worker --config config.yaml
|
|
85
|
+
|
|
86
|
+
# Terminal 3: start a worker on another machine (override endpoint)
|
|
87
|
+
uv run dimq worker --config config.yaml --endpoint tcp://orchestrator-host:5555
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Submitting tasks programmatically
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
import zmq
|
|
94
|
+
|
|
95
|
+
ctx = zmq.Context()
|
|
96
|
+
sock = ctx.socket(zmq.DEALER)
|
|
97
|
+
sock.connect("tcp://localhost:5556")
|
|
98
|
+
|
|
99
|
+
# Submit a task
|
|
100
|
+
sock.send_multipart([
|
|
101
|
+
b"SUBMIT",
|
|
102
|
+
b"my_app.tasks:process", # task type
|
|
103
|
+
b"task-001", # task ID
|
|
104
|
+
b'{"input_field": "value"}', # JSON payload matching the Pydantic input model
|
|
105
|
+
])
|
|
106
|
+
|
|
107
|
+
# Receive ACK
|
|
108
|
+
ack = sock.recv_multipart() # [b"ACK", b"task-001"]
|
|
109
|
+
|
|
110
|
+
# Later, query the result
|
|
111
|
+
sock.send_multipart([b"RESULT", b"task-001"])
|
|
112
|
+
reply = sock.recv_multipart()
|
|
113
|
+
# [b"RESULT_REPLY", b"task-001", b"COMPLETED", b'{"output_field": "result"}']
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Testing
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
# Run all tests (excludes e2e by default if Docker unavailable)
|
|
120
|
+
uv run pytest -v
|
|
121
|
+
|
|
122
|
+
# Run specific test modules
|
|
123
|
+
uv run pytest tests/test_orchestrator.py -v
|
|
124
|
+
uv run pytest tests/test_integration.py -v
|
|
125
|
+
|
|
126
|
+
# Run LoadTask tests (requires Rust extension to be built)
|
|
127
|
+
uv run pytest tests/test_load_task.py -v
|
|
128
|
+
|
|
129
|
+
# Run end-to-end Docker test (requires Docker daemon)
|
|
130
|
+
# Spins up 3 worker containers, submits load tasks, verifies adaptive tuning
|
|
131
|
+
uv run pytest tests/test_e2e_docker.py -v -s
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## Writing Custom Tasks
|
|
135
|
+
|
|
136
|
+
A task is a plain function with Pydantic-typed input and output. It can be sync or async.
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
# my_app/tasks.py
|
|
140
|
+
from pydantic import BaseModel
|
|
141
|
+
|
|
142
|
+
class ImageInput(BaseModel):
|
|
143
|
+
url: str
|
|
144
|
+
width: int
|
|
145
|
+
height: int
|
|
146
|
+
|
|
147
|
+
class ImageOutput(BaseModel):
|
|
148
|
+
thumbnail_path: str
|
|
149
|
+
original_size_bytes: int
|
|
150
|
+
|
|
151
|
+
def resize(input: ImageInput) -> ImageOutput:
|
|
152
|
+
# Your logic here
|
|
153
|
+
return ImageOutput(
|
|
154
|
+
thumbnail_path=f"/tmp/{input.width}x{input.height}.jpg",
|
|
155
|
+
original_size_bytes=1024,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
# Async tasks work the same way
|
|
159
|
+
async def fetch_and_resize(input: ImageInput) -> ImageOutput:
|
|
160
|
+
...
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Register it in your config:
|
|
164
|
+
|
|
165
|
+
```yaml
|
|
166
|
+
tasks:
|
|
167
|
+
- name: "my_app.tasks:resize"
|
|
168
|
+
max_retries: 2
|
|
169
|
+
timeout_seconds: 60
|
|
170
|
+
- name: "my_app.tasks:fetch_and_resize"
|
|
171
|
+
max_retries: 3
|
|
172
|
+
timeout_seconds: 120
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
The framework uses `inspect` to automatically extract:
|
|
176
|
+
- Input type from the first parameter's annotation
|
|
177
|
+
- Output type from the return annotation
|
|
178
|
+
- Whether the function is sync or async
|
|
179
|
+
|
|
180
|
+
Sync tasks run in a thread pool. Async tasks run natively in the event loop. If a task exceeds `timeout_seconds`, the worker cancels it and reports a timeout to the orchestrator, which handles retries.
|
|
181
|
+
|
|
182
|
+
## LoadTask (Rust Extension)
|
|
183
|
+
|
|
184
|
+
A built-in stress-testing task implemented in Rust (pyo3). It creates configurable CPU, I/O, and memory pressure while releasing the GIL so Python threading works efficiently.
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
import dimq_load_task
|
|
188
|
+
|
|
189
|
+
result = dimq_load_task.run(
|
|
190
|
+
duration_seconds=5.0,
|
|
191
|
+
concurrency=4, # number of CPU threads
|
|
192
|
+
cpu_load=0.7, # fraction of duration for CPU work (SHA-256 hashing)
|
|
193
|
+
io_load=0.2, # fraction for I/O (temp file writes)
|
|
194
|
+
memory_mb=100, # memory to allocate and touch
|
|
195
|
+
)
|
|
196
|
+
# result = {
|
|
197
|
+
# "phases": [
|
|
198
|
+
# {"type": "memory", "start_seconds": 0.0, "duration_seconds": 0.001},
|
|
199
|
+
# {"type": "cpu", "start_seconds": 0.001, "duration_seconds": 3.5},
|
|
200
|
+
# {"type": "io", "start_seconds": 3.501, "duration_seconds": 1.0},
|
|
201
|
+
# ],
|
|
202
|
+
# "total_duration_seconds": 4.501,
|
|
203
|
+
# "peak_memory_mb": 100,
|
|
204
|
+
# }
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
## Project Structure
|
|
208
|
+
|
|
209
|
+
```
|
|
210
|
+
DIMQ/
|
|
211
|
+
├── pyproject.toml
|
|
212
|
+
├── Dockerfile # Multi-stage build for worker containers
|
|
213
|
+
├── docker-compose.yml # 3 worker services for e2e testing
|
|
214
|
+
├── src/dimq/
|
|
215
|
+
│ ├── orchestrator.py # ZMQ ROUTER, dispatch, heartbeat, retry
|
|
216
|
+
│ ├── worker.py # ZMQ DEALER, task execution, timeout
|
|
217
|
+
│ ├── adaptive.py # Throughput-based parallelization tuning
|
|
218
|
+
│ ├── task.py # Task loading and introspection via inspect
|
|
219
|
+
│ ├── models.py # Pydantic models (TaskRecord, DimqConfig, etc.)
|
|
220
|
+
│ ├── config.py # YAML config loading
|
|
221
|
+
│ ├── cli.py # CLI entry points
|
|
222
|
+
│ └── tasks/
|
|
223
|
+
│ └── load.py # Pydantic wrapper for dimq_load_task
|
|
224
|
+
├── dimq_load_task/ # Rust extension (pyo3 + maturin)
|
|
225
|
+
│ ├── Cargo.toml
|
|
226
|
+
│ ├── pyproject.toml
|
|
227
|
+
│ └── src/lib.rs
|
|
228
|
+
├── e2e/
|
|
229
|
+
│ └── config.yaml # Worker config for Docker containers
|
|
230
|
+
└── tests/
|
|
231
|
+
├── test_orchestrator.py
|
|
232
|
+
├── test_worker.py
|
|
233
|
+
├── test_adaptive.py
|
|
234
|
+
├── test_integration.py
|
|
235
|
+
├── test_e2e_docker.py # Docker-based e2e test
|
|
236
|
+
├── test_load_task.py
|
|
237
|
+
├── test_tasks_load.py # LoadTask wrapper tests
|
|
238
|
+
├── test_models.py
|
|
239
|
+
├── test_config.py
|
|
240
|
+
├── test_task.py
|
|
241
|
+
└── sample_tasks.py # Test fixtures
|
|
242
|
+
```
|
|
243
|
+
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
# DIMQ - Distributed In-Memory Queue
|
|
2
|
+
|
|
3
|
+
A general-purpose distributed task processing framework built on ZeroMQ. Workers connect to a central orchestrator, receive tasks, and return results. The orchestrator adaptively tunes per-worker parallelization to maximize throughput.
|
|
4
|
+
|
|
5
|
+
## Architecture
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
+-------------------+
|
|
9
|
+
Clients -------->| Orchestrator |<-------- Workers
|
|
10
|
+
(DEALER) | (2x ROUTER) | (DEALER)
|
|
11
|
+
| |
|
|
12
|
+
SUBMIT/STATUS/ | - FIFO task queue | READY/HEARTBEAT/
|
|
13
|
+
RESULT queries | - Result storage | RESULT messages
|
|
14
|
+
| - Retry logic |
|
|
15
|
+
| - Adaptive tuning |
|
|
16
|
+
+-------------------+
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
**Orchestrator** runs two ZMQ ROUTER sockets: one for workers (registration, heartbeats, task dispatch/results) and one for clients (task submission, status queries, result retrieval).
|
|
20
|
+
|
|
21
|
+
**Workers** connect via ZMQ DEALER sockets, register with their CPU count, and pull tasks. They run tasks concurrently (sync tasks in a thread pool, async tasks natively) and report results back. Workers handle timeout cancellation locally; the orchestrator owns all retry decisions.
|
|
22
|
+
|
|
23
|
+
**Adaptive parallelization** starts each worker's parallel task limit at its CPU count, then probes upward. If throughput plateaus or drops, it scales back by one and enters steady mode, re-probing periodically.
|
|
24
|
+
|
|
25
|
+
**Tasks** are plain Python functions with Pydantic-typed input and output. No base class or decorator needed -- the framework introspects types via `inspect`.
|
|
26
|
+
|
|
27
|
+
## Prerequisites
|
|
28
|
+
|
|
29
|
+
- Python 3.9+
|
|
30
|
+
- [uv](https://docs.astral.sh/uv/)
|
|
31
|
+
- Rust toolchain (for the LoadTask extension only)
|
|
32
|
+
|
|
33
|
+
## Setup
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
# Install Python dependencies
|
|
37
|
+
uv sync
|
|
38
|
+
|
|
39
|
+
# (Optional) Build the Rust LoadTask extension
|
|
40
|
+
cd dimq_load_task
|
|
41
|
+
uv tool run maturin develop --uv
|
|
42
|
+
cd ..
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Running
|
|
46
|
+
|
|
47
|
+
Create a config file:
|
|
48
|
+
|
|
49
|
+
```yaml
|
|
50
|
+
# config.yaml
|
|
51
|
+
endpoint: "tcp://0.0.0.0:5555"
|
|
52
|
+
client_endpoint: "tcp://0.0.0.0:5556"
|
|
53
|
+
heartbeat_interval_seconds: 5
|
|
54
|
+
heartbeat_timeout_missed: 3
|
|
55
|
+
|
|
56
|
+
tasks:
|
|
57
|
+
- name: "my_app.tasks:process"
|
|
58
|
+
max_retries: 3
|
|
59
|
+
timeout_seconds: 30
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Start the orchestrator and one or more workers:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
# Terminal 1: start orchestrator
|
|
66
|
+
uv run dimq orchestrator --config config.yaml
|
|
67
|
+
|
|
68
|
+
# Terminal 2: start a worker (same machine)
|
|
69
|
+
uv run dimq worker --config config.yaml
|
|
70
|
+
|
|
71
|
+
# Terminal 3: start a worker on another machine (override endpoint)
|
|
72
|
+
uv run dimq worker --config config.yaml --endpoint tcp://orchestrator-host:5555
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Submitting tasks programmatically
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import zmq
|
|
79
|
+
|
|
80
|
+
ctx = zmq.Context()
|
|
81
|
+
sock = ctx.socket(zmq.DEALER)
|
|
82
|
+
sock.connect("tcp://localhost:5556")
|
|
83
|
+
|
|
84
|
+
# Submit a task
|
|
85
|
+
sock.send_multipart([
|
|
86
|
+
b"SUBMIT",
|
|
87
|
+
b"my_app.tasks:process", # task type
|
|
88
|
+
b"task-001", # task ID
|
|
89
|
+
b'{"input_field": "value"}', # JSON payload matching the Pydantic input model
|
|
90
|
+
])
|
|
91
|
+
|
|
92
|
+
# Receive ACK
|
|
93
|
+
ack = sock.recv_multipart() # [b"ACK", b"task-001"]
|
|
94
|
+
|
|
95
|
+
# Later, query the result
|
|
96
|
+
sock.send_multipart([b"RESULT", b"task-001"])
|
|
97
|
+
reply = sock.recv_multipart()
|
|
98
|
+
# [b"RESULT_REPLY", b"task-001", b"COMPLETED", b'{"output_field": "result"}']
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Testing
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
# Run all tests (excludes e2e by default if Docker unavailable)
|
|
105
|
+
uv run pytest -v
|
|
106
|
+
|
|
107
|
+
# Run specific test modules
|
|
108
|
+
uv run pytest tests/test_orchestrator.py -v
|
|
109
|
+
uv run pytest tests/test_integration.py -v
|
|
110
|
+
|
|
111
|
+
# Run LoadTask tests (requires Rust extension to be built)
|
|
112
|
+
uv run pytest tests/test_load_task.py -v
|
|
113
|
+
|
|
114
|
+
# Run end-to-end Docker test (requires Docker daemon)
|
|
115
|
+
# Spins up 3 worker containers, submits load tasks, verifies adaptive tuning
|
|
116
|
+
uv run pytest tests/test_e2e_docker.py -v -s
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Writing Custom Tasks
|
|
120
|
+
|
|
121
|
+
A task is a plain function with Pydantic-typed input and output. It can be sync or async.
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
# my_app/tasks.py
|
|
125
|
+
from pydantic import BaseModel
|
|
126
|
+
|
|
127
|
+
class ImageInput(BaseModel):
|
|
128
|
+
url: str
|
|
129
|
+
width: int
|
|
130
|
+
height: int
|
|
131
|
+
|
|
132
|
+
class ImageOutput(BaseModel):
|
|
133
|
+
thumbnail_path: str
|
|
134
|
+
original_size_bytes: int
|
|
135
|
+
|
|
136
|
+
def resize(input: ImageInput) -> ImageOutput:
|
|
137
|
+
# Your logic here
|
|
138
|
+
return ImageOutput(
|
|
139
|
+
thumbnail_path=f"/tmp/{input.width}x{input.height}.jpg",
|
|
140
|
+
original_size_bytes=1024,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
# Async tasks work the same way
|
|
144
|
+
async def fetch_and_resize(input: ImageInput) -> ImageOutput:
|
|
145
|
+
...
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Register it in your config:
|
|
149
|
+
|
|
150
|
+
```yaml
|
|
151
|
+
tasks:
|
|
152
|
+
- name: "my_app.tasks:resize"
|
|
153
|
+
max_retries: 2
|
|
154
|
+
timeout_seconds: 60
|
|
155
|
+
- name: "my_app.tasks:fetch_and_resize"
|
|
156
|
+
max_retries: 3
|
|
157
|
+
timeout_seconds: 120
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
The framework uses `inspect` to automatically extract:
|
|
161
|
+
- Input type from the first parameter's annotation
|
|
162
|
+
- Output type from the return annotation
|
|
163
|
+
- Whether the function is sync or async
|
|
164
|
+
|
|
165
|
+
Sync tasks run in a thread pool. Async tasks run natively in the event loop. If a task exceeds `timeout_seconds`, the worker cancels it and reports a timeout to the orchestrator, which handles retries.
|
|
166
|
+
|
|
167
|
+
## LoadTask (Rust Extension)
|
|
168
|
+
|
|
169
|
+
A built-in stress-testing task implemented in Rust (pyo3). It creates configurable CPU, I/O, and memory pressure while releasing the GIL so Python threading works efficiently.
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
import dimq_load_task
|
|
173
|
+
|
|
174
|
+
result = dimq_load_task.run(
|
|
175
|
+
duration_seconds=5.0,
|
|
176
|
+
concurrency=4, # number of CPU threads
|
|
177
|
+
cpu_load=0.7, # fraction of duration for CPU work (SHA-256 hashing)
|
|
178
|
+
io_load=0.2, # fraction for I/O (temp file writes)
|
|
179
|
+
memory_mb=100, # memory to allocate and touch
|
|
180
|
+
)
|
|
181
|
+
# result = {
|
|
182
|
+
# "phases": [
|
|
183
|
+
# {"type": "memory", "start_seconds": 0.0, "duration_seconds": 0.001},
|
|
184
|
+
# {"type": "cpu", "start_seconds": 0.001, "duration_seconds": 3.5},
|
|
185
|
+
# {"type": "io", "start_seconds": 3.501, "duration_seconds": 1.0},
|
|
186
|
+
# ],
|
|
187
|
+
# "total_duration_seconds": 4.501,
|
|
188
|
+
# "peak_memory_mb": 100,
|
|
189
|
+
# }
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## Project Structure
|
|
193
|
+
|
|
194
|
+
```
|
|
195
|
+
DIMQ/
|
|
196
|
+
├── pyproject.toml
|
|
197
|
+
├── Dockerfile # Multi-stage build for worker containers
|
|
198
|
+
├── docker-compose.yml # 3 worker services for e2e testing
|
|
199
|
+
├── src/dimq/
|
|
200
|
+
│ ├── orchestrator.py # ZMQ ROUTER, dispatch, heartbeat, retry
|
|
201
|
+
│ ├── worker.py # ZMQ DEALER, task execution, timeout
|
|
202
|
+
│ ├── adaptive.py # Throughput-based parallelization tuning
|
|
203
|
+
│ ├── task.py # Task loading and introspection via inspect
|
|
204
|
+
│ ├── models.py # Pydantic models (TaskRecord, DimqConfig, etc.)
|
|
205
|
+
│ ├── config.py # YAML config loading
|
|
206
|
+
│ ├── cli.py # CLI entry points
|
|
207
|
+
│ └── tasks/
|
|
208
|
+
│ └── load.py # Pydantic wrapper for dimq_load_task
|
|
209
|
+
├── dimq_load_task/ # Rust extension (pyo3 + maturin)
|
|
210
|
+
│ ├── Cargo.toml
|
|
211
|
+
│ ├── pyproject.toml
|
|
212
|
+
│ └── src/lib.rs
|
|
213
|
+
├── e2e/
|
|
214
|
+
│ └── config.yaml # Worker config for Docker containers
|
|
215
|
+
└── tests/
|
|
216
|
+
├── test_orchestrator.py
|
|
217
|
+
├── test_worker.py
|
|
218
|
+
├── test_adaptive.py
|
|
219
|
+
├── test_integration.py
|
|
220
|
+
├── test_e2e_docker.py # Docker-based e2e test
|
|
221
|
+
├── test_load_task.py
|
|
222
|
+
├── test_tasks_load.py # LoadTask wrapper tests
|
|
223
|
+
├── test_models.py
|
|
224
|
+
├── test_config.py
|
|
225
|
+
├── test_task.py
|
|
226
|
+
└── sample_tasks.py # Test fixtures
|
|
227
|
+
```
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["maturin>=1.0,<2.0"]
|
|
3
|
+
build-backend = "maturin"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "dimq-load-task"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "Rust-powered load generation task for DIMQ"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [{ name = "Walnut Geek", email = "wg@walnutgeek.com" }]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Rust",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.urls]
|
|
21
|
+
Homepage = "https://github.com/walnutgeek/dimq"
|
|
22
|
+
Repository = "https://github.com/walnutgeek/dimq"
|
|
23
|
+
|
|
24
|
+
[tool.maturin]
|
|
25
|
+
features = ["pyo3/extension-module"]
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
use pyo3::prelude::*;
|
|
2
|
+
use rayon::prelude::*;
|
|
3
|
+
use sha2::{Digest, Sha256};
|
|
4
|
+
use std::io::Write;
|
|
5
|
+
use std::time::{Duration, Instant};
|
|
6
|
+
|
|
7
|
+
#[pyfunction]
|
|
8
|
+
#[pyo3(signature = (duration_seconds=1.0, concurrency=1, cpu_load=0.7, io_load=0.2, memory_mb=50))]
|
|
9
|
+
fn run(
|
|
10
|
+
py: Python<'_>,
|
|
11
|
+
duration_seconds: f64,
|
|
12
|
+
concurrency: usize,
|
|
13
|
+
cpu_load: f64,
|
|
14
|
+
io_load: f64,
|
|
15
|
+
memory_mb: usize,
|
|
16
|
+
) -> PyResult<PyObject> {
|
|
17
|
+
let result = py.allow_threads(|| {
|
|
18
|
+
let start = Instant::now();
|
|
19
|
+
let mut phases: Vec<(String, f64, f64)> = Vec::new();
|
|
20
|
+
|
|
21
|
+
// Memory pressure
|
|
22
|
+
let mem_start = start.elapsed().as_secs_f64();
|
|
23
|
+
let mut _memory: Vec<u8> = vec![0u8; memory_mb * 1024 * 1024];
|
|
24
|
+
// Touch pages to ensure real allocation
|
|
25
|
+
for i in (0.._memory.len()).step_by(4096) {
|
|
26
|
+
_memory[i] = 1;
|
|
27
|
+
}
|
|
28
|
+
phases.push((
|
|
29
|
+
"memory".into(),
|
|
30
|
+
mem_start,
|
|
31
|
+
start.elapsed().as_secs_f64() - mem_start,
|
|
32
|
+
));
|
|
33
|
+
|
|
34
|
+
// CPU load using rayon
|
|
35
|
+
let cpu_duration = Duration::from_secs_f64(duration_seconds * cpu_load);
|
|
36
|
+
let cpu_start = start.elapsed().as_secs_f64();
|
|
37
|
+
let pool = rayon::ThreadPoolBuilder::new()
|
|
38
|
+
.num_threads(concurrency)
|
|
39
|
+
.build()
|
|
40
|
+
.unwrap();
|
|
41
|
+
pool.install(|| {
|
|
42
|
+
let deadline = Instant::now() + cpu_duration;
|
|
43
|
+
(0..concurrency).into_par_iter().for_each(|_| {
|
|
44
|
+
let mut hasher = Sha256::new();
|
|
45
|
+
let mut counter: u64 = 0;
|
|
46
|
+
while Instant::now() < deadline {
|
|
47
|
+
hasher.update(counter.to_le_bytes());
|
|
48
|
+
counter += 1;
|
|
49
|
+
if counter % 10000 == 0 {
|
|
50
|
+
let _ = hasher.finalize_reset();
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
phases.push((
|
|
56
|
+
"cpu".into(),
|
|
57
|
+
cpu_start,
|
|
58
|
+
start.elapsed().as_secs_f64() - cpu_start,
|
|
59
|
+
));
|
|
60
|
+
|
|
61
|
+
// IO load
|
|
62
|
+
let io_duration = Duration::from_secs_f64(duration_seconds * io_load);
|
|
63
|
+
let io_start_t = start.elapsed().as_secs_f64();
|
|
64
|
+
let io_deadline = Instant::now() + io_duration;
|
|
65
|
+
while Instant::now() < io_deadline {
|
|
66
|
+
if let Ok(mut tmpfile) = tempfile::tempfile() {
|
|
67
|
+
let data = vec![0u8; 1024 * 64]; // 64KB writes
|
|
68
|
+
let _ = tmpfile.write_all(&data);
|
|
69
|
+
let _ = tmpfile.flush();
|
|
70
|
+
}
|
|
71
|
+
std::thread::sleep(Duration::from_millis(10));
|
|
72
|
+
}
|
|
73
|
+
phases.push((
|
|
74
|
+
"io".into(),
|
|
75
|
+
io_start_t,
|
|
76
|
+
start.elapsed().as_secs_f64() - io_start_t,
|
|
77
|
+
));
|
|
78
|
+
|
|
79
|
+
let total = start.elapsed().as_secs_f64();
|
|
80
|
+
(phases, total, memory_mb)
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
// Build Python dict result
|
|
84
|
+
let dict = pyo3::types::PyDict::new_bound(py);
|
|
85
|
+
let phases_list = pyo3::types::PyList::empty_bound(py);
|
|
86
|
+
for (phase_type, start_time, duration) in &result.0 {
|
|
87
|
+
let phase_dict = pyo3::types::PyDict::new_bound(py);
|
|
88
|
+
phase_dict.set_item("type", phase_type)?;
|
|
89
|
+
phase_dict.set_item("start_seconds", start_time)?;
|
|
90
|
+
phase_dict.set_item("duration_seconds", duration)?;
|
|
91
|
+
phases_list.append(phase_dict)?;
|
|
92
|
+
}
|
|
93
|
+
dict.set_item("phases", phases_list)?;
|
|
94
|
+
dict.set_item("total_duration_seconds", result.1)?;
|
|
95
|
+
dict.set_item("peak_memory_mb", result.2)?;
|
|
96
|
+
|
|
97
|
+
Ok(dict.into())
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
#[pymodule]
|
|
101
|
+
fn dimq_load_task(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
102
|
+
m.add_function(wrap_pyfunction!(run, m)?)?;
|
|
103
|
+
Ok(())
|
|
104
|
+
}
|