reykit 1.1.11__py3-none-any.whl → 1.1.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reykit/rmultitask.py +543 -520
- reykit/rnumber.py +32 -9
- reykit/rtime.py +41 -2
- {reykit-1.1.11.dist-info → reykit-1.1.13.dist-info}/METADATA +1 -1
- {reykit-1.1.11.dist-info → reykit-1.1.13.dist-info}/RECORD +7 -7
- {reykit-1.1.11.dist-info → reykit-1.1.13.dist-info}/WHEEL +0 -0
- {reykit-1.1.11.dist-info → reykit-1.1.13.dist-info}/licenses/LICENSE +0 -0
reykit/rmultitask.py
CHANGED
@@ -10,362 +10,51 @@
|
|
10
10
|
|
11
11
|
|
12
12
|
from __future__ import annotations
|
13
|
-
from typing import Any, Literal
|
13
|
+
from typing import Any, Literal, overload
|
14
14
|
from collections.abc import Callable, Iterable, Generator, Coroutine
|
15
15
|
from threading import RLock as TRLock, get_ident as threading_get_ident
|
16
16
|
from concurrent.futures import ThreadPoolExecutor, Future as CFuture, as_completed as concurrent_as_completed
|
17
|
+
from queue import Queue as QQueue
|
17
18
|
from asyncio import (
|
18
19
|
Future as AFuture,
|
19
|
-
Queue as AQueue,
|
20
20
|
Lock as ALock,
|
21
|
+
Task as ATask,
|
22
|
+
Queue as AQueue,
|
23
|
+
sleep as asyncio_sleep,
|
21
24
|
run as asyncio_run,
|
22
25
|
gather as asyncio_gather,
|
23
|
-
iscoroutine
|
26
|
+
iscoroutine as asyncio_iscoroutine,
|
27
|
+
iscoroutinefunction as asyncio_iscoroutinefunction,
|
28
|
+
run_coroutine_threadsafe as asyncio_run_coroutine_threadsafe,
|
29
|
+
new_event_loop as asyncio_new_event_loop,
|
30
|
+
set_event_loop as asyncio_set_event_loop
|
24
31
|
)
|
25
|
-
from asyncio.queues import QueueEmpty
|
26
32
|
from aiohttp import ClientSession, ClientResponse
|
27
33
|
|
28
34
|
from .rexception import throw, check_most_one, check_response_code
|
29
|
-
from .rtime import
|
35
|
+
from .rtime import randn, RTimeMark
|
36
|
+
from .rtype import T
|
30
37
|
from .rwrap import wrap_thread
|
31
38
|
|
32
39
|
|
33
40
|
__all__ = (
|
41
|
+
'RThreadPool',
|
34
42
|
'async_run',
|
43
|
+
'async_sleep',
|
44
|
+
'async_wait',
|
35
45
|
'async_request',
|
36
|
-
'RThreadLock',
|
37
|
-
'RAsyncLock',
|
38
|
-
'RThreadPool',
|
39
46
|
'RAsyncPool'
|
40
47
|
)
|
41
48
|
|
42
49
|
|
43
|
-
def async_run(*coroutines: Coroutine) -> list:
|
44
|
-
"""
|
45
|
-
Asynchronous run `Coroutine` instances.
|
46
|
-
|
47
|
-
Parameters
|
48
|
-
----------
|
49
|
-
coroutines : `Coroutine` instances.
|
50
|
-
|
51
|
-
Returns
|
52
|
-
-------
|
53
|
-
Run result list.
|
54
|
-
"""
|
55
|
-
|
56
|
-
|
57
|
-
# Define.
|
58
|
-
async def gather_coroutine() -> AFuture:
|
59
|
-
"""
|
60
|
-
Get `Future` instance.
|
61
|
-
|
62
|
-
Returns
|
63
|
-
-------
|
64
|
-
Future instance.
|
65
|
-
"""
|
66
|
-
|
67
|
-
# Gather.
|
68
|
-
future = await asyncio_gather(*coroutines)
|
69
|
-
|
70
|
-
return future
|
71
|
-
|
72
|
-
|
73
|
-
# Run.
|
74
|
-
result = asyncio_run(gather_coroutine())
|
75
|
-
|
76
|
-
return result
|
77
|
-
|
78
|
-
|
79
|
-
async def async_request(
|
80
|
-
url: str,
|
81
|
-
params: dict | None = None,
|
82
|
-
data: dict | str | bytes | None = None,
|
83
|
-
json: dict | None = None,
|
84
|
-
headers: dict[str, str] = {},
|
85
|
-
timeout: float | None = None,
|
86
|
-
proxy: str | None = None,
|
87
|
-
method: Literal['get', 'post', 'put', 'patch', 'delete', 'options', 'head'] | None = None,
|
88
|
-
check: bool | int | Iterable[int] = False,
|
89
|
-
handler: str | tuple[str] | Callable[[ClientResponse], Coroutine | Any] | None = None
|
90
|
-
) -> Any:
|
91
|
-
"""
|
92
|
-
Get asynchronous `Coroutine` instance of send request.
|
93
|
-
|
94
|
-
Parameters
|
95
|
-
----------
|
96
|
-
url : Request URL.
|
97
|
-
params : Request URL add parameters.
|
98
|
-
data : Request body data. Conflict with parameter `json`.
|
99
|
-
- `dict`, Convert to `key=value&...`: format bytes.
|
100
|
-
Automatic set `Content-Type` to `application/x-www-form-urlencoded`.
|
101
|
-
- `dict and a certain value is 'bytes' type`: Key is parameter name and file name, value is file data.
|
102
|
-
Automatic set `Content-Type` to `multipart/form-data`.
|
103
|
-
- `str`: File path to read file bytes data.
|
104
|
-
Automatic set `Content-Type` to file media type, and `filename` to file name.
|
105
|
-
- `bytes`: File bytes data.
|
106
|
-
Automatic set `Content-Type` to file media type.
|
107
|
-
json : Request body data, convert to `JSON` format. Conflict with parameter `data`.
|
108
|
-
Automatic set `Content-Type` to `application/json`.
|
109
|
-
headers : Request header data.
|
110
|
-
timeout : Request maximun waiting time.
|
111
|
-
proxy : Proxy URL.
|
112
|
-
method : Request method.
|
113
|
-
- `None`: Automatic judge.
|
114
|
-
When parameter `data` or `json` not has value, then request method is `get`.
|
115
|
-
When parameter `data` or `json` has value, then request method is `post`.
|
116
|
-
- `Literal['get', 'post', 'put', 'patch', 'delete', 'options', 'head']`: Use this request method.
|
117
|
-
check : Check response code, and throw exception.
|
118
|
-
- `Literal[False]`: Not check.
|
119
|
-
- `Literal[True]`: Check if is between 200 and 299.
|
120
|
-
- `int`: Check if is this value.
|
121
|
-
- `Iterable`: Check if is in sequence.
|
122
|
-
handler : Response handler.
|
123
|
-
- `None`: Automatic handle.
|
124
|
-
`Response 'Content-Type' is 'application/json'`: Use `ClientResponse.json` method.
|
125
|
-
`Response 'Content-Type' is 'text/plain; charset=utf-8'`: Use `ClientResponse.text` method.
|
126
|
-
`Other`: Use `ClientResponse.read` method.
|
127
|
-
- `str`: Get this attribute.
|
128
|
-
`Callable`: Execute this method. When return `Coroutine`, then use `await` syntax execute `Coroutine`.
|
129
|
-
`Any`: Return this value.
|
130
|
-
- `tuple[str]`: Get these attribute.
|
131
|
-
`Callable`: Execute this method. When return `Coroutine`, then use `await` syntax execute `Coroutine`.
|
132
|
-
`Any`: Return this value.
|
133
|
-
- `Callable`, Execute this method. When return `Coroutine`, then use `await`: syntax execute `Coroutine`.
|
134
|
-
|
135
|
-
Returns
|
136
|
-
-------
|
137
|
-
Response handler result.
|
138
|
-
"""
|
139
|
-
|
140
|
-
# Check.
|
141
|
-
check_most_one(data, json)
|
142
|
-
|
143
|
-
# Handle parameter.
|
144
|
-
if method is None:
|
145
|
-
if data is None and json is None:
|
146
|
-
method = 'get'
|
147
|
-
else:
|
148
|
-
method = 'post'
|
149
|
-
|
150
|
-
# Session.
|
151
|
-
async with ClientSession() as session:
|
152
|
-
|
153
|
-
# Request.
|
154
|
-
async with session.request(
|
155
|
-
method,
|
156
|
-
url,
|
157
|
-
params=params,
|
158
|
-
data=data,
|
159
|
-
json=json,
|
160
|
-
headers=headers,
|
161
|
-
timeout=timeout,
|
162
|
-
proxy=proxy
|
163
|
-
) as response:
|
164
|
-
|
165
|
-
# Check code.
|
166
|
-
if check is not False:
|
167
|
-
if check is True:
|
168
|
-
range_ = None
|
169
|
-
else:
|
170
|
-
range_ = check
|
171
|
-
check_response_code(response.status, range_)
|
172
|
-
|
173
|
-
# Receive.
|
174
|
-
match handler:
|
175
|
-
|
176
|
-
## Auto.
|
177
|
-
case None:
|
178
|
-
match response.content_type:
|
179
|
-
case 'application/json':
|
180
|
-
result = await response.json()
|
181
|
-
case 'text/plain; charset=utf-8':
|
182
|
-
|
183
|
-
# Set encode type.
|
184
|
-
if response.get_encoding() == 'ISO-8859-1':
|
185
|
-
encoding = 'utf-8'
|
186
|
-
else:
|
187
|
-
encoding = None
|
188
|
-
|
189
|
-
result = await response.text(encoding=encoding)
|
190
|
-
case _:
|
191
|
-
result = await response.read()
|
192
|
-
|
193
|
-
## Attribute.
|
194
|
-
case str():
|
195
|
-
result = getattr(response, handler)
|
196
|
-
|
197
|
-
### Method.
|
198
|
-
if callable(result):
|
199
|
-
result = result()
|
200
|
-
|
201
|
-
#### Coroutine.
|
202
|
-
if iscoroutine(result):
|
203
|
-
result = await result
|
204
|
-
|
205
|
-
## Attributes.
|
206
|
-
case tuple():
|
207
|
-
result = []
|
208
|
-
for key in handler:
|
209
|
-
result_element = getattr(response, key)
|
210
|
-
|
211
|
-
### Method.
|
212
|
-
if callable(result_element):
|
213
|
-
result_element = result_element()
|
214
|
-
|
215
|
-
#### Coroutine.
|
216
|
-
if iscoroutine(result_element):
|
217
|
-
result_element = await result_element
|
218
|
-
|
219
|
-
result.append(result_element)
|
220
|
-
|
221
|
-
## Method.
|
222
|
-
case _ if callable(handler):
|
223
|
-
result = handler(response)
|
224
|
-
|
225
|
-
### Coroutine.
|
226
|
-
if iscoroutine(result):
|
227
|
-
result = await result
|
228
|
-
|
229
|
-
## Throw exception.
|
230
|
-
case _:
|
231
|
-
throw(TypeError, handler)
|
232
|
-
|
233
|
-
return result
|
234
|
-
|
235
|
-
|
236
|
-
class RThreadLock():
|
237
|
-
"""
|
238
|
-
Rey's `thread lock` type.
|
239
|
-
"""
|
240
|
-
|
241
|
-
|
242
|
-
def __init__(self) -> None:
|
243
|
-
"""
|
244
|
-
Build `thread lock` attributes.
|
245
|
-
"""
|
246
|
-
|
247
|
-
# Set attribute.
|
248
|
-
self.lock = TRLock()
|
249
|
-
self.acquire_thread_id: int | None = None
|
250
|
-
|
251
|
-
|
252
|
-
def acquire(
|
253
|
-
self,
|
254
|
-
timeout: float = None
|
255
|
-
) -> bool:
|
256
|
-
"""
|
257
|
-
Wait and acquire thread lock.
|
258
|
-
|
259
|
-
Parameters
|
260
|
-
----------
|
261
|
-
timeout : Maximum wait seconds.
|
262
|
-
- `None`: Not limit.
|
263
|
-
- `float`: Use this value.
|
264
|
-
|
265
|
-
Returns
|
266
|
-
-------
|
267
|
-
Whether acquire success.
|
268
|
-
"""
|
269
|
-
|
270
|
-
# Handle parameter.
|
271
|
-
if timeout is None:
|
272
|
-
timeout = -1
|
273
|
-
|
274
|
-
# Acquire.
|
275
|
-
result = self.lock.acquire(timeout=timeout)
|
276
|
-
|
277
|
-
# Update attribute.
|
278
|
-
if result:
|
279
|
-
thread_id = threading_get_ident()
|
280
|
-
self.acquire_thread_id = thread_id
|
281
|
-
|
282
|
-
return result
|
283
|
-
|
284
|
-
|
285
|
-
def release(self) -> None:
|
286
|
-
"""
|
287
|
-
Release thread lock.
|
288
|
-
"""
|
289
|
-
|
290
|
-
# Release.
|
291
|
-
self.lock.release()
|
292
|
-
|
293
|
-
# Update attribute.
|
294
|
-
self.acquire_thread_id = None
|
295
|
-
|
296
|
-
|
297
|
-
def __call__(self) -> None:
|
298
|
-
"""
|
299
|
-
Automatic judge, wait and acquire thread lock, or release thread lock.
|
300
|
-
"""
|
301
|
-
|
302
|
-
# Release.
|
303
|
-
thread_id = threading_get_ident()
|
304
|
-
if thread_id == self.acquire_thread_id:
|
305
|
-
self.release()
|
306
|
-
|
307
|
-
# Acquire.
|
308
|
-
else:
|
309
|
-
self.acquire()
|
310
|
-
|
311
|
-
|
312
|
-
class RAsyncLock():
|
313
|
-
"""
|
314
|
-
Rey's `asynchronous lock` type.
|
315
|
-
"""
|
316
|
-
|
317
|
-
|
318
|
-
def __init__(self) -> None:
|
319
|
-
"""
|
320
|
-
Build `asynchronous lock` attributes.
|
321
|
-
"""
|
322
|
-
|
323
|
-
# Set attribute.
|
324
|
-
self.lock = ALock()
|
325
|
-
|
326
|
-
|
327
|
-
def acquire(
|
328
|
-
self,
|
329
|
-
timeout: float = None
|
330
|
-
) -> bool:
|
331
|
-
"""
|
332
|
-
Wait and acquire thread lock.
|
333
|
-
|
334
|
-
Parameters
|
335
|
-
----------
|
336
|
-
timeout : Maximum wait seconds.
|
337
|
-
- `None`: Not limit.
|
338
|
-
- `float`: Use this value.
|
339
|
-
|
340
|
-
Returns
|
341
|
-
-------
|
342
|
-
Whether acquire success.
|
343
|
-
"""
|
344
|
-
|
345
|
-
# Handle parameter.
|
346
|
-
if timeout is None:
|
347
|
-
timeout = -1
|
348
|
-
|
349
|
-
# Acquire.
|
350
|
-
result = self.lock.acquire()
|
351
|
-
|
352
|
-
return result
|
353
|
-
|
354
|
-
|
355
|
-
def release(self) -> None:
|
356
|
-
"""
|
357
|
-
Release thread lock.
|
358
|
-
"""
|
359
|
-
|
360
|
-
# Release.
|
361
|
-
self.lock.release()
|
362
|
-
|
363
|
-
|
364
50
|
class RThreadPool(object):
|
365
51
|
"""
|
366
52
|
Rey's `thread pool` type.
|
367
53
|
"""
|
368
54
|
|
55
|
+
Queue = QQueue
|
56
|
+
Lock = TRLock
|
57
|
+
|
369
58
|
|
370
59
|
def __init__(
|
371
60
|
self,
|
@@ -380,18 +69,18 @@ class RThreadPool(object):
|
|
380
69
|
Parameters
|
381
70
|
----------
|
382
71
|
task : Thread task.
|
383
|
-
args :
|
72
|
+
args : ATask default position arguments.
|
384
73
|
_max_workers : Maximum number of threads.
|
385
74
|
- `None`: Number of CPU + 4, 32 maximum.
|
386
75
|
- `int`: Use this value, no maximum limit.
|
387
|
-
kwargs :
|
76
|
+
kwargs : ATask default keyword arguments.
|
388
77
|
"""
|
389
78
|
|
390
79
|
# Set attribute.
|
391
80
|
self.task = task
|
392
81
|
self.args = args
|
393
82
|
self.kwargs = kwargs
|
394
|
-
self.
|
83
|
+
self.pool = ThreadPoolExecutor(
|
395
84
|
_max_workers,
|
396
85
|
task.__name__
|
397
86
|
)
|
@@ -404,16 +93,16 @@ class RThreadPool(object):
|
|
404
93
|
**kwargs: Any
|
405
94
|
) -> CFuture:
|
406
95
|
"""
|
407
|
-
|
96
|
+
Start a task.
|
408
97
|
|
409
98
|
Parameters
|
410
99
|
----------
|
411
|
-
args :
|
412
|
-
kwargs :
|
100
|
+
args : ATask position arguments, after default position arguments.
|
101
|
+
kwargs : ATask keyword arguments, after default keyword arguments.
|
413
102
|
|
414
103
|
Returns
|
415
104
|
-------
|
416
|
-
|
105
|
+
ATask instance.
|
417
106
|
"""
|
418
107
|
|
419
108
|
# Set parameter.
|
@@ -426,8 +115,8 @@ class RThreadPool(object):
|
|
426
115
|
**kwargs
|
427
116
|
}
|
428
117
|
|
429
|
-
#
|
430
|
-
future = self.
|
118
|
+
# Add.
|
119
|
+
future = self.pool.submit(
|
431
120
|
self.task,
|
432
121
|
*func_args,
|
433
122
|
**func_kwargs
|
@@ -445,7 +134,7 @@ class RThreadPool(object):
|
|
445
134
|
**kwargs: tuple
|
446
135
|
) -> list[CFuture]:
|
447
136
|
"""
|
448
|
-
|
137
|
+
Batch start tasks.
|
449
138
|
parameters sequence will combine one by one, and discard excess parameters.
|
450
139
|
|
451
140
|
Parameters
|
@@ -455,7 +144,7 @@ class RThreadPool(object):
|
|
455
144
|
|
456
145
|
Returns
|
457
146
|
-------
|
458
|
-
|
147
|
+
ATask instance list.
|
459
148
|
|
460
149
|
Examples
|
461
150
|
--------
|
@@ -483,7 +172,7 @@ class RThreadPool(object):
|
|
483
172
|
)
|
484
173
|
params_zip = zip(args_zip, kwargs_zip)
|
485
174
|
|
486
|
-
# Batch
|
175
|
+
# Batch add.
|
487
176
|
futures = [
|
488
177
|
self.one(*args_, **dict(kwargs_))
|
489
178
|
for args_, kwargs_ in params_zip
|
@@ -495,6 +184,34 @@ class RThreadPool(object):
|
|
495
184
|
return futures
|
496
185
|
|
497
186
|
|
187
|
+
def repeat(
|
188
|
+
self,
|
189
|
+
number: int
|
190
|
+
) -> list[CFuture]:
|
191
|
+
"""
|
192
|
+
Batch start tasks, and only with default parameters.
|
193
|
+
|
194
|
+
Parameters
|
195
|
+
----------
|
196
|
+
number : Number of add.
|
197
|
+
|
198
|
+
Returns
|
199
|
+
-------
|
200
|
+
ATask instance list.
|
201
|
+
"""
|
202
|
+
|
203
|
+
# Batch add.
|
204
|
+
futures = [
|
205
|
+
self.one()
|
206
|
+
for _ in range(number)
|
207
|
+
]
|
208
|
+
|
209
|
+
# Save.
|
210
|
+
self.futures.extend(futures)
|
211
|
+
|
212
|
+
return futures
|
213
|
+
|
214
|
+
|
498
215
|
def generate(
|
499
216
|
self,
|
500
217
|
timeout: float | None = None
|
@@ -508,85 +225,444 @@ class RThreadPool(object):
|
|
508
225
|
- `None`: Infinite.
|
509
226
|
- `float`: Set this seconds.
|
510
227
|
|
511
|
-
Returns
|
512
|
-
-------
|
513
|
-
Generator of added task instance.
|
514
|
-
"""
|
228
|
+
Returns
|
229
|
+
-------
|
230
|
+
Generator of added task instance.
|
231
|
+
"""
|
232
|
+
|
233
|
+
# Build.
|
234
|
+
generator = concurrent_as_completed(
|
235
|
+
self.futures,
|
236
|
+
timeout
|
237
|
+
)
|
238
|
+
|
239
|
+
return generator
|
240
|
+
|
241
|
+
|
242
|
+
def join(
|
243
|
+
self,
|
244
|
+
timeout: float | None = None
|
245
|
+
) -> None:
|
246
|
+
"""
|
247
|
+
Block until all tasks are done.
|
248
|
+
|
249
|
+
Parameters
|
250
|
+
----------
|
251
|
+
timeout : Call generator maximum waiting seconds, timeout throw exception.
|
252
|
+
- `None`: Infinite.
|
253
|
+
- `float`: Set this seconds.
|
254
|
+
"""
|
255
|
+
|
256
|
+
# Generator.
|
257
|
+
generator = self.generate(timeout)
|
258
|
+
|
259
|
+
# Wait.
|
260
|
+
for _ in generator:
|
261
|
+
pass
|
262
|
+
|
263
|
+
|
264
|
+
def __iter__(self) -> Generator:
|
265
|
+
"""
|
266
|
+
Return the generator of task result.
|
267
|
+
|
268
|
+
Returns
|
269
|
+
-------
|
270
|
+
Generator of task result.
|
271
|
+
"""
|
272
|
+
|
273
|
+
# Generator.
|
274
|
+
generator = self.generate()
|
275
|
+
self.futures.clear()
|
276
|
+
|
277
|
+
# Generate.
|
278
|
+
for future in generator:
|
279
|
+
yield future.result()
|
280
|
+
|
281
|
+
|
282
|
+
@property
|
283
|
+
def thread_id(self) -> int:
|
284
|
+
"""
|
285
|
+
Get current thread ID.
|
286
|
+
|
287
|
+
Returns
|
288
|
+
-------
|
289
|
+
Current thread ID.
|
290
|
+
"""
|
291
|
+
|
292
|
+
# Get.
|
293
|
+
thread_id = threading_get_ident()
|
294
|
+
|
295
|
+
return thread_id
|
296
|
+
|
297
|
+
|
298
|
+
__call__ = one
|
299
|
+
|
300
|
+
|
301
|
+
__mul__ = repeat
|
302
|
+
|
303
|
+
|
304
|
+
@overload
|
305
|
+
def async_run(
|
306
|
+
coroutine: Coroutine[Any, Any, T] | ATask[Any, Any, T] | Callable[[], Coroutine[Any, Any, T]],
|
307
|
+
*,
|
308
|
+
return_exceptions: bool = False
|
309
|
+
) -> T: ...
|
310
|
+
|
311
|
+
@overload
|
312
|
+
def async_run(
|
313
|
+
*coroutines: Coroutine[Any, Any, T] | ATask[Any, Any, T] | Callable[[], Coroutine[Any, Any, T]],
|
314
|
+
return_exceptions: bool = False
|
315
|
+
) -> list[T]: ...
|
316
|
+
|
317
|
+
def async_run(
|
318
|
+
*coroutines: Coroutine[Any, Any, T] | ATask[Any, Any, T] | Callable[[], Coroutine[Any, Any, T]],
|
319
|
+
return_exceptions: bool = False
|
320
|
+
) -> T | list[T]:
|
321
|
+
"""
|
322
|
+
Asynchronous run coroutines.
|
323
|
+
|
324
|
+
Parameters
|
325
|
+
----------
|
326
|
+
coroutines : `Coroutine` instances or `ATask` instances or `Coroutine` function.
|
327
|
+
return_exceptions : Whether return exception instances, otherwise throw first exception.
|
328
|
+
|
329
|
+
Returns
|
330
|
+
-------
|
331
|
+
run results.
|
332
|
+
"""
|
333
|
+
|
334
|
+
# Handle parameter.
|
335
|
+
coroutines = [
|
336
|
+
coroutine()
|
337
|
+
if asyncio_iscoroutinefunction(coroutine)
|
338
|
+
else coroutine
|
339
|
+
for coroutine in coroutines
|
340
|
+
]
|
341
|
+
|
342
|
+
# Define.
|
343
|
+
async def async_run_coroutine() -> list:
|
344
|
+
"""
|
345
|
+
Asynchronous run coroutines.
|
346
|
+
|
347
|
+
Returns
|
348
|
+
-------
|
349
|
+
Run result list.
|
350
|
+
"""
|
351
|
+
|
352
|
+
# Gather.
|
353
|
+
results = await asyncio_gather(*coroutines, return_exceptions=return_exceptions)
|
354
|
+
|
355
|
+
return results
|
356
|
+
|
357
|
+
|
358
|
+
# Run.
|
359
|
+
coroutine = async_run_coroutine()
|
360
|
+
results = asyncio_run(coroutine)
|
361
|
+
|
362
|
+
# One.
|
363
|
+
if len(results) == 1:
|
364
|
+
results = results[0]
|
365
|
+
|
366
|
+
return results
|
367
|
+
|
368
|
+
|
369
|
+
@overload
|
370
|
+
async def async_sleep(
|
371
|
+
*,
|
372
|
+
precision: None = None
|
373
|
+
) -> int: ...
|
374
|
+
|
375
|
+
@overload
|
376
|
+
async def async_sleep(
|
377
|
+
second: int,
|
378
|
+
*,
|
379
|
+
precision: None = None
|
380
|
+
) -> int: ...
|
381
|
+
|
382
|
+
@overload
|
383
|
+
async def async_sleep(
|
384
|
+
low: int = 0,
|
385
|
+
high: int = 10,
|
386
|
+
*,
|
387
|
+
precision: None = None
|
388
|
+
) -> int: ...
|
389
|
+
|
390
|
+
@overload
|
391
|
+
async def async_sleep(
|
392
|
+
*thresholds: float,
|
393
|
+
precision: None = None
|
394
|
+
) -> float: ...
|
395
|
+
|
396
|
+
@overload
|
397
|
+
async def async_sleep(
|
398
|
+
*thresholds: float,
|
399
|
+
precision: Literal[0] = None
|
400
|
+
) -> int: ...
|
401
|
+
|
402
|
+
@overload
|
403
|
+
async def async_sleep(
|
404
|
+
*thresholds: float,
|
405
|
+
precision: int = None
|
406
|
+
) -> float: ...
|
407
|
+
|
408
|
+
async def async_sleep(
|
409
|
+
*thresholds: float,
|
410
|
+
precision: int | None = None
|
411
|
+
) -> float:
|
412
|
+
"""
|
413
|
+
Sleep random seconds, in the coroutine.
|
414
|
+
|
415
|
+
Parameters
|
416
|
+
----------
|
417
|
+
thresholds : Low and high thresholds of random range, range contains thresholds.
|
418
|
+
- When `length is 0`, then low and high thresholds is `0` and `10`.
|
419
|
+
- When `length is 1`, then sleep this value.
|
420
|
+
- When `length is 2`, then low and high thresholds is `thresholds[0]` and `thresholds[1]`.
|
421
|
+
|
422
|
+
precision : Precision of random range, that is maximum decimal digits of sleep seconds.
|
423
|
+
- `None`: Set to Maximum decimal digits of element of parameter `thresholds`.
|
424
|
+
- `int`: Set to this value.
|
425
|
+
|
426
|
+
Returns
|
427
|
+
-------
|
428
|
+
Random seconds.
|
429
|
+
- When parameters `precision` is `0`, then return int.
|
430
|
+
- When parameters `precision` is `greater than 0`, then return float.
|
431
|
+
"""
|
432
|
+
|
433
|
+
# Handle parameter.
|
434
|
+
if len(thresholds) == 1:
|
435
|
+
second = thresholds[0]
|
436
|
+
else:
|
437
|
+
second = randn(*thresholds, precision=precision)
|
438
|
+
|
439
|
+
# Sleep.
|
440
|
+
await asyncio_sleep(second)
|
441
|
+
|
442
|
+
return second
|
443
|
+
|
444
|
+
|
445
|
+
async def async_wait(
|
446
|
+
func: Callable[..., bool],
|
447
|
+
*args: Any,
|
448
|
+
_interval: float = 1,
|
449
|
+
_timeout: float | None = None,
|
450
|
+
_raising: bool = True,
|
451
|
+
**kwargs: Any
|
452
|
+
) -> float | None:
|
453
|
+
"""
|
454
|
+
Wait success.
|
455
|
+
|
456
|
+
Parameters
|
457
|
+
----------
|
458
|
+
func : Function to be decorated, must return `bool` value.
|
459
|
+
args : Position arguments of decorated function.
|
460
|
+
_interval : Interval seconds.
|
461
|
+
_timeout : Timeout seconds, timeout throw exception.
|
462
|
+
- `None`: Infinite time.
|
463
|
+
- `float`: Use this time.
|
464
|
+
_raising : When timeout, whether throw exception, otherwise return None.
|
465
|
+
kwargs : Keyword arguments of decorated function.
|
466
|
+
|
467
|
+
Returns
|
468
|
+
-------
|
469
|
+
Total spend seconds or None.
|
470
|
+
"""
|
471
|
+
|
472
|
+
# Set parameter.
|
473
|
+
rtm = RTimeMark()
|
474
|
+
rtm()
|
475
|
+
|
476
|
+
# Not set timeout.
|
477
|
+
if _timeout is None:
|
478
|
+
|
479
|
+
## Wait.
|
480
|
+
while True:
|
481
|
+
success = func(*args, **kwargs)
|
482
|
+
if success: break
|
483
|
+
await async_sleep(_interval)
|
484
|
+
|
485
|
+
# Set timeout.
|
486
|
+
else:
|
487
|
+
|
488
|
+
## Wait.
|
489
|
+
while True:
|
490
|
+
success = func(*args, **kwargs)
|
491
|
+
if success: break
|
492
|
+
|
493
|
+
## Timeout.
|
494
|
+
rtm()
|
495
|
+
if rtm.total_spend > _timeout:
|
496
|
+
|
497
|
+
### Throw exception.
|
498
|
+
if _raising:
|
499
|
+
throw(TimeoutError, _timeout)
|
500
|
+
|
501
|
+
return
|
502
|
+
|
503
|
+
## Sleep.
|
504
|
+
await async_sleep(_interval)
|
515
505
|
|
516
|
-
|
517
|
-
|
506
|
+
## Return.
|
507
|
+
rtm()
|
508
|
+
return rtm.total_spend
|
518
509
|
|
519
|
-
# Build.
|
520
|
-
generator = concurrent_as_completed(
|
521
|
-
futures,
|
522
|
-
timeout
|
523
|
-
)
|
524
510
|
|
525
|
-
|
511
|
+
async def async_request(
|
512
|
+
url: str,
|
513
|
+
params: dict | None = None,
|
514
|
+
data: dict | str | bytes | None = None,
|
515
|
+
json: dict | None = None,
|
516
|
+
headers: dict[str, str] = {},
|
517
|
+
timeout: float | None = None,
|
518
|
+
proxy: str | None = None,
|
519
|
+
method: Literal['get', 'post', 'put', 'patch', 'delete', 'options', 'head'] | None = None,
|
520
|
+
check: bool | int | Iterable[int] = False,
|
521
|
+
handler: str | tuple[str] | Callable[[ClientResponse], Coroutine | Any] | None = None
|
522
|
+
) -> Any:
|
523
|
+
"""
|
524
|
+
Get asynchronous `Coroutine` instance of send request.
|
526
525
|
|
526
|
+
Parameters
|
527
|
+
----------
|
528
|
+
url : Request URL.
|
529
|
+
params : Request URL add parameters.
|
530
|
+
data : Request body data. Conflict with parameter `json`.
|
531
|
+
- `dict`, Convert to `key=value&...`: format bytes.
|
532
|
+
Automatic set `Content-Type` to `application/x-www-form-urlencoded`.
|
533
|
+
- `dict and a certain value is 'bytes' type`: Key is parameter name and file name, value is file data.
|
534
|
+
Automatic set `Content-Type` to `multipart/form-data`.
|
535
|
+
- `str`: File path to read file bytes data.
|
536
|
+
Automatic set `Content-Type` to file media type, and `filename` to file name.
|
537
|
+
- `bytes`: File bytes data.
|
538
|
+
Automatic set `Content-Type` to file media type.
|
539
|
+
json : Request body data, convert to `JSON` format. Conflict with parameter `data`.
|
540
|
+
Automatic set `Content-Type` to `application/json`.
|
541
|
+
headers : Request header data.
|
542
|
+
timeout : Request maximun waiting time.
|
543
|
+
proxy : Proxy URL.
|
544
|
+
method : Request method.
|
545
|
+
- `None`: Automatic judge.
|
546
|
+
When parameter `data` or `json` not has value, then request method is `get`.
|
547
|
+
When parameter `data` or `json` has value, then request method is `post`.
|
548
|
+
- `Literal['get', 'post', 'put', 'patch', 'delete', 'options', 'head']`: Use this request method.
|
549
|
+
check : Check response code, and throw exception.
|
550
|
+
- `Literal[False]`: Not check.
|
551
|
+
- `Literal[True]`: Check if is between 200 and 299.
|
552
|
+
- `int`: Check if is this value.
|
553
|
+
- `Iterable`: Check if is in sequence.
|
554
|
+
handler : Response handler.
|
555
|
+
- `None`: Automatic handle.
|
556
|
+
`Response 'Content-Type' is 'application/json'`: Use `ClientResponse.json` method.
|
557
|
+
`Response 'Content-Type' is 'text/plain; charset=utf-8'`: Use `ClientResponse.text` method.
|
558
|
+
`Other`: Use `ClientResponse.read` method.
|
559
|
+
- `str`: Get this attribute.
|
560
|
+
`Callable`: Execute this method. When return `Coroutine`, then use `await` syntax execute `Coroutine`.
|
561
|
+
`Any`: Return this value.
|
562
|
+
- `tuple[str]`: Get these attribute.
|
563
|
+
`Callable`: Execute this method. When return `Coroutine`, then use `await` syntax execute `Coroutine`.
|
564
|
+
`Any`: Return this value.
|
565
|
+
- `Callable`, Execute this method. When return `Coroutine`, then use `await`: syntax execute `Coroutine`.
|
527
566
|
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
"""
|
533
|
-
Add and start a batch of tasks to the thread pool, and only with default parameters.
|
567
|
+
Returns
|
568
|
+
-------
|
569
|
+
Response handler result.
|
570
|
+
"""
|
534
571
|
|
535
|
-
|
536
|
-
|
537
|
-
number : Number of add.
|
572
|
+
# Check.
|
573
|
+
check_most_one(data, json)
|
538
574
|
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
|
575
|
+
# Handle parameter.
|
576
|
+
if method is None:
|
577
|
+
if data is None and json is None:
|
578
|
+
method = 'get'
|
579
|
+
else:
|
580
|
+
method = 'post'
|
543
581
|
|
544
|
-
|
545
|
-
|
546
|
-
self.one()
|
547
|
-
for _ in range(number)
|
548
|
-
]
|
582
|
+
# Session.
|
583
|
+
async with ClientSession() as session:
|
549
584
|
|
550
|
-
#
|
551
|
-
|
585
|
+
# Request.
|
586
|
+
async with session.request(
|
587
|
+
method,
|
588
|
+
url,
|
589
|
+
params=params,
|
590
|
+
data=data,
|
591
|
+
json=json,
|
592
|
+
headers=headers,
|
593
|
+
timeout=timeout,
|
594
|
+
proxy=proxy
|
595
|
+
) as response:
|
552
596
|
|
553
|
-
|
597
|
+
# Check code.
|
598
|
+
if check is not False:
|
599
|
+
if check is True:
|
600
|
+
range_ = None
|
601
|
+
else:
|
602
|
+
range_ = check
|
603
|
+
check_response_code(response.status, range_)
|
554
604
|
|
605
|
+
# Receive.
|
606
|
+
match handler:
|
555
607
|
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
608
|
+
## Auto.
|
609
|
+
case None:
|
610
|
+
match response.content_type:
|
611
|
+
case 'application/json':
|
612
|
+
result = await response.json()
|
613
|
+
case 'text/plain; charset=utf-8':
|
560
614
|
|
561
|
-
|
562
|
-
|
615
|
+
# Set encode type.
|
616
|
+
if response.get_encoding() == 'ISO-8859-1':
|
617
|
+
encoding = 'utf-8'
|
618
|
+
else:
|
619
|
+
encoding = None
|
563
620
|
|
564
|
-
|
565
|
-
|
566
|
-
|
621
|
+
result = await response.text(encoding=encoding)
|
622
|
+
case _:
|
623
|
+
result = await response.read()
|
567
624
|
|
625
|
+
## Attribute.
|
626
|
+
case str():
|
627
|
+
result = getattr(response, handler)
|
568
628
|
|
569
|
-
|
570
|
-
|
571
|
-
|
629
|
+
### Method.
|
630
|
+
if callable(result):
|
631
|
+
result = result()
|
572
632
|
|
573
|
-
|
574
|
-
|
575
|
-
|
576
|
-
"""
|
633
|
+
#### Coroutine.
|
634
|
+
if asyncio_iscoroutine(result):
|
635
|
+
result = await result
|
577
636
|
|
578
|
-
|
579
|
-
|
637
|
+
## Attributes.
|
638
|
+
case tuple():
|
639
|
+
result = []
|
640
|
+
for key in handler:
|
641
|
+
result_element = getattr(response, key)
|
580
642
|
|
581
|
-
|
582
|
-
|
583
|
-
|
643
|
+
### Method.
|
644
|
+
if callable(result_element):
|
645
|
+
result_element = result_element()
|
584
646
|
|
647
|
+
#### Coroutine.
|
648
|
+
if asyncio_iscoroutine(result_element):
|
649
|
+
result_element = await result_element
|
585
650
|
|
586
|
-
|
651
|
+
result.append(result_element)
|
652
|
+
|
653
|
+
## Method.
|
654
|
+
case _ if callable(handler):
|
655
|
+
result = handler(response)
|
587
656
|
|
657
|
+
### Coroutine.
|
658
|
+
if asyncio_iscoroutine(result):
|
659
|
+
result = await result
|
588
660
|
|
589
|
-
|
661
|
+
## Throw exception.
|
662
|
+
case _:
|
663
|
+
throw(TypeError, handler)
|
664
|
+
|
665
|
+
return result
|
590
666
|
|
591
667
|
|
592
668
|
class RAsyncPool(object):
|
@@ -594,13 +670,14 @@ class RAsyncPool(object):
|
|
594
670
|
Rey's `asynchronous pool` type.
|
595
671
|
"""
|
596
672
|
|
673
|
+
Queue = AQueue
|
674
|
+
Lock = ALock
|
675
|
+
|
597
676
|
|
598
677
|
def __init__(
|
599
678
|
self,
|
600
|
-
|
679
|
+
task: Callable[..., Coroutine],
|
601
680
|
*args: Any,
|
602
|
-
_max_async: int = 10,
|
603
|
-
_exc_handler: Callable | None = None,
|
604
681
|
**kwargs: Any
|
605
682
|
) -> None:
|
606
683
|
"""
|
@@ -610,76 +687,31 @@ class RAsyncPool(object):
|
|
610
687
|
----------
|
611
688
|
async_func : Function of create asynchronous `Coroutine`.
|
612
689
|
args : Function default position arguments.
|
613
|
-
_max_async : Maximum number of asynchronous.
|
614
|
-
_exc_handler : `Coroutine` execution exception handler, will return value.
|
615
690
|
kwargs : Function default keyword arguments.
|
616
691
|
"""
|
617
692
|
|
618
693
|
# Set attribute.
|
619
|
-
self.
|
694
|
+
self.task = task
|
620
695
|
self.args = args
|
621
696
|
self.kwargs = kwargs
|
622
|
-
self.
|
623
|
-
self.
|
624
|
-
self.queue_output = AQueue()
|
625
|
-
self.queue_count = 0
|
697
|
+
self.loop = asyncio_new_event_loop()
|
698
|
+
self.futures: list[AFuture] = []
|
626
699
|
|
627
700
|
# Start.
|
628
|
-
self.
|
701
|
+
self._start_loop()
|
629
702
|
|
630
703
|
|
631
704
|
@wrap_thread
|
632
|
-
def
|
633
|
-
self,
|
634
|
-
worker_n: int
|
635
|
-
) -> None:
|
705
|
+
def _start_loop(self) -> None:
|
636
706
|
"""
|
637
|
-
Start
|
638
|
-
|
639
|
-
Parameters
|
640
|
-
----------
|
641
|
-
worker_n : Number of execute asynchronous `Coroutine` workers.
|
707
|
+
Start event loop.
|
642
708
|
"""
|
643
709
|
|
710
|
+
# Set.
|
711
|
+
asyncio_set_event_loop(self.loop)
|
644
712
|
|
645
|
-
|
646
|
-
|
647
|
-
"""
|
648
|
-
Worker of execute asynchronous `Coroutine`.
|
649
|
-
"""
|
650
|
-
|
651
|
-
# Loop.
|
652
|
-
while True:
|
653
|
-
|
654
|
-
# Get parameter.
|
655
|
-
args, kwargs = await self.queue_input.get()
|
656
|
-
|
657
|
-
# Execute.
|
658
|
-
try:
|
659
|
-
result = await self.async_func(*args, **kwargs)
|
660
|
-
|
661
|
-
# Handle exception.
|
662
|
-
except:
|
663
|
-
if self.exc_handler is not None:
|
664
|
-
result = self.exc_handler()
|
665
|
-
await self.queue_output.put(result)
|
666
|
-
|
667
|
-
## Count.
|
668
|
-
else:
|
669
|
-
self.queue_count -= 1
|
670
|
-
|
671
|
-
else:
|
672
|
-
await self.queue_output.put(result)
|
673
|
-
|
674
|
-
|
675
|
-
# Create.
|
676
|
-
coroutines = [
|
677
|
-
async_worker()
|
678
|
-
for _ in range(worker_n)
|
679
|
-
]
|
680
|
-
|
681
|
-
# Start.
|
682
|
-
async_run(*coroutines)
|
713
|
+
## Start and block.
|
714
|
+
self.loop.run_forever()
|
683
715
|
|
684
716
|
|
685
717
|
def one(
|
@@ -688,7 +720,7 @@ class RAsyncPool(object):
|
|
688
720
|
**kwargs: Any
|
689
721
|
) -> None:
|
690
722
|
"""
|
691
|
-
|
723
|
+
Start a task.
|
692
724
|
|
693
725
|
Parameters
|
694
726
|
----------
|
@@ -705,16 +737,15 @@ class RAsyncPool(object):
|
|
705
737
|
**self.kwargs,
|
706
738
|
**kwargs
|
707
739
|
}
|
708
|
-
item = (
|
709
|
-
func_args,
|
710
|
-
func_kwargs
|
711
|
-
)
|
712
740
|
|
713
|
-
#
|
714
|
-
self.
|
741
|
+
# Create.
|
742
|
+
coroutine = self.task(*func_args, **func_kwargs)
|
743
|
+
|
744
|
+
# Add.
|
745
|
+
future = asyncio_run_coroutine_threadsafe(coroutine, self.loop)
|
715
746
|
|
716
|
-
#
|
717
|
-
self.
|
747
|
+
# Save.
|
748
|
+
self.futures.append(future)
|
718
749
|
|
719
750
|
|
720
751
|
def batch(
|
@@ -723,7 +754,7 @@ class RAsyncPool(object):
|
|
723
754
|
**kwargs: tuple
|
724
755
|
) -> None:
|
725
756
|
"""
|
726
|
-
|
757
|
+
Batch start tasks.
|
727
758
|
parameters sequence will combine one by one, and discard excess parameters.
|
728
759
|
|
729
760
|
Parameters
|
@@ -758,7 +789,7 @@ class RAsyncPool(object):
|
|
758
789
|
)
|
759
790
|
params_zip = zip(args_zip, kwargs_zip)
|
760
791
|
|
761
|
-
# Batch
|
792
|
+
# Batch add.
|
762
793
|
for args_, kwargs_ in params_zip:
|
763
794
|
self.one(*args_, **dict(kwargs_))
|
764
795
|
|
@@ -768,100 +799,92 @@ class RAsyncPool(object):
|
|
768
799
|
number: int
|
769
800
|
) -> list[CFuture]:
|
770
801
|
"""
|
771
|
-
|
802
|
+
Batch start tasks, and only with default parameters.
|
772
803
|
|
773
804
|
Parameters
|
774
805
|
----------
|
775
806
|
number : Number of add.
|
776
807
|
"""
|
777
808
|
|
778
|
-
# Batch
|
809
|
+
# Batch add.
|
779
810
|
for _ in range(number):
|
780
811
|
self.one()
|
781
812
|
|
782
813
|
|
783
|
-
def
|
814
|
+
def generate(
|
784
815
|
self,
|
785
816
|
timeout: float | None = None
|
786
|
-
) ->
|
817
|
+
) -> Generator[CFuture]:
|
787
818
|
"""
|
788
|
-
|
819
|
+
Return the generator of added task instance.
|
789
820
|
|
790
821
|
Parameters
|
791
822
|
----------
|
792
|
-
timeout :
|
823
|
+
timeout : Call generator maximum waiting seconds, timeout throw exception.
|
824
|
+
- `None`: Infinite.
|
825
|
+
- `float`: Set this seconds.
|
793
826
|
|
794
827
|
Returns
|
795
828
|
-------
|
796
|
-
|
829
|
+
Generator of added task instance.
|
797
830
|
"""
|
798
831
|
|
799
|
-
#
|
800
|
-
|
801
|
-
|
802
|
-
|
803
|
-
|
804
|
-
# Loop.
|
805
|
-
while True:
|
806
|
-
|
807
|
-
# Judge.
|
808
|
-
if not self.queue_output.empty():
|
809
|
-
|
810
|
-
# Get.
|
811
|
-
try:
|
812
|
-
result = self.queue_output.get_nowait()
|
813
|
-
except QueueEmpty:
|
814
|
-
pass
|
815
|
-
else:
|
816
|
-
|
817
|
-
# Count.
|
818
|
-
self.queue_count -= 1
|
819
|
-
|
820
|
-
return result
|
821
|
-
|
822
|
-
# Timeout.
|
823
|
-
if timeout is not None:
|
824
|
-
rtm()
|
825
|
-
if rtm.total_spend > timeout:
|
826
|
-
throw(TimeoutError, timeout)
|
832
|
+
# Build.
|
833
|
+
generator = concurrent_as_completed(
|
834
|
+
self.futures,
|
835
|
+
timeout
|
836
|
+
)
|
827
837
|
|
828
|
-
|
829
|
-
sleep(0.01)
|
838
|
+
return generator
|
830
839
|
|
831
840
|
|
832
|
-
def join(
|
833
|
-
|
834
|
-
|
841
|
+
def join(
|
842
|
+
self,
|
843
|
+
timeout: float | None = None
|
844
|
+
) -> None:
|
835
845
|
"""
|
846
|
+
Block until all tasks are done.
|
836
847
|
|
837
|
-
|
838
|
-
|
848
|
+
Parameters
|
849
|
+
----------
|
850
|
+
timeout : Call generator maximum waiting seconds, timeout throw exception.
|
851
|
+
- `None`: Infinite.
|
852
|
+
- `float`: Set this seconds.
|
853
|
+
"""
|
839
854
|
|
840
|
-
|
841
|
-
|
842
|
-
break
|
855
|
+
# Generator.
|
856
|
+
generator = self.generate(timeout)
|
843
857
|
|
844
|
-
|
858
|
+
# Wait.
|
859
|
+
for _ in generator:
|
860
|
+
pass
|
845
861
|
|
846
862
|
|
847
863
|
def __iter__(self) -> Generator:
|
848
864
|
"""
|
849
|
-
Return the generator of result
|
865
|
+
Return the generator of task result.
|
850
866
|
|
851
867
|
Returns
|
852
868
|
-------
|
853
|
-
Generator of result
|
869
|
+
Generator of task result.
|
854
870
|
"""
|
855
871
|
|
872
|
+
# Generator.
|
873
|
+
generator = self.generate()
|
874
|
+
self.futures.clear()
|
875
|
+
|
856
876
|
# Generate.
|
857
|
-
|
877
|
+
for future in generator:
|
878
|
+
yield future.result()
|
879
|
+
|
858
880
|
|
859
|
-
|
860
|
-
|
861
|
-
|
881
|
+
def __del__(self) -> None:
|
882
|
+
"""
|
883
|
+
End loop.
|
884
|
+
"""
|
862
885
|
|
863
|
-
|
864
|
-
|
886
|
+
# Stop.
|
887
|
+
self.loop.stop()
|
865
888
|
|
866
889
|
|
867
890
|
__call__ = one
|