fluentmap 0.1.1__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: fluentmap
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: A drop-in Python map replacement featuring parallel and batch processing.
|
|
5
5
|
Author: leavers
|
|
6
6
|
Author-Email: Leavers <leavers930@gmail.com>
|
|
@@ -39,6 +39,7 @@ processing.
|
|
|
39
39
|
- Use `batch_size`/`chunk_size` to send parameters in batches/chunks.
|
|
40
40
|
- Use `num_prepare` to prepare data in advance for better performance.
|
|
41
41
|
- Call `on_return` hook to process the return value of each task.
|
|
42
|
+
- Support the builtin `strict` keyword.
|
|
42
43
|
|
|
43
44
|
## Installation
|
|
44
45
|
|
|
@@ -13,6 +13,7 @@ processing.
|
|
|
13
13
|
- Use `batch_size`/`chunk_size` to send parameters in batches/chunks.
|
|
14
14
|
- Use `num_prepare` to prepare data in advance for better performance.
|
|
15
15
|
- Call `on_return` hook to process the return value of each task.
|
|
16
|
+
- Support the builtin `strict` keyword.
|
|
16
17
|
|
|
17
18
|
## Installation
|
|
18
19
|
|
|
@@ -26,7 +26,7 @@ from typing import (
|
|
|
26
26
|
runtime_checkable,
|
|
27
27
|
)
|
|
28
28
|
|
|
29
|
-
__version__ = "0.1.
|
|
29
|
+
__version__ = "0.1.2"
|
|
30
30
|
__all__ = ("Arguments", "map")
|
|
31
31
|
|
|
32
32
|
|
|
@@ -61,6 +61,42 @@ class Arguments:
|
|
|
61
61
|
return f"{self.__class__.__name__}(args={self.args}, kwargs={self.kwargs})"
|
|
62
62
|
|
|
63
63
|
|
|
64
|
+
def _zip_args(
|
|
65
|
+
args: Iterable[Any],
|
|
66
|
+
*more_args: Iterable[Any],
|
|
67
|
+
strict: bool = False,
|
|
68
|
+
) -> Iterator[Tuple[Any, ...]]:
|
|
69
|
+
if not strict:
|
|
70
|
+
yield from zip(args, *more_args)
|
|
71
|
+
return
|
|
72
|
+
|
|
73
|
+
# Same checks and messages as the strict builtin map (zip(strict=True) is only
|
|
74
|
+
# available on Python 3.10+).
|
|
75
|
+
iterators = [iter(args), *(iter(it) for it in more_args)]
|
|
76
|
+
while True:
|
|
77
|
+
values = []
|
|
78
|
+
for i, iterator in enumerate(iterators):
|
|
79
|
+
try:
|
|
80
|
+
values.append(next(iterator))
|
|
81
|
+
except StopIteration:
|
|
82
|
+
if i:
|
|
83
|
+
plural = " " if i == 1 else "s 1-"
|
|
84
|
+
raise ValueError(
|
|
85
|
+
f"map() argument {i + 1} is shorter than argument{plural}{i}"
|
|
86
|
+
) from None
|
|
87
|
+
for j, other in enumerate(iterators[1:], start=1):
|
|
88
|
+
try:
|
|
89
|
+
next(other)
|
|
90
|
+
except StopIteration:
|
|
91
|
+
continue
|
|
92
|
+
plural = " " if j == 1 else "s 1-"
|
|
93
|
+
raise ValueError(
|
|
94
|
+
f"map() argument {j + 1} is longer than argument{plural}{j}"
|
|
95
|
+
) from None
|
|
96
|
+
return
|
|
97
|
+
yield tuple(values)
|
|
98
|
+
|
|
99
|
+
|
|
64
100
|
def _simple_map(
|
|
65
101
|
fn: Callable[..., T],
|
|
66
102
|
args: Iterable[Any],
|
|
@@ -68,11 +104,12 @@ def _simple_map(
|
|
|
68
104
|
batch_size: int = 0,
|
|
69
105
|
num_prepare: int = 0,
|
|
70
106
|
on_return: Optional[Callable[[T], R]] = None,
|
|
107
|
+
strict: bool = False,
|
|
71
108
|
) -> Iterator[Union[T, R]]:
|
|
72
109
|
is_batch_args = True if batch_size > 1 else False
|
|
73
110
|
|
|
74
111
|
if more_args:
|
|
75
|
-
iterable_args = cast(Iterable[Any],
|
|
112
|
+
iterable_args = cast(Iterable[Any], _zip_args(args, *more_args, strict=strict))
|
|
76
113
|
extract_args = True
|
|
77
114
|
else:
|
|
78
115
|
iterable_args = args
|
|
@@ -109,28 +146,33 @@ def _simple_map(
|
|
|
109
146
|
undefined = object()
|
|
110
147
|
queue: Queue = Queue()
|
|
111
148
|
exc_event = Event()
|
|
149
|
+
worker_exc: List[BaseException] = []
|
|
112
150
|
|
|
113
151
|
def worker():
|
|
114
152
|
batch: List[Any] = []
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
break
|
|
118
|
-
if is_batch_args:
|
|
119
|
-
batch.append(args)
|
|
120
|
-
if len(batch) < batch_size:
|
|
121
|
-
continue
|
|
122
|
-
queue.put(batch)
|
|
123
|
-
batch = []
|
|
124
|
-
else:
|
|
125
|
-
queue.put(args)
|
|
126
|
-
del args
|
|
127
|
-
while queue.qsize() >= num_prepare:
|
|
153
|
+
try:
|
|
154
|
+
for args in iterable_args:
|
|
128
155
|
if exc_event.is_set():
|
|
129
156
|
break
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
157
|
+
if is_batch_args:
|
|
158
|
+
batch.append(args)
|
|
159
|
+
if len(batch) < batch_size:
|
|
160
|
+
continue
|
|
161
|
+
queue.put(batch)
|
|
162
|
+
batch = []
|
|
163
|
+
else:
|
|
164
|
+
queue.put(args)
|
|
165
|
+
del args
|
|
166
|
+
while queue.qsize() >= num_prepare:
|
|
167
|
+
if exc_event.is_set():
|
|
168
|
+
break
|
|
169
|
+
exc_event.wait(0.05)
|
|
170
|
+
if not exc_event.is_set() and batch:
|
|
171
|
+
queue.put(batch)
|
|
172
|
+
except BaseException as exc:
|
|
173
|
+
worker_exc.append(exc)
|
|
174
|
+
finally:
|
|
175
|
+
queue.put(undefined)
|
|
134
176
|
|
|
135
177
|
worker_thread = Thread(target=worker, daemon=True)
|
|
136
178
|
worker_thread.start()
|
|
@@ -140,6 +182,8 @@ def _simple_map(
|
|
|
140
182
|
except Empty:
|
|
141
183
|
continue
|
|
142
184
|
if data is undefined:
|
|
185
|
+
if worker_exc:
|
|
186
|
+
raise worker_exc[0]
|
|
143
187
|
break
|
|
144
188
|
try:
|
|
145
189
|
yield submit(data)
|
|
@@ -194,13 +238,14 @@ def _concurrent_chunk_map(
|
|
|
194
238
|
num_prepare: int = 0,
|
|
195
239
|
sort_by_completion: bool = False,
|
|
196
240
|
on_return: Optional[Callable[[T], R]] = None,
|
|
241
|
+
strict: bool = False,
|
|
197
242
|
) -> Iterator[Union[T, R]]:
|
|
198
243
|
assert chunk_size > 1
|
|
199
244
|
is_batch_args = True if batch_size > 1 else False
|
|
200
245
|
|
|
201
246
|
args_iter: Iterator[Any]
|
|
202
247
|
if more_args:
|
|
203
|
-
args_iter = iter(
|
|
248
|
+
args_iter = iter(_zip_args(args, *more_args, strict=strict))
|
|
204
249
|
|
|
205
250
|
# Whether to deconstruct iterable args in submit, this helps sumbit identify
|
|
206
251
|
# whether an iterable fn_args is a batch generated by _concurrent_map itself
|
|
@@ -423,12 +468,13 @@ def _concurrent_map(
|
|
|
423
468
|
num_prepare: int = 0,
|
|
424
469
|
sort_by_completion: bool = False,
|
|
425
470
|
on_return: Optional[Callable[[T], R]] = None,
|
|
471
|
+
strict: bool = False,
|
|
426
472
|
) -> Iterator[Union[T, R]]:
|
|
427
473
|
is_batch_args = True if batch_size > 1 else False
|
|
428
474
|
|
|
429
475
|
args_iter: Iterator[Any]
|
|
430
476
|
if more_args:
|
|
431
|
-
args_iter = iter(
|
|
477
|
+
args_iter = iter(_zip_args(args, *more_args, strict=strict))
|
|
432
478
|
|
|
433
479
|
# Whether to deconstruct iterable args in submit, this helps sumbit identify
|
|
434
480
|
# whether an iterable fn_args is a batch generated by _concurrent_map itself
|
|
@@ -623,8 +669,10 @@ def map(
|
|
|
623
669
|
*more_args: Iterable[Any],
|
|
624
670
|
executor: Optional[SupportsSubmit] = None,
|
|
625
671
|
batch_size: Optional[int] = None,
|
|
672
|
+
chunk_size: Optional[int] = None,
|
|
626
673
|
num_prepare: Optional[int] = None,
|
|
627
674
|
sort_by_completion: bool = False,
|
|
675
|
+
strict: bool = False,
|
|
628
676
|
) -> Iterator[T]: ...
|
|
629
677
|
|
|
630
678
|
|
|
@@ -635,9 +683,11 @@ def map(
|
|
|
635
683
|
*more_args: Iterable[Any],
|
|
636
684
|
executor: Optional[SupportsSubmit] = None,
|
|
637
685
|
batch_size: Optional[int] = None,
|
|
686
|
+
chunk_size: Optional[int] = None,
|
|
638
687
|
num_prepare: Optional[int] = None,
|
|
639
688
|
sort_by_completion: bool = False,
|
|
640
689
|
on_return: Callable[[T], R],
|
|
690
|
+
strict: bool = False,
|
|
641
691
|
) -> Iterator[R]: ...
|
|
642
692
|
|
|
643
693
|
|
|
@@ -651,6 +701,7 @@ def map(
|
|
|
651
701
|
num_prepare: Optional[int] = None,
|
|
652
702
|
sort_by_completion: bool = False,
|
|
653
703
|
on_return: Optional[Callable[[T], R]] = None,
|
|
704
|
+
strict: bool = False,
|
|
654
705
|
) -> Iterator[Union[T, R]]:
|
|
655
706
|
"""Drop-in replacement of builtin ``map`` but adds an argument ``executor`` for
|
|
656
707
|
executing function in an executor.
|
|
@@ -718,6 +769,10 @@ def map(
|
|
|
718
769
|
It accept each result as its sole argument and its return will be treated as
|
|
719
770
|
as the final return of each invocation of ``fn``.
|
|
720
771
|
|
|
772
|
+
:type strict: bool, optional, default to ``False``
|
|
773
|
+
:param strict: If set to ``True``, raise ``ValueError`` when the iterables have
|
|
774
|
+
different lengths, same as the strict builtin ``map``.
|
|
775
|
+
|
|
721
776
|
:rtype: Iterator
|
|
722
777
|
:return: An iterator which yield all results of ``fn``.
|
|
723
778
|
"""
|
|
@@ -741,6 +796,7 @@ def map(
|
|
|
741
796
|
batch_size=batch_size,
|
|
742
797
|
num_prepare=num_prepare,
|
|
743
798
|
on_return=on_return,
|
|
799
|
+
strict=strict,
|
|
744
800
|
)
|
|
745
801
|
|
|
746
802
|
if chunk_size is None or chunk_size <= 1:
|
|
@@ -753,6 +809,7 @@ def map(
|
|
|
753
809
|
num_prepare=num_prepare,
|
|
754
810
|
sort_by_completion=sort_by_completion,
|
|
755
811
|
on_return=on_return,
|
|
812
|
+
strict=strict,
|
|
756
813
|
)
|
|
757
814
|
return _concurrent_chunk_map(
|
|
758
815
|
fn,
|
|
@@ -764,4 +821,5 @@ def map(
|
|
|
764
821
|
num_prepare=num_prepare,
|
|
765
822
|
sort_by_completion=sort_by_completion,
|
|
766
823
|
on_return=on_return,
|
|
824
|
+
strict=strict,
|
|
767
825
|
)
|
|
File without changes
|