fluentmap 0.1.1__tar.gz → 0.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: fluentmap
3
- Version: 0.1.1
3
+ Version: 0.1.2
4
4
  Summary: A drop-in Python map replacement featuring parallel and batch processing.
5
5
  Author: leavers
6
6
  Author-Email: Leavers <leavers930@gmail.com>
@@ -39,6 +39,7 @@ processing.
39
39
  - Use `batch_size`/`chunk_size` to send parameters in batches/chunks.
40
40
  - Use `num_prepare` to prepare data in advance for better performance.
41
41
  - Call `on_return` hook to process the return value of each task.
42
+ - Support the builtin `strict` keyword.
42
43
 
43
44
  ## Installation
44
45
 
@@ -13,6 +13,7 @@ processing.
13
13
  - Use `batch_size`/`chunk_size` to send parameters in batches/chunks.
14
14
  - Use `num_prepare` to prepare data in advance for better performance.
15
15
  - Call `on_return` hook to process the return value of each task.
16
+ - Support the builtin `strict` keyword.
16
17
 
17
18
  ## Installation
18
19
 
@@ -26,7 +26,7 @@ from typing import (
26
26
  runtime_checkable,
27
27
  )
28
28
 
29
- __version__ = "0.1.1"
29
+ __version__ = "0.1.2"
30
30
  __all__ = ("Arguments", "map")
31
31
 
32
32
 
@@ -61,6 +61,42 @@ class Arguments:
61
61
  return f"{self.__class__.__name__}(args={self.args}, kwargs={self.kwargs})"
62
62
 
63
63
 
64
+ def _zip_args(
65
+ args: Iterable[Any],
66
+ *more_args: Iterable[Any],
67
+ strict: bool = False,
68
+ ) -> Iterator[Tuple[Any, ...]]:
69
+ if not strict:
70
+ yield from zip(args, *more_args)
71
+ return
72
+
73
+ # Same checks and messages as the strict builtin map (zip(strict=True) is only
74
+ # available on Python 3.10+).
75
+ iterators = [iter(args), *(iter(it) for it in more_args)]
76
+ while True:
77
+ values = []
78
+ for i, iterator in enumerate(iterators):
79
+ try:
80
+ values.append(next(iterator))
81
+ except StopIteration:
82
+ if i:
83
+ plural = " " if i == 1 else "s 1-"
84
+ raise ValueError(
85
+ f"map() argument {i + 1} is shorter than argument{plural}{i}"
86
+ ) from None
87
+ for j, other in enumerate(iterators[1:], start=1):
88
+ try:
89
+ next(other)
90
+ except StopIteration:
91
+ continue
92
+ plural = " " if j == 1 else "s 1-"
93
+ raise ValueError(
94
+ f"map() argument {j + 1} is longer than argument{plural}{j}"
95
+ ) from None
96
+ return
97
+ yield tuple(values)
98
+
99
+
64
100
  def _simple_map(
65
101
  fn: Callable[..., T],
66
102
  args: Iterable[Any],
@@ -68,11 +104,12 @@ def _simple_map(
68
104
  batch_size: int = 0,
69
105
  num_prepare: int = 0,
70
106
  on_return: Optional[Callable[[T], R]] = None,
107
+ strict: bool = False,
71
108
  ) -> Iterator[Union[T, R]]:
72
109
  is_batch_args = True if batch_size > 1 else False
73
110
 
74
111
  if more_args:
75
- iterable_args = cast(Iterable[Any], zip(args, *more_args))
112
+ iterable_args = cast(Iterable[Any], _zip_args(args, *more_args, strict=strict))
76
113
  extract_args = True
77
114
  else:
78
115
  iterable_args = args
@@ -109,28 +146,33 @@ def _simple_map(
109
146
  undefined = object()
110
147
  queue: Queue = Queue()
111
148
  exc_event = Event()
149
+ worker_exc: List[BaseException] = []
112
150
 
113
151
  def worker():
114
152
  batch: List[Any] = []
115
- for args in iterable_args:
116
- if exc_event.is_set():
117
- break
118
- if is_batch_args:
119
- batch.append(args)
120
- if len(batch) < batch_size:
121
- continue
122
- queue.put(batch)
123
- batch = []
124
- else:
125
- queue.put(args)
126
- del args
127
- while queue.qsize() >= num_prepare:
153
+ try:
154
+ for args in iterable_args:
128
155
  if exc_event.is_set():
129
156
  break
130
- exc_event.wait(0.05)
131
- if not exc_event.is_set() and batch:
132
- queue.put(batch)
133
- queue.put(undefined)
157
+ if is_batch_args:
158
+ batch.append(args)
159
+ if len(batch) < batch_size:
160
+ continue
161
+ queue.put(batch)
162
+ batch = []
163
+ else:
164
+ queue.put(args)
165
+ del args
166
+ while queue.qsize() >= num_prepare:
167
+ if exc_event.is_set():
168
+ break
169
+ exc_event.wait(0.05)
170
+ if not exc_event.is_set() and batch:
171
+ queue.put(batch)
172
+ except BaseException as exc:
173
+ worker_exc.append(exc)
174
+ finally:
175
+ queue.put(undefined)
134
176
 
135
177
  worker_thread = Thread(target=worker, daemon=True)
136
178
  worker_thread.start()
@@ -140,6 +182,8 @@ def _simple_map(
140
182
  except Empty:
141
183
  continue
142
184
  if data is undefined:
185
+ if worker_exc:
186
+ raise worker_exc[0]
143
187
  break
144
188
  try:
145
189
  yield submit(data)
@@ -194,13 +238,14 @@ def _concurrent_chunk_map(
194
238
  num_prepare: int = 0,
195
239
  sort_by_completion: bool = False,
196
240
  on_return: Optional[Callable[[T], R]] = None,
241
+ strict: bool = False,
197
242
  ) -> Iterator[Union[T, R]]:
198
243
  assert chunk_size > 1
199
244
  is_batch_args = True if batch_size > 1 else False
200
245
 
201
246
  args_iter: Iterator[Any]
202
247
  if more_args:
203
- args_iter = iter(zip(args, *more_args))
248
+ args_iter = iter(_zip_args(args, *more_args, strict=strict))
204
249
 
205
250
  # Whether to deconstruct iterable args in submit, this helps sumbit identify
206
251
  # whether an iterable fn_args is a batch generated by _concurrent_map itself
@@ -423,12 +468,13 @@ def _concurrent_map(
423
468
  num_prepare: int = 0,
424
469
  sort_by_completion: bool = False,
425
470
  on_return: Optional[Callable[[T], R]] = None,
471
+ strict: bool = False,
426
472
  ) -> Iterator[Union[T, R]]:
427
473
  is_batch_args = True if batch_size > 1 else False
428
474
 
429
475
  args_iter: Iterator[Any]
430
476
  if more_args:
431
- args_iter = iter(zip(args, *more_args))
477
+ args_iter = iter(_zip_args(args, *more_args, strict=strict))
432
478
 
433
479
  # Whether to deconstruct iterable args in submit, this helps sumbit identify
434
480
  # whether an iterable fn_args is a batch generated by _concurrent_map itself
@@ -623,8 +669,10 @@ def map(
623
669
  *more_args: Iterable[Any],
624
670
  executor: Optional[SupportsSubmit] = None,
625
671
  batch_size: Optional[int] = None,
672
+ chunk_size: Optional[int] = None,
626
673
  num_prepare: Optional[int] = None,
627
674
  sort_by_completion: bool = False,
675
+ strict: bool = False,
628
676
  ) -> Iterator[T]: ...
629
677
 
630
678
 
@@ -635,9 +683,11 @@ def map(
635
683
  *more_args: Iterable[Any],
636
684
  executor: Optional[SupportsSubmit] = None,
637
685
  batch_size: Optional[int] = None,
686
+ chunk_size: Optional[int] = None,
638
687
  num_prepare: Optional[int] = None,
639
688
  sort_by_completion: bool = False,
640
689
  on_return: Callable[[T], R],
690
+ strict: bool = False,
641
691
  ) -> Iterator[R]: ...
642
692
 
643
693
 
@@ -651,6 +701,7 @@ def map(
651
701
  num_prepare: Optional[int] = None,
652
702
  sort_by_completion: bool = False,
653
703
  on_return: Optional[Callable[[T], R]] = None,
704
+ strict: bool = False,
654
705
  ) -> Iterator[Union[T, R]]:
655
706
  """Drop-in replacement of builtin ``map`` but adds an argument ``executor`` for
656
707
  executing function in an executor.
@@ -718,6 +769,10 @@ def map(
718
769
  It accept each result as its sole argument and its return will be treated as
719
770
  as the final return of each invocation of ``fn``.
720
771
 
772
+ :type strict: bool, optional, default to ``False``
773
+ :param strict: If set to ``True``, raise ``ValueError`` when the iterables have
774
+ different lengths, same as the strict builtin ``map``.
775
+
721
776
  :rtype: Iterator
722
777
  :return: An iterator which yield all results of ``fn``.
723
778
  """
@@ -741,6 +796,7 @@ def map(
741
796
  batch_size=batch_size,
742
797
  num_prepare=num_prepare,
743
798
  on_return=on_return,
799
+ strict=strict,
744
800
  )
745
801
 
746
802
  if chunk_size is None or chunk_size <= 1:
@@ -753,6 +809,7 @@ def map(
753
809
  num_prepare=num_prepare,
754
810
  sort_by_completion=sort_by_completion,
755
811
  on_return=on_return,
812
+ strict=strict,
756
813
  )
757
814
  return _concurrent_chunk_map(
758
815
  fn,
@@ -764,4 +821,5 @@ def map(
764
821
  num_prepare=num_prepare,
765
822
  sort_by_completion=sort_by_completion,
766
823
  on_return=on_return,
824
+ strict=strict,
767
825
  )
@@ -27,7 +27,7 @@ dependencies = []
27
27
  requires-python = ">=3.8"
28
28
  readme = "README.md"
29
29
  dynamic = []
30
- version = "0.1.1"
30
+ version = "0.1.2"
31
31
 
32
32
  [project.license]
33
33
  text = "MIT"
File without changes