concurrent-iterator 0.4.0__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- concurrent_iterator-1.0.0/PKG-INFO +169 -0
- concurrent_iterator-1.0.0/README.md +142 -0
- concurrent_iterator-1.0.0/concurrent_iterator/__init__.py +91 -0
- concurrent_iterator-1.0.0/concurrent_iterator/_abc.py +95 -0
- concurrent_iterator-0.4.0/concurrent_iterator/utils.py → concurrent_iterator-1.0.0/concurrent_iterator/_internal.py +13 -2
- concurrent_iterator-1.0.0/concurrent_iterator/dummy.py +57 -0
- concurrent_iterator-1.0.0/concurrent_iterator/process.py +267 -0
- concurrent_iterator-1.0.0/concurrent_iterator/thread.py +204 -0
- concurrent_iterator-1.0.0/concurrent_iterator.egg-info/PKG-INFO +169 -0
- {concurrent_iterator-0.4.0 → concurrent_iterator-1.0.0}/concurrent_iterator.egg-info/SOURCES.txt +3 -3
- {concurrent_iterator-0.4.0 → concurrent_iterator-1.0.0}/pyproject.toml +9 -14
- concurrent_iterator-1.0.0/tests/test_abc.py +147 -0
- concurrent_iterator-1.0.0/tests/test_dummy.py +40 -0
- concurrent_iterator-1.0.0/tests/test_process.py +386 -0
- concurrent_iterator-1.0.0/tests/test_thread.py +140 -0
- concurrent_iterator-0.4.0/PKG-INFO +0 -91
- concurrent_iterator-0.4.0/README.md +0 -58
- concurrent_iterator-0.4.0/concurrent_iterator/__init__.py +0 -64
- concurrent_iterator-0.4.0/concurrent_iterator/dummy.py +0 -52
- concurrent_iterator-0.4.0/concurrent_iterator/process.py +0 -175
- concurrent_iterator-0.4.0/concurrent_iterator/thread.py +0 -138
- concurrent_iterator-0.4.0/concurrent_iterator.egg-info/PKG-INFO +0 -91
- concurrent_iterator-0.4.0/concurrent_iterator.egg-info/zip-safe +0 -1
- concurrent_iterator-0.4.0/setup.py +0 -33
- concurrent_iterator-0.4.0/tests/test_dummy.py +0 -24
- concurrent_iterator-0.4.0/tests/test_process.py +0 -150
- concurrent_iterator-0.4.0/tests/test_thread.py +0 -32
- {concurrent_iterator-0.4.0 → concurrent_iterator-1.0.0}/LICENSE +0 -0
- {concurrent_iterator-0.4.0 → concurrent_iterator-1.0.0}/concurrent_iterator/py.typed +0 -0
- {concurrent_iterator-0.4.0 → concurrent_iterator-1.0.0}/concurrent_iterator.egg-info/dependency_links.txt +0 -0
- {concurrent_iterator-0.4.0 → concurrent_iterator-1.0.0}/concurrent_iterator.egg-info/top_level.txt +0 -0
- {concurrent_iterator-0.4.0 → concurrent_iterator-1.0.0}/setup.cfg +0 -0
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: concurrent-iterator
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Classes to run producers (iterators) and consumers (coroutines) in a background thread/process.
|
|
5
|
+
Author-email: Javier Ruere <javier@ruere.com.ar>
|
|
6
|
+
License: LGPLv3
|
|
7
|
+
Project-URL: Homepage, https://github.com/jruere/concurrent-iterator
|
|
8
|
+
Keywords: concurrency,parallelism,iterator,iterable,pipeline
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
|
+
Classifier: Operating System :: POSIX
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# concurrent-iterator
|
|
29
|
+
|
|
30
|
+
[](https://pypi.org/pypi/concurrent-iterator/)
|
|
31
|
+
[](https://pypi.org/pypi/concurrent-iterator/)
|
|
32
|
+
|
|
33
|
+
## Intro
|
|
34
|
+
|
|
35
|
+
Classes to run Producers (iterators) and Consumers (coroutines)
|
|
36
|
+
introducing concurrency.
|
|
37
|
+
|
|
38
|
+
There are many libraries to create pipelines with stages running in
|
|
39
|
+
separate processes, a nice one is
|
|
40
|
+
[parallelpipe](https://pypi.org/project/parallelpipe/), but this library
|
|
41
|
+
does something different. It runs the entire upstream iterable (the
|
|
42
|
+
Producer) in a background thread or process, while the downstream
|
|
43
|
+
consumption code stays in the foreground. It's a more coarse library but
|
|
44
|
+
easier to integrate since things keep looking like normal generators.
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
```sh
|
|
49
|
+
pip install concurrent-iterator
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Implementations
|
|
53
|
+
|
|
54
|
+
There are three modules, each providing a `Producer`, a `Consumer`, and
|
|
55
|
+
(for `thread`) a `MultiProducer`:
|
|
56
|
+
|
|
57
|
+
* `dummy`: non-concurrent implementation, useful for testing or as a
|
|
58
|
+
drop-in replacement when concurrency is not needed.
|
|
59
|
+
* `thread`: uses a background thread to run the iterable or coroutine.
|
|
60
|
+
Useful for IO-bound work.
|
|
61
|
+
* `process`: uses a background process to run the iterable or coroutine.
|
|
62
|
+
Useful for CPU or IO-bound work. It has the complications of dealing
|
|
63
|
+
with processes (different memory spaces, logging, etc).
|
|
64
|
+
For logging, module
|
|
65
|
+
[`multiprocessing-logging`](https://github.com/jruere/multiprocessing-logging)
|
|
66
|
+
can be used.
|
|
67
|
+
|
|
68
|
+
All `Producer`, `MultiProducer`, and `Consumer` classes implement the
|
|
69
|
+
`IProducer` and `IConsumer` interfaces defined in `concurrent_iterator`
|
|
70
|
+
(the package root). They support:
|
|
71
|
+
|
|
72
|
+
* **Iteration** (`__next__` / `__iter__`) — Producers yield values from
|
|
73
|
+
the background iterable.
|
|
74
|
+
* **Sending** (`Consumer.send`) — Consumers forward values to the
|
|
75
|
+
background coroutine.
|
|
76
|
+
* **Explicit close** (`close()`) — stops the background worker and
|
|
77
|
+
releases resources.
|
|
78
|
+
* **State inspection** (`closed` property) — checks whether the worker
|
|
79
|
+
has been closed.
|
|
80
|
+
* **Context manager** (`with` statement) — `close()` is called
|
|
81
|
+
automatically on exit.
|
|
82
|
+
|
|
83
|
+
## Usage
|
|
84
|
+
|
|
85
|
+
### Producer
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
import time
|
|
89
|
+
from concurrent_iterator.thread import Producer
|
|
90
|
+
|
|
91
|
+
def slow_generator(count):
|
|
92
|
+
for i in range(count):
|
|
93
|
+
time.sleep(0.5) # Simulate a slow producer
|
|
94
|
+
yield i
|
|
95
|
+
|
|
96
|
+
# Pre-calculate up to 5 values while the main thread consumes them.
|
|
97
|
+
with Producer(slow_generator(10), maxsize=5) as items:
|
|
98
|
+
for item in items:
|
|
99
|
+
print(item) # Do some time-consuming task
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
In this example, while the main thread processes each `item`,
|
|
103
|
+
`slow_generator` continues running in a background thread and
|
|
104
|
+
pre-calculates up to 5 values.
|
|
105
|
+
The `with` statement ensures the background thread is properly cleaned
|
|
106
|
+
up.
|
|
107
|
+
|
|
108
|
+
### Consumer
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from concurrent_iterator.thread import Consumer
|
|
112
|
+
|
|
113
|
+
def worker():
|
|
114
|
+
while True:
|
|
115
|
+
value = yield # Coroutine protocol
|
|
116
|
+
# ... do work with value ...
|
|
117
|
+
|
|
118
|
+
coro = worker()
|
|
119
|
+
next(coro) # Prime the coroutine
|
|
120
|
+
|
|
121
|
+
with Consumer(coro, maxsize=5) as consumer:
|
|
122
|
+
consumer.send("a")
|
|
123
|
+
consumer.send("b")
|
|
124
|
+
# close() is called automatically on exit, which also closes the coroutine.
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The Consumer runs the coroutine in a background thread. `send()`
|
|
128
|
+
enqueues values; if the value cannot be enqueued within `timeout`
|
|
129
|
+
(default `0`, never blocks), it raises
|
|
130
|
+
`concurrent_iterator.WillNotConsume`, signaling the caller to retry.
|
|
131
|
+
|
|
132
|
+
### Context manager
|
|
133
|
+
|
|
134
|
+
Both Producers and Consumers can be used as context managers. When the
|
|
135
|
+
`with` block exits, `close()` is called automatically, ensuring
|
|
136
|
+
background threads or processes are terminated even on exceptions.
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
from concurrent_iterator.thread import Producer
|
|
140
|
+
|
|
141
|
+
with Producer(range(100), maxsize=10) as items:
|
|
142
|
+
for item in items:
|
|
143
|
+
if item > 5:
|
|
144
|
+
break # close() is still called by the with block
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
## Exception handling
|
|
148
|
+
|
|
149
|
+
Exceptions raised by generators are forwarded to the main thread. When a
|
|
150
|
+
generator raises, the producer forwards the exception, re-raising it from
|
|
151
|
+
`__next__()`. After an exception, the producer is terminated and raises
|
|
152
|
+
`StopIteration` on subsequent calls.
|
|
153
|
+
|
|
154
|
+
For `process.Producer`, if the child process terminates unexpectedly,
|
|
155
|
+
`__next__()` raises `RuntimeError("Child process died.")`, and becomes
|
|
156
|
+
closed.
|
|
157
|
+
|
|
158
|
+
## Limitations
|
|
159
|
+
|
|
160
|
+
> **Limitations:** `process.Producer`/`process.Consumer` with generators,
|
|
161
|
+
> coroutines and other unpicklable objects require the `fork` start
|
|
162
|
+
> method. Python 3.14 changes the default on Linux from `fork` to
|
|
163
|
+
> `forkserver`, and `spawn`/`forkserver` cannot pickle generators
|
|
164
|
+
> (`TypeError: cannot pickle 'generator' object`). In those cases
|
|
165
|
+
> `process.Producer`/`process.Consumer` raises `RuntimeError` with a
|
|
166
|
+
> clear message. Use `thread.Producer`/`thread.Consumer`, a picklable
|
|
167
|
+
> iterable (e.g. `iter([1,2,3])`), or pass
|
|
168
|
+
> `mp_context=multiprocessing.get_context('fork')` where `fork` is
|
|
169
|
+
> available (Linux).
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# concurrent-iterator
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/pypi/concurrent-iterator/)
|
|
4
|
+
[](https://pypi.org/pypi/concurrent-iterator/)
|
|
5
|
+
|
|
6
|
+
## Intro
|
|
7
|
+
|
|
8
|
+
Classes to run Producers (iterators) and Consumers (coroutines)
|
|
9
|
+
introducing concurrency.
|
|
10
|
+
|
|
11
|
+
There are many libraries to create pipelines with stages running in
|
|
12
|
+
separate processes, a nice one is
|
|
13
|
+
[parallelpipe](https://pypi.org/project/parallelpipe/), but this library
|
|
14
|
+
does something different. It runs the entire upstream iterable (the
|
|
15
|
+
Producer) in a background thread or process, while the downstream
|
|
16
|
+
consumption code stays in the foreground. It's a more coarse library but
|
|
17
|
+
easier to integrate since things keep looking like normal generators.
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
pip install concurrent-iterator
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Implementations
|
|
26
|
+
|
|
27
|
+
There are three modules, each providing a `Producer`, a `Consumer`, and
|
|
28
|
+
(for `thread`) a `MultiProducer`:
|
|
29
|
+
|
|
30
|
+
* `dummy`: non-concurrent implementation, useful for testing or as a
|
|
31
|
+
drop-in replacement when concurrency is not needed.
|
|
32
|
+
* `thread`: uses a background thread to run the iterable or coroutine.
|
|
33
|
+
Useful for IO-bound work.
|
|
34
|
+
* `process`: uses a background process to run the iterable or coroutine.
|
|
35
|
+
Useful for CPU or IO-bound work. It has the complications of dealing
|
|
36
|
+
with processes (different memory spaces, logging, etc).
|
|
37
|
+
For logging, module
|
|
38
|
+
[`multiprocessing-logging`](https://github.com/jruere/multiprocessing-logging)
|
|
39
|
+
can be used.
|
|
40
|
+
|
|
41
|
+
All `Producer`, `MultiProducer`, and `Consumer` classes implement the
|
|
42
|
+
`IProducer` and `IConsumer` interfaces defined in `concurrent_iterator`
|
|
43
|
+
(the package root). They support:
|
|
44
|
+
|
|
45
|
+
* **Iteration** (`__next__` / `__iter__`) — Producers yield values from
|
|
46
|
+
the background iterable.
|
|
47
|
+
* **Sending** (`Consumer.send`) — Consumers forward values to the
|
|
48
|
+
background coroutine.
|
|
49
|
+
* **Explicit close** (`close()`) — stops the background worker and
|
|
50
|
+
releases resources.
|
|
51
|
+
* **State inspection** (`closed` property) — checks whether the worker
|
|
52
|
+
has been closed.
|
|
53
|
+
* **Context manager** (`with` statement) — `close()` is called
|
|
54
|
+
automatically on exit.
|
|
55
|
+
|
|
56
|
+
## Usage
|
|
57
|
+
|
|
58
|
+
### Producer
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import time
|
|
62
|
+
from concurrent_iterator.thread import Producer
|
|
63
|
+
|
|
64
|
+
def slow_generator(count):
|
|
65
|
+
for i in range(count):
|
|
66
|
+
time.sleep(0.5) # Simulate a slow producer
|
|
67
|
+
yield i
|
|
68
|
+
|
|
69
|
+
# Pre-calculate up to 5 values while the main thread consumes them.
|
|
70
|
+
with Producer(slow_generator(10), maxsize=5) as items:
|
|
71
|
+
for item in items:
|
|
72
|
+
print(item) # Do some time-consuming task
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
In this example, while the main thread processes each `item`,
|
|
76
|
+
`slow_generator` continues running in a background thread and
|
|
77
|
+
pre-calculates up to 5 values.
|
|
78
|
+
The `with` statement ensures the background thread is properly cleaned
|
|
79
|
+
up.
|
|
80
|
+
|
|
81
|
+
### Consumer
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from concurrent_iterator.thread import Consumer
|
|
85
|
+
|
|
86
|
+
def worker():
|
|
87
|
+
while True:
|
|
88
|
+
value = yield # Coroutine protocol
|
|
89
|
+
# ... do work with value ...
|
|
90
|
+
|
|
91
|
+
coro = worker()
|
|
92
|
+
next(coro) # Prime the coroutine
|
|
93
|
+
|
|
94
|
+
with Consumer(coro, maxsize=5) as consumer:
|
|
95
|
+
consumer.send("a")
|
|
96
|
+
consumer.send("b")
|
|
97
|
+
# close() is called automatically on exit, which also closes the coroutine.
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
The Consumer runs the coroutine in a background thread. `send()`
|
|
101
|
+
enqueues values; if the value cannot be enqueued within `timeout`
|
|
102
|
+
(default `0`, never blocks), it raises
|
|
103
|
+
`concurrent_iterator.WillNotConsume`, signaling the caller to retry.
|
|
104
|
+
|
|
105
|
+
### Context manager
|
|
106
|
+
|
|
107
|
+
Both Producers and Consumers can be used as context managers. When the
|
|
108
|
+
`with` block exits, `close()` is called automatically, ensuring
|
|
109
|
+
background threads or processes are terminated even on exceptions.
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
from concurrent_iterator.thread import Producer
|
|
113
|
+
|
|
114
|
+
with Producer(range(100), maxsize=10) as items:
|
|
115
|
+
for item in items:
|
|
116
|
+
if item > 5:
|
|
117
|
+
break # close() is still called by the with block
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## Exception handling
|
|
121
|
+
|
|
122
|
+
Exceptions raised by generators are forwarded to the main thread. When a
|
|
123
|
+
generator raises, the producer forwards the exception, re-raising it from
|
|
124
|
+
`__next__()`. After an exception, the producer is terminated and raises
|
|
125
|
+
`StopIteration` on subsequent calls.
|
|
126
|
+
|
|
127
|
+
For `process.Producer`, if the child process terminates unexpectedly,
|
|
128
|
+
`__next__()` raises `RuntimeError("Child process died.")`, and becomes
|
|
129
|
+
closed.
|
|
130
|
+
|
|
131
|
+
## Limitations
|
|
132
|
+
|
|
133
|
+
> **Limitations:** `process.Producer`/`process.Consumer` with generators,
|
|
134
|
+
> coroutines and other unpicklable objects require the `fork` start
|
|
135
|
+
> method. Python 3.14 changes the default on Linux from `fork` to
|
|
136
|
+
> `forkserver`, and `spawn`/`forkserver` cannot pickle generators
|
|
137
|
+
> (`TypeError: cannot pickle 'generator' object`). In those cases
|
|
138
|
+
> `process.Producer`/`process.Consumer` raises `RuntimeError` with a
|
|
139
|
+
> clear message. Use `thread.Producer`/`thread.Consumer`, a picklable
|
|
140
|
+
> iterable (e.g. `iter([1,2,3])`), or pass
|
|
141
|
+
> `mp_context=multiprocessing.get_context('fork')` where `fork` is
|
|
142
|
+
> available (Linux).
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABCMeta, abstractmethod
|
|
4
|
+
from collections.abc import Iterator
|
|
5
|
+
from typing import Any, Generic, Protocol, TypeVar
|
|
6
|
+
|
|
7
|
+
T_co = TypeVar("T_co", covariant=True)
|
|
8
|
+
T_contra = TypeVar("T_contra", contravariant=True)
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"ConsumerCoroutine",
|
|
12
|
+
"IConsumer",
|
|
13
|
+
"IProducer",
|
|
14
|
+
"WillNotConsume",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class IProducer(Iterator[T_co], metaclass=ABCMeta):
|
|
19
|
+
"""Interface for Producers.
|
|
20
|
+
|
|
21
|
+
Implementations of this interface are "normal" iterators that accept an
|
|
22
|
+
iterator and return its values, optionally running the given iterator
|
|
23
|
+
in parallel and buffering a number of values.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
@abstractmethod
|
|
27
|
+
def __next__(self) -> T_co:
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
@abstractmethod
|
|
31
|
+
def close(self) -> None:
|
|
32
|
+
"""Close the producer and release resources."""
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
@abstractmethod
|
|
36
|
+
def closed(self) -> bool:
|
|
37
|
+
"""Whether the producer has been closed."""
|
|
38
|
+
|
|
39
|
+
@abstractmethod
|
|
40
|
+
def __enter__(self) -> IProducer[T_co]:
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
@abstractmethod
|
|
44
|
+
def __exit__(self, *args: object) -> bool:
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class WillNotConsume(Exception):
|
|
49
|
+
"""The consumer refuses to accept the given value."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ConsumerCoroutine(Protocol[T_contra]):
|
|
53
|
+
"""Generator-based coroutine fed by Consumers."""
|
|
54
|
+
|
|
55
|
+
def send(self, value: T_contra) -> Any: ...
|
|
56
|
+
def close(self) -> None: ...
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class IConsumer(Generic[T_contra], metaclass=ABCMeta):
|
|
60
|
+
"""Wraps coroutine like objects to execute them in parallel."""
|
|
61
|
+
|
|
62
|
+
@abstractmethod
|
|
63
|
+
def send(self, value: T_contra, timeout: float = 0) -> None:
|
|
64
|
+
"""Feeds a value to the consumer.
|
|
65
|
+
|
|
66
|
+
:param value: Value to send.
|
|
67
|
+
:param timeout: Maximum time to block waiting for queue space.
|
|
68
|
+
If the value cannot be enqueued within this time,
|
|
69
|
+
raises WillNotConsume. A value of 0 (the default)
|
|
70
|
+
never blocks and raises immediately if there is
|
|
71
|
+
no space.
|
|
72
|
+
:raises WillNotConsume: When the given value is not accepted. It may be
|
|
73
|
+
accepted on retry.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
@abstractmethod
|
|
77
|
+
def close(self) -> None:
|
|
78
|
+
"""Waits for the IConsumer to finish up and be destroyed."""
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
@abstractmethod
|
|
82
|
+
def closed(self) -> bool:
|
|
83
|
+
"""Whether the consumer has been closed."""
|
|
84
|
+
|
|
85
|
+
@abstractmethod
|
|
86
|
+
def __enter__(self) -> IConsumer[T_contra]:
|
|
87
|
+
pass
|
|
88
|
+
|
|
89
|
+
@abstractmethod
|
|
90
|
+
def __exit__(self, *args: object) -> bool:
|
|
91
|
+
pass
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
from typing import Literal, TypeVar
|
|
7
|
+
|
|
8
|
+
from concurrent_iterator import IConsumer, IProducer
|
|
9
|
+
|
|
10
|
+
T_co = TypeVar("T_co", covariant=True)
|
|
11
|
+
T_contra = TypeVar("T_contra", contravariant=True)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class BaseProducer(IProducer[T_co], ABC):
|
|
15
|
+
"""Shared lifecycle for producers; subclasses implement `__next__` and `_do_close`."""
|
|
16
|
+
|
|
17
|
+
def __init__(self) -> None:
|
|
18
|
+
self._closed = False
|
|
19
|
+
|
|
20
|
+
@property
|
|
21
|
+
def closed(self) -> bool:
|
|
22
|
+
return self._closed
|
|
23
|
+
|
|
24
|
+
def __iter__(self) -> Iterator[T_co]:
|
|
25
|
+
return self
|
|
26
|
+
|
|
27
|
+
def __enter__(self) -> BaseProducer[T_co]:
|
|
28
|
+
return self
|
|
29
|
+
|
|
30
|
+
def __exit__(self, *args: object) -> Literal[False]:
|
|
31
|
+
self.close()
|
|
32
|
+
return False
|
|
33
|
+
|
|
34
|
+
def __del__(self) -> None:
|
|
35
|
+
try:
|
|
36
|
+
if not hasattr(self, "_closed"):
|
|
37
|
+
return
|
|
38
|
+
self.close()
|
|
39
|
+
except Exception:
|
|
40
|
+
logging.getLogger(type(self).__module__ + "." + type(self).__name__).exception(
|
|
41
|
+
"Exception in __del__"
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def close(self) -> None:
|
|
45
|
+
if self._closed:
|
|
46
|
+
return
|
|
47
|
+
|
|
48
|
+
self._closed = True
|
|
49
|
+
self._do_close()
|
|
50
|
+
|
|
51
|
+
@abstractmethod
|
|
52
|
+
def _do_close(self) -> None:
|
|
53
|
+
"""Release resources; called once by `close()` after marking closed."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class BaseConsumer(IConsumer[T_contra], ABC):
|
|
57
|
+
"""Shared lifecycle for consumers; subclasses implement `send` and `_do_close`."""
|
|
58
|
+
|
|
59
|
+
def __init__(self) -> None:
|
|
60
|
+
self._closed = False
|
|
61
|
+
self._error: BaseException | None = None
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def closed(self) -> bool:
|
|
65
|
+
return self._closed
|
|
66
|
+
|
|
67
|
+
def __enter__(self) -> BaseConsumer[T_contra]:
|
|
68
|
+
return self
|
|
69
|
+
|
|
70
|
+
def __exit__(self, *args: object) -> Literal[False]:
|
|
71
|
+
self.close()
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
def __del__(self) -> None:
|
|
75
|
+
try:
|
|
76
|
+
if not hasattr(self, "_closed"):
|
|
77
|
+
return
|
|
78
|
+
self.close()
|
|
79
|
+
except Exception:
|
|
80
|
+
logging.getLogger(type(self).__module__ + "." + type(self).__name__).exception(
|
|
81
|
+
"Exception in __del__"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def close(self) -> None:
|
|
85
|
+
if self._closed:
|
|
86
|
+
return
|
|
87
|
+
|
|
88
|
+
self._closed = True
|
|
89
|
+
self._do_close()
|
|
90
|
+
if self._error is not None:
|
|
91
|
+
raise self._error
|
|
92
|
+
|
|
93
|
+
@abstractmethod
|
|
94
|
+
def _do_close(self) -> None:
|
|
95
|
+
"""Release resources; called once by `close()` after marking closed."""
|
|
@@ -2,11 +2,22 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from collections.abc import Callable
|
|
4
4
|
from functools import wraps
|
|
5
|
-
from typing import Any, TypeVar
|
|
5
|
+
from typing import Any, TypeVar
|
|
6
6
|
|
|
7
7
|
R = TypeVar("R")
|
|
8
8
|
|
|
9
9
|
|
|
10
|
+
class StopIterationSentinel:
|
|
11
|
+
"""Sentinel to signal the end of data."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ExceptionInUserIterable:
|
|
15
|
+
"""Wraps an exception raised by a user iterable."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, exception: BaseException) -> None:
|
|
18
|
+
self.exception = exception
|
|
19
|
+
|
|
20
|
+
|
|
10
21
|
def check_open(func: Callable[..., R]) -> Callable[..., R]:
|
|
11
22
|
@wraps(func)
|
|
12
23
|
def _f(self: Any, *args: Any, **kwargs: Any) -> R:
|
|
@@ -14,4 +25,4 @@ def check_open(func: Callable[..., R]) -> Callable[..., R]:
|
|
|
14
25
|
raise ValueError(f"{func.__name__} operation on closed Consumer")
|
|
15
26
|
return func(self, *args, **kwargs)
|
|
16
27
|
|
|
17
|
-
return
|
|
28
|
+
return _f
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
from typing import TypeVar
|
|
5
|
+
|
|
6
|
+
from concurrent_iterator import ConsumerCoroutine
|
|
7
|
+
from concurrent_iterator._abc import BaseConsumer, BaseProducer
|
|
8
|
+
from concurrent_iterator._internal import check_open
|
|
9
|
+
|
|
10
|
+
T_co = TypeVar("T_co", covariant=True)
|
|
11
|
+
T_contra = TypeVar("T_contra", contravariant=True)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Producer(BaseProducer[T_co]):
|
|
15
|
+
"""Dummy implementation that doesn't use concurrency."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, iterable: Iterable[T_co], maxsize: int | None = None) -> None:
|
|
18
|
+
"""In this implementation, `maxsize` is included to ease replacing
|
|
19
|
+
implementations but it's ignored.
|
|
20
|
+
"""
|
|
21
|
+
assert (
|
|
22
|
+
maxsize is None or maxsize > 0
|
|
23
|
+
), f"`maxsize` must be None or positive, but is {maxsize}."
|
|
24
|
+
|
|
25
|
+
super().__init__()
|
|
26
|
+
self._iterator = iter(iterable)
|
|
27
|
+
|
|
28
|
+
def __next__(self) -> T_co:
|
|
29
|
+
if self._closed:
|
|
30
|
+
raise StopIteration
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
return next(self._iterator)
|
|
34
|
+
except Exception:
|
|
35
|
+
self.close()
|
|
36
|
+
raise
|
|
37
|
+
|
|
38
|
+
def _do_close(self) -> None:
|
|
39
|
+
pass
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Consumer(BaseConsumer[T_contra]):
|
|
43
|
+
"""Dummy implementation that doesn't use concurrency."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, coroutine: ConsumerCoroutine[T_contra]) -> None:
|
|
46
|
+
super().__init__()
|
|
47
|
+
self._coroutine = coroutine
|
|
48
|
+
|
|
49
|
+
@check_open
|
|
50
|
+
def send(self, value: T_contra, timeout: float = 0) -> None:
|
|
51
|
+
"""`timeout` is not supported and fails if greater than 0."""
|
|
52
|
+
assert timeout in (0, 0.0), f"`timeout` is not supported in this implementation: {timeout}."
|
|
53
|
+
|
|
54
|
+
self._coroutine.send(value)
|
|
55
|
+
|
|
56
|
+
def _do_close(self) -> None:
|
|
57
|
+
self._coroutine.close()
|