oj-io-chains 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- io_chains/__init__.py +0 -0
- io_chains/links/__init__.py +0 -0
- io_chains/links/chain.py +90 -0
- io_chains/links/link.py +90 -0
- io_chains/links/linkable.py +19 -0
- io_chains/pubsub/__init__.py +0 -0
- io_chains/pubsub/callback_subscriber.py +20 -0
- io_chains/pubsub/collector.py +43 -0
- io_chains/pubsub/publisher.py +41 -0
- io_chains/pubsub/sentinel.py +14 -0
- io_chains/pubsub/subscriber.py +8 -0
- io_chains/utils/__init__.py +0 -0
- io_chains/utils/converters.py +32 -0
- oj_io_chains-0.0.2.dist-info/METADATA +316 -0
- oj_io_chains-0.0.2.dist-info/RECORD +18 -0
- oj_io_chains-0.0.2.dist-info/WHEEL +5 -0
- oj_io_chains-0.0.2.dist-info/licenses/LICENSE +21 -0
- oj_io_chains-0.0.2.dist-info/top_level.txt +1 -0
io_chains/__init__.py
ADDED
|
File without changes
|
|
File without changes
|
io_chains/links/chain.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
from asyncio import create_task, gather
|
|
2
|
+
from typing import Any, AsyncGenerator, Callable, Iterable, Optional, Union
|
|
3
|
+
|
|
4
|
+
from io_chains.links.linkable import Linkable
|
|
5
|
+
from io_chains.pubsub.sentinel import EndOfStream
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _output_of(linkable: Linkable) -> Linkable:
|
|
9
|
+
"""Return the linkable that actually publishes output for this linkable.
|
|
10
|
+
For a Chain, that's its last internal link. For a Link, it's itself."""
|
|
11
|
+
if isinstance(linkable, Chain):
|
|
12
|
+
return _output_of(linkable._links[-1])
|
|
13
|
+
return linkable
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _input_of(linkable: Linkable) -> Linkable:
|
|
17
|
+
"""Return the linkable that receives pushed input for this linkable.
|
|
18
|
+
For a Chain, that's its first internal link. For a Link, it's itself."""
|
|
19
|
+
if isinstance(linkable, Chain):
|
|
20
|
+
return _input_of(linkable._links[0])
|
|
21
|
+
return linkable
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Chain(Linkable):
|
|
25
|
+
"""
|
|
26
|
+
An ordered sequence of Linkables that runs as a single unit.
|
|
27
|
+
|
|
28
|
+
Chain is the orchestrator: it wires its internal links together,
|
|
29
|
+
manages their concurrent execution, and presents itself as a single
|
|
30
|
+
Linkable to the outside world.
|
|
31
|
+
|
|
32
|
+
From the outside, a Chain looks like any other Linkable:
|
|
33
|
+
- It accepts input via in_iter or push() (routed to the first link)
|
|
34
|
+
- It publishes output through its last link's subscribers
|
|
35
|
+
- await chain() runs the whole pipeline; no external gather() needed
|
|
36
|
+
|
|
37
|
+
A Chain's elements can be Links or other Chains.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
*args,
|
|
43
|
+
links: list[Linkable],
|
|
44
|
+
source: Union[Callable, Iterable, None] = None,
|
|
45
|
+
subscribers: Union[Iterable[Callable], Callable, None] = None,
|
|
46
|
+
**kwargs,
|
|
47
|
+
) -> None:
|
|
48
|
+
super().__init__(*args, **kwargs)
|
|
49
|
+
self._links: list[Linkable] = links
|
|
50
|
+
|
|
51
|
+
# Wire each linkable's output to the next linkable's input
|
|
52
|
+
for i in range(len(links) - 1):
|
|
53
|
+
_output_of(links[i]).subscribers = [links[i + 1]]
|
|
54
|
+
|
|
55
|
+
# Attach external input to the first link
|
|
56
|
+
if source is not None:
|
|
57
|
+
_input_of(links[0]).input = source
|
|
58
|
+
|
|
59
|
+
# Attach external subscribers to the last link's output
|
|
60
|
+
if subscribers is not None:
|
|
61
|
+
_output_of(links[-1]).subscribers = subscribers
|
|
62
|
+
|
|
63
|
+
# --- Linkable interface ---
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
async def input(self) -> AsyncGenerator:
|
|
67
|
+
"""Delegates to the first link's input."""
|
|
68
|
+
async for datum in _input_of(self._links[0]).input:
|
|
69
|
+
yield datum
|
|
70
|
+
|
|
71
|
+
@input.setter
|
|
72
|
+
def input(self, in_iter) -> None:
|
|
73
|
+
"""Route input assignment to the first link."""
|
|
74
|
+
_input_of(self._links[0]).input = in_iter
|
|
75
|
+
|
|
76
|
+
def push(self, datum: Any) -> None:
|
|
77
|
+
"""Route pushed data to the first link."""
|
|
78
|
+
_input_of(self._links[0]).push(datum)
|
|
79
|
+
|
|
80
|
+
async def _fill_queue_from_input(self) -> None:
|
|
81
|
+
"""Chain delegates input filling to its first link."""
|
|
82
|
+
pass
|
|
83
|
+
|
|
84
|
+
async def run(self) -> None:
|
|
85
|
+
"""Gather all internal links concurrently. This is the key Chain promise:
|
|
86
|
+
callers just await chain() — no external gather() or create_task() needed."""
|
|
87
|
+
await gather(*[create_task(link()) for link in self._links])
|
|
88
|
+
|
|
89
|
+
async def __call__(self) -> None:
|
|
90
|
+
return await self.run()
|
io_chains/links/link.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
from asyncio import Queue, Task, create_task, gather
|
|
2
|
+
from inspect import isfunction, isawaitable
|
|
3
|
+
from logging import getLogger
|
|
4
|
+
from typing import Any, AsyncGenerator, AsyncIterable, Callable, Iterable, Optional, Union
|
|
5
|
+
|
|
6
|
+
from io_chains.links.linkable import Linkable
|
|
7
|
+
from io_chains.pubsub.sentinel import END_OF_STREAM, EndOfStream
|
|
8
|
+
|
|
9
|
+
logger = getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Link(Linkable):
|
|
13
|
+
def __init__(
|
|
14
|
+
self,
|
|
15
|
+
*args,
|
|
16
|
+
source: Union[Callable, Iterable, None] = None,
|
|
17
|
+
transformer: Optional[Callable] = None,
|
|
18
|
+
queue: Optional[Queue] = None,
|
|
19
|
+
**kwargs
|
|
20
|
+
) -> None:
|
|
21
|
+
super().__init__(*args, **kwargs)
|
|
22
|
+
self._input: Union[AsyncIterable, Callable, Iterable, None] = None
|
|
23
|
+
self.input = source
|
|
24
|
+
|
|
25
|
+
self._transformer: Optional[Callable] = None
|
|
26
|
+
self.transformer = transformer
|
|
27
|
+
self._transforming: bool = True
|
|
28
|
+
|
|
29
|
+
self._queue: Queue = queue or Queue()
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
async def input(self) -> AsyncGenerator:
|
|
33
|
+
if self._input:
|
|
34
|
+
if hasattr(self._input, '__aiter__'):
|
|
35
|
+
async for each in self._input:
|
|
36
|
+
yield each
|
|
37
|
+
elif isfunction(self._input):
|
|
38
|
+
async for each in self._input():
|
|
39
|
+
yield each
|
|
40
|
+
else:
|
|
41
|
+
for each in self._input:
|
|
42
|
+
yield each
|
|
43
|
+
|
|
44
|
+
@input.setter
|
|
45
|
+
def input(self, in_obj: Union[AsyncIterable, Callable, Iterable, None] = None) -> None:
|
|
46
|
+
if in_obj:
|
|
47
|
+
if not isinstance(in_obj, (AsyncIterable, Callable, Iterable)):
|
|
48
|
+
raise TypeError(f'source must be Callable or Iterable, got {type(in_obj)}')
|
|
49
|
+
self._input = in_obj
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def transformer(self) -> Any:
|
|
53
|
+
return self._transformer
|
|
54
|
+
|
|
55
|
+
@transformer.setter
|
|
56
|
+
def transformer(self, transformer: Optional[Callable] = None) -> None:
|
|
57
|
+
self._transformer = transformer
|
|
58
|
+
|
|
59
|
+
async def _fill_queue_from_input(self) -> None:
|
|
60
|
+
if self._input:
|
|
61
|
+
try:
|
|
62
|
+
async for datum in self.input:
|
|
63
|
+
self.push(datum)
|
|
64
|
+
self.push(END_OF_STREAM)
|
|
65
|
+
except (StopIteration, StopAsyncIteration) as exc_stop:
|
|
66
|
+
logger.warning(f'STOPPING FILL FROM INPUT: {exc_stop}')
|
|
67
|
+
raise
|
|
68
|
+
|
|
69
|
+
async def _update_subscribers(self) -> None:
|
|
70
|
+
should_continue: bool = True
|
|
71
|
+
while should_continue:
|
|
72
|
+
datum: Any = await self._queue.get()
|
|
73
|
+
if not isinstance(datum, EndOfStream) and self._transformer:
|
|
74
|
+
result = self._transformer(datum)
|
|
75
|
+
datum = await result if isawaitable(result) else result
|
|
76
|
+
await self.publish(datum)
|
|
77
|
+
should_continue = not isinstance(datum, EndOfStream)
|
|
78
|
+
|
|
79
|
+
def push(self, datum: Any) -> None:
|
|
80
|
+
self._queue.put_nowait(datum)
|
|
81
|
+
|
|
82
|
+
async def run(self) -> None:
|
|
83
|
+
tasks: list[Task] = [
|
|
84
|
+
create_task(self._fill_queue_from_input()),
|
|
85
|
+
create_task(self._update_subscribers()),
|
|
86
|
+
]
|
|
87
|
+
await gather(*tasks)
|
|
88
|
+
|
|
89
|
+
async def __call__(self) -> Optional[tuple]:
|
|
90
|
+
return await self.run()
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from abc import abstractmethod
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from io_chains.pubsub.publisher import Publisher
|
|
5
|
+
from io_chains.pubsub.subscriber import Subscriber
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Linkable(Publisher, Subscriber):
|
|
9
|
+
@abstractmethod
|
|
10
|
+
async def input(self, *args, **kwargs) -> Any:
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
@abstractmethod
|
|
14
|
+
async def _fill_queue_from_input(self, *args, **kwargs) -> None:
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
@abstractmethod
|
|
18
|
+
async def __call__(self, *args, **kwargs) -> None:
|
|
19
|
+
pass
|
|
File without changes
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from logging import getLogger
|
|
2
|
+
from typing import Any, Callable
|
|
3
|
+
|
|
4
|
+
from io_chains.pubsub.sentinel import EndOfStream
|
|
5
|
+
from io_chains.pubsub.subscriber import Subscriber
|
|
6
|
+
|
|
7
|
+
logger = getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CallbackSubscriber(Subscriber):
|
|
11
|
+
def __init__(self, callback: Callable):
|
|
12
|
+
self._callback = callback
|
|
13
|
+
|
|
14
|
+
def push(self, datum: Any) -> Any:
|
|
15
|
+
if isinstance(datum, EndOfStream):
|
|
16
|
+
return
|
|
17
|
+
try:
|
|
18
|
+
return self._callback(datum)
|
|
19
|
+
except Exception as e:
|
|
20
|
+
logger.exception(f"CallbackSubscriber.push: {self._callback}: {e}")
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from asyncio import Queue
|
|
2
|
+
from logging import getLogger
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from io_chains.pubsub.sentinel import EndOfStream
|
|
6
|
+
from io_chains.pubsub.subscriber import Subscriber
|
|
7
|
+
from io_chains.utils.converters import iter_over_async
|
|
8
|
+
|
|
9
|
+
logger = getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Collector(Subscriber):
|
|
13
|
+
"""
|
|
14
|
+
Subscriber that buffers items in a queue for iteration.
|
|
15
|
+
|
|
16
|
+
Supports both async and sync iteration protocols directly:
|
|
17
|
+
async for item in collector: ...
|
|
18
|
+
for item in collector: ...
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self):
|
|
22
|
+
self._queue: Queue = Queue()
|
|
23
|
+
|
|
24
|
+
def push(self, datum: Any) -> None:
|
|
25
|
+
try:
|
|
26
|
+
self._queue.put_nowait(datum)
|
|
27
|
+
except Exception as e:
|
|
28
|
+
logger.exception(f"Collector.push: {e}")
|
|
29
|
+
|
|
30
|
+
def __aiter__(self):
|
|
31
|
+
return self
|
|
32
|
+
|
|
33
|
+
async def __anext__(self) -> Any:
|
|
34
|
+
next_item = await self._queue.get()
|
|
35
|
+
if isinstance(next_item, EndOfStream):
|
|
36
|
+
raise StopAsyncIteration
|
|
37
|
+
return next_item
|
|
38
|
+
|
|
39
|
+
def __iter__(self):
|
|
40
|
+
async def _async_gen():
|
|
41
|
+
async for item in self:
|
|
42
|
+
yield item
|
|
43
|
+
yield from iter_over_async(_async_gen())
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
from abc import ABC
|
|
2
|
+
from typing import Callable, Iterable, Union, Any
|
|
3
|
+
|
|
4
|
+
from io_chains.pubsub.subscriber import Subscriber
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Publisher(ABC):
|
|
8
|
+
def __init__(
|
|
9
|
+
self,
|
|
10
|
+
*args,
|
|
11
|
+
subscribers: Union[Iterable[Callable], None, Callable] = None,
|
|
12
|
+
**kwargs
|
|
13
|
+
) -> None:
|
|
14
|
+
self._subscribers: list = []
|
|
15
|
+
self.subscribers = subscribers
|
|
16
|
+
|
|
17
|
+
@property
|
|
18
|
+
def subscribers(self) -> list[Callable | Subscriber]:
|
|
19
|
+
return self._subscribers
|
|
20
|
+
|
|
21
|
+
@subscribers.setter
|
|
22
|
+
def subscribers(self, subscribers: Union[Iterable[Callable], None, Callable]) -> None:
|
|
23
|
+
if not subscribers:
|
|
24
|
+
return
|
|
25
|
+
if isinstance(subscribers, (Callable, Subscriber)):
|
|
26
|
+
self._subscribers.append(subscribers)
|
|
27
|
+
elif isinstance(subscribers, Iterable):
|
|
28
|
+
for subscriber in subscribers:
|
|
29
|
+
if isinstance(subscriber, (Callable, Subscriber)):
|
|
30
|
+
self._subscribers.append(subscriber)
|
|
31
|
+
else:
|
|
32
|
+
raise TypeError('subscribers must be a Subscriber or Iterable[Subscriber]')
|
|
33
|
+
|
|
34
|
+
async def publish(self, datum) -> None:
|
|
35
|
+
for subscriber in self.subscribers:
|
|
36
|
+
if isinstance(subscriber, Subscriber):
|
|
37
|
+
subscriber.push(datum)
|
|
38
|
+
elif isinstance(subscriber, Callable):
|
|
39
|
+
subscriber(datum)
|
|
40
|
+
else:
|
|
41
|
+
raise TypeError('subscriber must be directly Callable or Subscriber')
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
class EndOfStream:
|
|
2
|
+
"""Sentinel signaling end of data in a stream. Implemented as a singleton."""
|
|
3
|
+
_instance = None
|
|
4
|
+
|
|
5
|
+
def __new__(cls):
|
|
6
|
+
if cls._instance is None:
|
|
7
|
+
cls._instance = super().__new__(cls)
|
|
8
|
+
return cls._instance
|
|
9
|
+
|
|
10
|
+
def __repr__(self):
|
|
11
|
+
return 'END_OF_STREAM'
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
END_OF_STREAM = EndOfStream()
|
|
File without changes
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from asyncio import get_running_loop, run
|
|
2
|
+
from logging import getLogger
|
|
3
|
+
from typing import Any, AsyncIterable, Generator, Callable
|
|
4
|
+
|
|
5
|
+
import nest_asyncio
|
|
6
|
+
|
|
7
|
+
logger = getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def iter_over_async(a_iter: AsyncIterable) -> Generator[Any, Any, Any]:
|
|
11
|
+
ait = a_iter.__aiter__()
|
|
12
|
+
|
|
13
|
+
async def get_next():
|
|
14
|
+
try:
|
|
15
|
+
obj = await ait.__anext__()
|
|
16
|
+
return False, obj
|
|
17
|
+
except StopAsyncIteration:
|
|
18
|
+
return True, None
|
|
19
|
+
|
|
20
|
+
runner: Callable = run
|
|
21
|
+
try:
|
|
22
|
+
loop = get_running_loop()
|
|
23
|
+
nest_asyncio.apply(loop=loop)
|
|
24
|
+
runner = loop.run_until_complete
|
|
25
|
+
except Exception as e:
|
|
26
|
+
logger.warning(f'\n\n{__name__=}: {e}\n\n')
|
|
27
|
+
|
|
28
|
+
is_done: bool = False
|
|
29
|
+
while not is_done:
|
|
30
|
+
is_done, next_obj = runner(get_next())
|
|
31
|
+
if not is_done:
|
|
32
|
+
yield next_obj
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: oj-io-chains
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Lightweight async data pipelines using a publisher/subscriber pattern
|
|
5
|
+
Author-email: Speedy <speedy@ownjoo.org>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/ownjoo-org/io-chains
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: httpx
|
|
12
|
+
Requires-Dist: nest-asyncio
|
|
13
|
+
Dynamic: license-file
|
|
14
|
+
|
|
15
|
+
# io-chains
|
|
16
|
+
|
|
17
|
+
A lightweight Python library for building async data pipelines using a publisher/subscriber pattern. Chain together data sources, transformations, and consumers with minimal boilerplate.
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install oj-io-chains
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Core Concepts
|
|
26
|
+
|
|
27
|
+
### The Pipeline Model
|
|
28
|
+
|
|
29
|
+
A pipeline is built from **Links** coordinated by a **Chain**:
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
[source] → Link → [transformer] → Subscriber(s)
|
|
33
|
+
├── CallbackSubscriber (side effects)
|
|
34
|
+
├── Collector (collect results)
|
|
35
|
+
└── Link (next stage)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Each `Link`:
|
|
39
|
+
- Pulls items from a source (list, generator, async generator, or callable)
|
|
40
|
+
- Optionally transforms each item (sync or async)
|
|
41
|
+
- Publishes each item to one or more subscribers
|
|
42
|
+
|
|
43
|
+
A `Chain` wires multiple Links together and runs them concurrently — the caller just `await chain()`.
|
|
44
|
+
|
|
45
|
+
### End-of-stream
|
|
46
|
+
|
|
47
|
+
`EndOfStream` is the sentinel that signals end-of-stream through the pipeline. `None` is valid data and flows through the pipeline normally.
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## Classes
|
|
52
|
+
|
|
53
|
+
### `Link`
|
|
54
|
+
|
|
55
|
+
A single processing unit: source → transformer → subscribers.
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from io_chains.links.link import Link
|
|
59
|
+
|
|
60
|
+
link = Link(
|
|
61
|
+
source=..., # input source (optional)
|
|
62
|
+
transformer=..., # transform function (optional)
|
|
63
|
+
subscribers=[...], # subscribers (optional)
|
|
64
|
+
)
|
|
65
|
+
await link()
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
**Parameters**
|
|
69
|
+
|
|
70
|
+
| Parameter | Type | Description |
|
|
71
|
+
|---|---|---|
|
|
72
|
+
| `source` | `Iterable`, callable, or `AsyncIterable` | Input data source. If a callable, it is called and the result iterated. If omitted, the Link acts as a subscriber-only stage that receives data via `push()`. |
|
|
73
|
+
| `transformer` | callable | Applied to each item before publishing. Can be sync or async. |
|
|
74
|
+
| `subscribers` | `Subscriber`, callable, or list of either | Where output is sent. |
|
|
75
|
+
|
|
76
|
+
**Source types**
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
# List or any iterable
|
|
80
|
+
Link(source=[1, 2, 3])
|
|
81
|
+
|
|
82
|
+
# Generator expression
|
|
83
|
+
Link(source=(x * 2 for x in range(10)))
|
|
84
|
+
|
|
85
|
+
# Async generator function (called automatically)
|
|
86
|
+
async def my_source():
|
|
87
|
+
yield 1
|
|
88
|
+
yield 2
|
|
89
|
+
|
|
90
|
+
Link(source=my_source)
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
### `Chain`
|
|
96
|
+
|
|
97
|
+
Orchestrates multiple Links as a single pipeline. Wires them together and manages concurrent execution internally.
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from io_chains.links.chain import Chain
|
|
101
|
+
|
|
102
|
+
pipeline = Chain(
|
|
103
|
+
source=..., # attached to the first link (optional)
|
|
104
|
+
links=[...], # ordered list of Links or Chains
|
|
105
|
+
subscribers=[...], # attached to the last link's output (optional)
|
|
106
|
+
)
|
|
107
|
+
await pipeline() # no gather() needed
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
A Chain is itself a Linkable — it can be nested inside another Chain or used as a subscriber of an external Link.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
### `Collector`
|
|
115
|
+
|
|
116
|
+
Buffers items for iteration after the pipeline completes. Supports both async and sync iteration directly.
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from io_chains.pubsub.collector import Collector
|
|
120
|
+
|
|
121
|
+
results = Collector()
|
|
122
|
+
|
|
123
|
+
# async iteration (preferred)
|
|
124
|
+
async for item in results:
|
|
125
|
+
print(item)
|
|
126
|
+
|
|
127
|
+
# sync iteration
|
|
128
|
+
for item in results:
|
|
129
|
+
print(item)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
### `CallbackSubscriber`
|
|
135
|
+
|
|
136
|
+
Calls a function for each item. Good for side effects (logging, writing, printing).
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
from io_chains.pubsub.callback_subscriber import CallbackSubscriber
|
|
140
|
+
|
|
141
|
+
sub = CallbackSubscriber(callback=lambda x: print(x))
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
## Usage Examples
|
|
147
|
+
|
|
148
|
+
### Simple transformation
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
import asyncio
|
|
152
|
+
from io_chains.links.link import Link
|
|
153
|
+
from io_chains.pubsub.collector import Collector
|
|
154
|
+
|
|
155
|
+
async def main():
|
|
156
|
+
results = Collector()
|
|
157
|
+
link = Link(
|
|
158
|
+
source=[1, 2, 3],
|
|
159
|
+
transformer=lambda x: x * 2,
|
|
160
|
+
subscribers=[results],
|
|
161
|
+
)
|
|
162
|
+
await link()
|
|
163
|
+
|
|
164
|
+
async for item in results:
|
|
165
|
+
print(item) # 2, 4, 6
|
|
166
|
+
|
|
167
|
+
asyncio.run(main())
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Multi-stage pipeline with Chain
|
|
171
|
+
|
|
172
|
+
`Chain` manages concurrent execution — no `gather()` or `create_task()` needed.
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
import asyncio
|
|
176
|
+
from io_chains.links.chain import Chain
|
|
177
|
+
from io_chains.links.link import Link
|
|
178
|
+
from io_chains.pubsub.collector import Collector
|
|
179
|
+
|
|
180
|
+
async def main():
|
|
181
|
+
results = Collector()
|
|
182
|
+
|
|
183
|
+
pipeline = Chain(
|
|
184
|
+
source=['a', 'b', 'c'],
|
|
185
|
+
links=[
|
|
186
|
+
Link(transformer=str.upper),
|
|
187
|
+
Link(transformer=lambda x: f'item: {x}'),
|
|
188
|
+
],
|
|
189
|
+
subscribers=[results],
|
|
190
|
+
)
|
|
191
|
+
await pipeline()
|
|
192
|
+
|
|
193
|
+
async for item in results:
|
|
194
|
+
print(item)
|
|
195
|
+
# item: A
|
|
196
|
+
# item: B
|
|
197
|
+
# item: C
|
|
198
|
+
|
|
199
|
+
asyncio.run(main())
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Async generator source with enrichment
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
from httpx import AsyncClient
|
|
206
|
+
from io_chains.links.chain import Chain
|
|
207
|
+
from io_chains.links.link import Link
|
|
208
|
+
from io_chains.pubsub.collector import Collector
|
|
209
|
+
|
|
210
|
+
async def fetch_records():
|
|
211
|
+
async with AsyncClient() as client:
|
|
212
|
+
response = await client.get('https://api.example.com/records')
|
|
213
|
+
for record in response.json():
|
|
214
|
+
yield record
|
|
215
|
+
|
|
216
|
+
async def main():
|
|
217
|
+
results = Collector()
|
|
218
|
+
|
|
219
|
+
pipeline = Chain(
|
|
220
|
+
source=fetch_records,
|
|
221
|
+
links=[
|
|
222
|
+
Link(transformer=lambda r: {**r, 'name': r['name'].upper()}),
|
|
223
|
+
],
|
|
224
|
+
subscribers=[results],
|
|
225
|
+
)
|
|
226
|
+
await pipeline()
|
|
227
|
+
|
|
228
|
+
async for record in results:
|
|
229
|
+
print(record)
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
### Multiple subscribers (fan-out)
|
|
233
|
+
|
|
234
|
+
Every subscriber receives every item independently.
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
from io_chains.pubsub.callback_subscriber import CallbackSubscriber
|
|
238
|
+
|
|
239
|
+
audit = []
|
|
240
|
+
results = Collector()
|
|
241
|
+
|
|
242
|
+
pipeline = Chain(
|
|
243
|
+
source=[1, 2, 3],
|
|
244
|
+
links=[Link(transformer=lambda x: x * 2)],
|
|
245
|
+
subscribers=[
|
|
246
|
+
results,
|
|
247
|
+
CallbackSubscriber(callback=lambda x: audit.append(x)),
|
|
248
|
+
],
|
|
249
|
+
)
|
|
250
|
+
await pipeline()
|
|
251
|
+
# results contains [2, 4, 6], audit == [2, 4, 6]
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
### Nested Chains
|
|
255
|
+
|
|
256
|
+
Chains can be nested — each is a black-box Linkable from the outside.
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
normalise = Chain(links=[
|
|
260
|
+
Link(transformer=lambda x: abs(x)),
|
|
261
|
+
Link(transformer=lambda x: round(x, 2)),
|
|
262
|
+
])
|
|
263
|
+
|
|
264
|
+
stringify = Chain(links=[
|
|
265
|
+
Link(transformer=lambda x: x * 100),
|
|
266
|
+
Link(transformer=lambda x: f'{x:.0f}%'),
|
|
267
|
+
])
|
|
268
|
+
|
|
269
|
+
pipeline = Chain(
|
|
270
|
+
source=[-0.156, 0.999, -0.301],
|
|
271
|
+
links=[normalise, stringify],
|
|
272
|
+
subscribers=[results],
|
|
273
|
+
)
|
|
274
|
+
await pipeline()
|
|
275
|
+
# results contains ['16%', '100%', '30%']
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
---
|
|
279
|
+
|
|
280
|
+
## Architecture
|
|
281
|
+
|
|
282
|
+
```
|
|
283
|
+
Subscriber (ABC)
|
|
284
|
+
├── CallbackSubscriber
|
|
285
|
+
└── Collector
|
|
286
|
+
|
|
287
|
+
Publisher (ABC)
|
|
288
|
+
└── Linkable(Publisher, Subscriber) (ABC)
|
|
289
|
+
├── Link
|
|
290
|
+
└── Chain
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
**`Publisher`** manages a list of subscribers and distributes data to all of them via `publish(datum)`.
|
|
294
|
+
|
|
295
|
+
**`Subscriber`** defines the `push(datum)` interface for receiving data.
|
|
296
|
+
|
|
297
|
+
**`Linkable`** combines both — it can receive data (as a subscriber in one pipeline) and emit data (as a publisher to its own subscribers).
|
|
298
|
+
|
|
299
|
+
**`Link`** implements `Linkable`. Internally it uses an `asyncio.Queue` to decouple the input reader from the subscriber publisher, allowing both to run concurrently.
|
|
300
|
+
|
|
301
|
+
**`Chain`** implements `Linkable`. It wires its internal Links/Chains together, runs them all concurrently via `gather`, and presents a single `await chain()` interface to callers.
|
|
302
|
+
|
|
303
|
+
---
|
|
304
|
+
|
|
305
|
+
## Development
|
|
306
|
+
|
|
307
|
+
```bash
|
|
308
|
+
# Install in editable mode
|
|
309
|
+
pip install -e .
|
|
310
|
+
|
|
311
|
+
# Run unit tests
|
|
312
|
+
python -m pytest test/unit -v
|
|
313
|
+
|
|
314
|
+
# Run user acceptance tests (requires network)
|
|
315
|
+
python -m pytest test/ua -v
|
|
316
|
+
```
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
io_chains/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
io_chains/links/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
io_chains/links/chain.py,sha256=6YUJaV3j8rF-KWLqCRSYHTAxSg1-PIl89S9Gmh2DeZg,3213
|
|
4
|
+
io_chains/links/link.py,sha256=hr2e7uZM27zoZ5sUtnspWj-hRqRvOA-_Kru4-vAw2uc,3123
|
|
5
|
+
io_chains/links/linkable.py,sha256=T3dcqWiWPW-poJ8bZWppJOIlDo7_uurp7BJOuUbTptI,472
|
|
6
|
+
io_chains/pubsub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
io_chains/pubsub/callback_subscriber.py,sha256=u6X5Ke1pIabuhlDvPaqwjgOt2RZz0Fx62SFJ8zmx3As,580
|
|
8
|
+
io_chains/pubsub/collector.py,sha256=rSsb40tag1_C2_1sPYn05R_QgtA4hderURY67EcHURA,1162
|
|
9
|
+
io_chains/pubsub/publisher.py,sha256=imsKvbWuLAYo7wgkiihkk-SRRJ5BUUVGcMCXBIo92jE,1436
|
|
10
|
+
io_chains/pubsub/sentinel.py,sha256=TxeKHhaTepmEMqpM3Ad85Ju11M7M2EHdP7hqhuwTOmY,345
|
|
11
|
+
io_chains/pubsub/subscriber.py,sha256=UYQqrF73xlJdciDOkjnpWwDYTLPhbsmUc97W-IPY0UU,156
|
|
12
|
+
io_chains/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
io_chains/utils/converters.py,sha256=5qWZMzsCKNd42VIz4mEBUJAXGmUf4h6ZsHOFWcV8yFQ,837
|
|
14
|
+
oj_io_chains-0.0.2.dist-info/licenses/LICENSE,sha256=UP1iQrwdoUrddtmnfBS3KnwDAxXSc-BDFzaol_U7jnc,1067
|
|
15
|
+
oj_io_chains-0.0.2.dist-info/METADATA,sha256=wDpLa2pf7lyHPnBTmr-E1qhLE43mEBoVL2lsy7MxadQ,7657
|
|
16
|
+
oj_io_chains-0.0.2.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
17
|
+
oj_io_chains-0.0.2.dist-info/top_level.txt,sha256=FJ2Hjlj4utzqxEG1_7tKsfv6OVX59QUrurUyzqm4d8k,10
|
|
18
|
+
oj_io_chains-0.0.2.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 ownjoo.org
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
io_chains
|