oj-io-chains 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oj_io_chains-0.0.2/LICENSE +21 -0
- oj_io_chains-0.0.2/PKG-INFO +316 -0
- oj_io_chains-0.0.2/README.md +302 -0
- oj_io_chains-0.0.2/io_chains/__init__.py +0 -0
- oj_io_chains-0.0.2/io_chains/links/__init__.py +0 -0
- oj_io_chains-0.0.2/io_chains/links/chain.py +90 -0
- oj_io_chains-0.0.2/io_chains/links/link.py +90 -0
- oj_io_chains-0.0.2/io_chains/links/linkable.py +19 -0
- oj_io_chains-0.0.2/io_chains/pubsub/__init__.py +0 -0
- oj_io_chains-0.0.2/io_chains/pubsub/callback_subscriber.py +20 -0
- oj_io_chains-0.0.2/io_chains/pubsub/collector.py +43 -0
- oj_io_chains-0.0.2/io_chains/pubsub/publisher.py +41 -0
- oj_io_chains-0.0.2/io_chains/pubsub/sentinel.py +14 -0
- oj_io_chains-0.0.2/io_chains/pubsub/subscriber.py +8 -0
- oj_io_chains-0.0.2/io_chains/utils/__init__.py +0 -0
- oj_io_chains-0.0.2/io_chains/utils/converters.py +32 -0
- oj_io_chains-0.0.2/oj_io_chains.egg-info/PKG-INFO +316 -0
- oj_io_chains-0.0.2/oj_io_chains.egg-info/SOURCES.txt +21 -0
- oj_io_chains-0.0.2/oj_io_chains.egg-info/dependency_links.txt +1 -0
- oj_io_chains-0.0.2/oj_io_chains.egg-info/requires.txt +2 -0
- oj_io_chains-0.0.2/oj_io_chains.egg-info/top_level.txt +1 -0
- oj_io_chains-0.0.2/pyproject.toml +23 -0
- oj_io_chains-0.0.2/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 ownjoo.org
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: oj-io-chains
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Lightweight async data pipelines using a publisher/subscriber pattern
|
|
5
|
+
Author-email: Speedy <speedy@ownjoo.org>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/ownjoo-org/io-chains
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: httpx
|
|
12
|
+
Requires-Dist: nest-asyncio
|
|
13
|
+
Dynamic: license-file
|
|
14
|
+
|
|
15
|
+
# io-chains
|
|
16
|
+
|
|
17
|
+
A lightweight Python library for building async data pipelines using a publisher/subscriber pattern. Chain together data sources, transformations, and consumers with minimal boilerplate.
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install oj-io-chains
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Core Concepts
|
|
26
|
+
|
|
27
|
+
### The Pipeline Model
|
|
28
|
+
|
|
29
|
+
A pipeline is built from **Links** coordinated by a **Chain**:
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
[source] → Link → [transformer] → Subscriber(s)
|
|
33
|
+
├── CallbackSubscriber (side effects)
|
|
34
|
+
├── Collector (collect results)
|
|
35
|
+
└── Link (next stage)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Each `Link`:
|
|
39
|
+
- Pulls items from a source (list, generator, async generator, or callable)
|
|
40
|
+
- Optionally transforms each item (sync or async)
|
|
41
|
+
- Publishes each item to one or more subscribers
|
|
42
|
+
|
|
43
|
+
A `Chain` wires multiple Links together and runs them concurrently — the caller just `await chain()`.
|
|
44
|
+
|
|
45
|
+
### End-of-stream
|
|
46
|
+
|
|
47
|
+
`EndOfStream` is the sentinel that signals end-of-stream through the pipeline. `None` is valid data and flows through the pipeline normally.
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## Classes
|
|
52
|
+
|
|
53
|
+
### `Link`
|
|
54
|
+
|
|
55
|
+
A single processing unit: source → transformer → subscribers.
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from io_chains.links.link import Link
|
|
59
|
+
|
|
60
|
+
link = Link(
|
|
61
|
+
source=..., # input source (optional)
|
|
62
|
+
transformer=..., # transform function (optional)
|
|
63
|
+
subscribers=[...], # subscribers (optional)
|
|
64
|
+
)
|
|
65
|
+
await link()
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
**Parameters**
|
|
69
|
+
|
|
70
|
+
| Parameter | Type | Description |
|
|
71
|
+
|---|---|---|
|
|
72
|
+
| `source` | `Iterable`, callable, or `AsyncIterable` | Input data source. If a callable, it is called and the result iterated. If omitted, the Link acts as a subscriber-only stage that receives data via `push()`. |
|
|
73
|
+
| `transformer` | callable | Applied to each item before publishing. Can be sync or async. |
|
|
74
|
+
| `subscribers` | `Subscriber`, callable, or list of either | Where output is sent. |
|
|
75
|
+
|
|
76
|
+
**Source types**
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
# List or any iterable
|
|
80
|
+
Link(source=[1, 2, 3])
|
|
81
|
+
|
|
82
|
+
# Generator expression
|
|
83
|
+
Link(source=(x * 2 for x in range(10)))
|
|
84
|
+
|
|
85
|
+
# Async generator function (called automatically)
|
|
86
|
+
async def my_source():
|
|
87
|
+
yield 1
|
|
88
|
+
yield 2
|
|
89
|
+
|
|
90
|
+
Link(source=my_source)
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
### `Chain`
|
|
96
|
+
|
|
97
|
+
Orchestrates multiple Links as a single pipeline. Wires them together and manages concurrent execution internally.
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from io_chains.links.chain import Chain
|
|
101
|
+
|
|
102
|
+
pipeline = Chain(
|
|
103
|
+
source=..., # attached to the first link (optional)
|
|
104
|
+
links=[...], # ordered list of Links or Chains
|
|
105
|
+
subscribers=[...], # attached to the last link's output (optional)
|
|
106
|
+
)
|
|
107
|
+
await pipeline() # no gather() needed
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
A Chain is itself a Linkable — it can be nested inside another Chain or used as a subscriber of an external Link.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
### `Collector`
|
|
115
|
+
|
|
116
|
+
Buffers items for iteration after the pipeline completes. Supports both async and sync iteration directly.
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from io_chains.pubsub.collector import Collector
|
|
120
|
+
|
|
121
|
+
results = Collector()
|
|
122
|
+
|
|
123
|
+
# async iteration (preferred)
|
|
124
|
+
async for item in results:
|
|
125
|
+
print(item)
|
|
126
|
+
|
|
127
|
+
# sync iteration
|
|
128
|
+
for item in results:
|
|
129
|
+
print(item)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
### `CallbackSubscriber`
|
|
135
|
+
|
|
136
|
+
Calls a function for each item. Good for side effects (logging, writing, printing).
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
from io_chains.pubsub.callback_subscriber import CallbackSubscriber
|
|
140
|
+
|
|
141
|
+
sub = CallbackSubscriber(callback=lambda x: print(x))
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
## Usage Examples
|
|
147
|
+
|
|
148
|
+
### Simple transformation
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
import asyncio
|
|
152
|
+
from io_chains.links.link import Link
|
|
153
|
+
from io_chains.pubsub.collector import Collector
|
|
154
|
+
|
|
155
|
+
async def main():
|
|
156
|
+
results = Collector()
|
|
157
|
+
link = Link(
|
|
158
|
+
source=[1, 2, 3],
|
|
159
|
+
transformer=lambda x: x * 2,
|
|
160
|
+
subscribers=[results],
|
|
161
|
+
)
|
|
162
|
+
await link()
|
|
163
|
+
|
|
164
|
+
async for item in results:
|
|
165
|
+
print(item) # 2, 4, 6
|
|
166
|
+
|
|
167
|
+
asyncio.run(main())
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Multi-stage pipeline with Chain
|
|
171
|
+
|
|
172
|
+
`Chain` manages concurrent execution — no `gather()` or `create_task()` needed.
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
import asyncio
|
|
176
|
+
from io_chains.links.chain import Chain
|
|
177
|
+
from io_chains.links.link import Link
|
|
178
|
+
from io_chains.pubsub.collector import Collector
|
|
179
|
+
|
|
180
|
+
async def main():
|
|
181
|
+
results = Collector()
|
|
182
|
+
|
|
183
|
+
pipeline = Chain(
|
|
184
|
+
source=['a', 'b', 'c'],
|
|
185
|
+
links=[
|
|
186
|
+
Link(transformer=str.upper),
|
|
187
|
+
Link(transformer=lambda x: f'item: {x}'),
|
|
188
|
+
],
|
|
189
|
+
subscribers=[results],
|
|
190
|
+
)
|
|
191
|
+
await pipeline()
|
|
192
|
+
|
|
193
|
+
async for item in results:
|
|
194
|
+
print(item)
|
|
195
|
+
# item: A
|
|
196
|
+
# item: B
|
|
197
|
+
# item: C
|
|
198
|
+
|
|
199
|
+
asyncio.run(main())
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Async generator source with enrichment
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
from httpx import AsyncClient
|
|
206
|
+
from io_chains.links.chain import Chain
|
|
207
|
+
from io_chains.links.link import Link
|
|
208
|
+
from io_chains.pubsub.collector import Collector
|
|
209
|
+
|
|
210
|
+
async def fetch_records():
|
|
211
|
+
async with AsyncClient() as client:
|
|
212
|
+
response = await client.get('https://api.example.com/records')
|
|
213
|
+
for record in response.json():
|
|
214
|
+
yield record
|
|
215
|
+
|
|
216
|
+
async def main():
|
|
217
|
+
results = Collector()
|
|
218
|
+
|
|
219
|
+
pipeline = Chain(
|
|
220
|
+
source=fetch_records,
|
|
221
|
+
links=[
|
|
222
|
+
Link(transformer=lambda r: {**r, 'name': r['name'].upper()}),
|
|
223
|
+
],
|
|
224
|
+
subscribers=[results],
|
|
225
|
+
)
|
|
226
|
+
await pipeline()
|
|
227
|
+
|
|
228
|
+
async for record in results:
|
|
229
|
+
print(record)
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
### Multiple subscribers (fan-out)
|
|
233
|
+
|
|
234
|
+
Every subscriber receives every item independently.
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
from io_chains.pubsub.callback_subscriber import CallbackSubscriber
|
|
238
|
+
|
|
239
|
+
audit = []
|
|
240
|
+
results = Collector()
|
|
241
|
+
|
|
242
|
+
pipeline = Chain(
|
|
243
|
+
source=[1, 2, 3],
|
|
244
|
+
links=[Link(transformer=lambda x: x * 2)],
|
|
245
|
+
subscribers=[
|
|
246
|
+
results,
|
|
247
|
+
CallbackSubscriber(callback=lambda x: audit.append(x)),
|
|
248
|
+
],
|
|
249
|
+
)
|
|
250
|
+
await pipeline()
|
|
251
|
+
# results contains [2, 4, 6], audit == [2, 4, 6]
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
### Nested Chains
|
|
255
|
+
|
|
256
|
+
Chains can be nested — each is a black-box Linkable from the outside.
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
normalise = Chain(links=[
|
|
260
|
+
Link(transformer=lambda x: abs(x)),
|
|
261
|
+
Link(transformer=lambda x: round(x, 2)),
|
|
262
|
+
])
|
|
263
|
+
|
|
264
|
+
stringify = Chain(links=[
|
|
265
|
+
Link(transformer=lambda x: x * 100),
|
|
266
|
+
Link(transformer=lambda x: f'{x:.0f}%'),
|
|
267
|
+
])
|
|
268
|
+
|
|
269
|
+
pipeline = Chain(
|
|
270
|
+
source=[-0.156, 0.999, -0.301],
|
|
271
|
+
links=[normalise, stringify],
|
|
272
|
+
subscribers=[results],
|
|
273
|
+
)
|
|
274
|
+
await pipeline()
|
|
275
|
+
# results contains ['16%', '100%', '30%']
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
---
|
|
279
|
+
|
|
280
|
+
## Architecture
|
|
281
|
+
|
|
282
|
+
```
|
|
283
|
+
Subscriber (ABC)
|
|
284
|
+
├── CallbackSubscriber
|
|
285
|
+
└── Collector
|
|
286
|
+
|
|
287
|
+
Publisher (ABC)
|
|
288
|
+
└── Linkable(Publisher, Subscriber) (ABC)
|
|
289
|
+
├── Link
|
|
290
|
+
└── Chain
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
**`Publisher`** manages a list of subscribers and distributes data to all of them via `publish(datum)`.
|
|
294
|
+
|
|
295
|
+
**`Subscriber`** defines the `push(datum)` interface for receiving data.
|
|
296
|
+
|
|
297
|
+
**`Linkable`** combines both — it can receive data (as a subscriber in one pipeline) and emit data (as a publisher to its own subscribers).
|
|
298
|
+
|
|
299
|
+
**`Link`** implements `Linkable`. Internally it uses an `asyncio.Queue` to decouple the input reader from the subscriber publisher, allowing both to run concurrently.
|
|
300
|
+
|
|
301
|
+
**`Chain`** implements `Linkable`. It wires its internal Links/Chains together, runs them all concurrently via `gather`, and presents a single `await chain()` interface to callers.
|
|
302
|
+
|
|
303
|
+
---
|
|
304
|
+
|
|
305
|
+
## Development
|
|
306
|
+
|
|
307
|
+
```bash
|
|
308
|
+
# Install in editable mode
|
|
309
|
+
pip install -e .
|
|
310
|
+
|
|
311
|
+
# Run unit tests
|
|
312
|
+
python -m pytest test/unit -v
|
|
313
|
+
|
|
314
|
+
# Run user acceptance tests (requires network)
|
|
315
|
+
python -m pytest test/ua -v
|
|
316
|
+
```
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
# io-chains
|
|
2
|
+
|
|
3
|
+
A lightweight Python library for building async data pipelines using a publisher/subscriber pattern. Chain together data sources, transformations, and consumers with minimal boilerplate.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install oj-io-chains
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Core Concepts
|
|
12
|
+
|
|
13
|
+
### The Pipeline Model
|
|
14
|
+
|
|
15
|
+
A pipeline is built from **Links** coordinated by a **Chain**:
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
[source] → Link → [transformer] → Subscriber(s)
|
|
19
|
+
├── CallbackSubscriber (side effects)
|
|
20
|
+
├── Collector (collect results)
|
|
21
|
+
└── Link (next stage)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Each `Link`:
|
|
25
|
+
- Pulls items from a source (list, generator, async generator, or callable)
|
|
26
|
+
- Optionally transforms each item (sync or async)
|
|
27
|
+
- Publishes each item to one or more subscribers
|
|
28
|
+
|
|
29
|
+
A `Chain` wires multiple Links together and runs them concurrently — the caller just `await chain()`.
|
|
30
|
+
|
|
31
|
+
### End-of-stream
|
|
32
|
+
|
|
33
|
+
`EndOfStream` is the sentinel that signals end-of-stream through the pipeline. `None` is valid data and flows through the pipeline normally.
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
## Classes
|
|
38
|
+
|
|
39
|
+
### `Link`
|
|
40
|
+
|
|
41
|
+
A single processing unit: source → transformer → subscribers.
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
from io_chains.links.link import Link
|
|
45
|
+
|
|
46
|
+
link = Link(
|
|
47
|
+
source=..., # input source (optional)
|
|
48
|
+
transformer=..., # transform function (optional)
|
|
49
|
+
subscribers=[...], # subscribers (optional)
|
|
50
|
+
)
|
|
51
|
+
await link()
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
**Parameters**
|
|
55
|
+
|
|
56
|
+
| Parameter | Type | Description |
|
|
57
|
+
|---|---|---|
|
|
58
|
+
| `source` | `Iterable`, callable, or `AsyncIterable` | Input data source. If a callable, it is called and the result iterated. If omitted, the Link acts as a subscriber-only stage that receives data via `push()`. |
|
|
59
|
+
| `transformer` | callable | Applied to each item before publishing. Can be sync or async. |
|
|
60
|
+
| `subscribers` | `Subscriber`, callable, or list of either | Where output is sent. |
|
|
61
|
+
|
|
62
|
+
**Source types**
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
# List or any iterable
|
|
66
|
+
Link(source=[1, 2, 3])
|
|
67
|
+
|
|
68
|
+
# Generator expression
|
|
69
|
+
Link(source=(x * 2 for x in range(10)))
|
|
70
|
+
|
|
71
|
+
# Async generator function (called automatically)
|
|
72
|
+
async def my_source():
|
|
73
|
+
yield 1
|
|
74
|
+
yield 2
|
|
75
|
+
|
|
76
|
+
Link(source=my_source)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
### `Chain`
|
|
82
|
+
|
|
83
|
+
Orchestrates multiple Links as a single pipeline. Wires them together and manages concurrent execution internally.
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from io_chains.links.chain import Chain
|
|
87
|
+
|
|
88
|
+
pipeline = Chain(
|
|
89
|
+
source=..., # attached to the first link (optional)
|
|
90
|
+
links=[...], # ordered list of Links or Chains
|
|
91
|
+
subscribers=[...], # attached to the last link's output (optional)
|
|
92
|
+
)
|
|
93
|
+
await pipeline() # no gather() needed
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
A Chain is itself a Linkable — it can be nested inside another Chain or used as a subscriber of an external Link.
|
|
97
|
+
|
|
98
|
+
---
|
|
99
|
+
|
|
100
|
+
### `Collector`
|
|
101
|
+
|
|
102
|
+
Buffers items for iteration after the pipeline completes. Supports both async and sync iteration directly.
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from io_chains.pubsub.collector import Collector
|
|
106
|
+
|
|
107
|
+
results = Collector()
|
|
108
|
+
|
|
109
|
+
# async iteration (preferred)
|
|
110
|
+
async for item in results:
|
|
111
|
+
print(item)
|
|
112
|
+
|
|
113
|
+
# sync iteration
|
|
114
|
+
for item in results:
|
|
115
|
+
print(item)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
### `CallbackSubscriber`
|
|
121
|
+
|
|
122
|
+
Calls a function for each item. Good for side effects (logging, writing, printing).
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
from io_chains.pubsub.callback_subscriber import CallbackSubscriber
|
|
126
|
+
|
|
127
|
+
sub = CallbackSubscriber(callback=lambda x: print(x))
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## Usage Examples
|
|
133
|
+
|
|
134
|
+
### Simple transformation
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
import asyncio
|
|
138
|
+
from io_chains.links.link import Link
|
|
139
|
+
from io_chains.pubsub.collector import Collector
|
|
140
|
+
|
|
141
|
+
async def main():
|
|
142
|
+
results = Collector()
|
|
143
|
+
link = Link(
|
|
144
|
+
source=[1, 2, 3],
|
|
145
|
+
transformer=lambda x: x * 2,
|
|
146
|
+
subscribers=[results],
|
|
147
|
+
)
|
|
148
|
+
await link()
|
|
149
|
+
|
|
150
|
+
async for item in results:
|
|
151
|
+
print(item) # 2, 4, 6
|
|
152
|
+
|
|
153
|
+
asyncio.run(main())
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
### Multi-stage pipeline with Chain
|
|
157
|
+
|
|
158
|
+
`Chain` manages concurrent execution — no `gather()` or `create_task()` needed.
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
import asyncio
|
|
162
|
+
from io_chains.links.chain import Chain
|
|
163
|
+
from io_chains.links.link import Link
|
|
164
|
+
from io_chains.pubsub.collector import Collector
|
|
165
|
+
|
|
166
|
+
async def main():
|
|
167
|
+
results = Collector()
|
|
168
|
+
|
|
169
|
+
pipeline = Chain(
|
|
170
|
+
source=['a', 'b', 'c'],
|
|
171
|
+
links=[
|
|
172
|
+
Link(transformer=str.upper),
|
|
173
|
+
Link(transformer=lambda x: f'item: {x}'),
|
|
174
|
+
],
|
|
175
|
+
subscribers=[results],
|
|
176
|
+
)
|
|
177
|
+
await pipeline()
|
|
178
|
+
|
|
179
|
+
async for item in results:
|
|
180
|
+
print(item)
|
|
181
|
+
# item: A
|
|
182
|
+
# item: B
|
|
183
|
+
# item: C
|
|
184
|
+
|
|
185
|
+
asyncio.run(main())
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
### Async generator source with enrichment
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
from httpx import AsyncClient
|
|
192
|
+
from io_chains.links.chain import Chain
|
|
193
|
+
from io_chains.links.link import Link
|
|
194
|
+
from io_chains.pubsub.collector import Collector
|
|
195
|
+
|
|
196
|
+
async def fetch_records():
|
|
197
|
+
async with AsyncClient() as client:
|
|
198
|
+
response = await client.get('https://api.example.com/records')
|
|
199
|
+
for record in response.json():
|
|
200
|
+
yield record
|
|
201
|
+
|
|
202
|
+
async def main():
|
|
203
|
+
results = Collector()
|
|
204
|
+
|
|
205
|
+
pipeline = Chain(
|
|
206
|
+
source=fetch_records,
|
|
207
|
+
links=[
|
|
208
|
+
Link(transformer=lambda r: {**r, 'name': r['name'].upper()}),
|
|
209
|
+
],
|
|
210
|
+
subscribers=[results],
|
|
211
|
+
)
|
|
212
|
+
await pipeline()
|
|
213
|
+
|
|
214
|
+
async for record in results:
|
|
215
|
+
print(record)
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
### Multiple subscribers (fan-out)
|
|
219
|
+
|
|
220
|
+
Every subscriber receives every item independently.
|
|
221
|
+
|
|
222
|
+
```python
|
|
223
|
+
from io_chains.pubsub.callback_subscriber import CallbackSubscriber
|
|
224
|
+
|
|
225
|
+
audit = []
|
|
226
|
+
results = Collector()
|
|
227
|
+
|
|
228
|
+
pipeline = Chain(
|
|
229
|
+
source=[1, 2, 3],
|
|
230
|
+
links=[Link(transformer=lambda x: x * 2)],
|
|
231
|
+
subscribers=[
|
|
232
|
+
results,
|
|
233
|
+
CallbackSubscriber(callback=lambda x: audit.append(x)),
|
|
234
|
+
],
|
|
235
|
+
)
|
|
236
|
+
await pipeline()
|
|
237
|
+
# results contains [2, 4, 6], audit == [2, 4, 6]
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
### Nested Chains
|
|
241
|
+
|
|
242
|
+
Chains can be nested — each is a black-box Linkable from the outside.
|
|
243
|
+
|
|
244
|
+
```python
|
|
245
|
+
normalise = Chain(links=[
|
|
246
|
+
Link(transformer=lambda x: abs(x)),
|
|
247
|
+
Link(transformer=lambda x: round(x, 2)),
|
|
248
|
+
])
|
|
249
|
+
|
|
250
|
+
stringify = Chain(links=[
|
|
251
|
+
Link(transformer=lambda x: x * 100),
|
|
252
|
+
Link(transformer=lambda x: f'{x:.0f}%'),
|
|
253
|
+
])
|
|
254
|
+
|
|
255
|
+
pipeline = Chain(
|
|
256
|
+
source=[-0.156, 0.999, -0.301],
|
|
257
|
+
links=[normalise, stringify],
|
|
258
|
+
subscribers=[results],
|
|
259
|
+
)
|
|
260
|
+
await pipeline()
|
|
261
|
+
# results contains ['16%', '100%', '30%']
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
---
|
|
265
|
+
|
|
266
|
+
## Architecture
|
|
267
|
+
|
|
268
|
+
```
|
|
269
|
+
Subscriber (ABC)
|
|
270
|
+
├── CallbackSubscriber
|
|
271
|
+
└── Collector
|
|
272
|
+
|
|
273
|
+
Publisher (ABC)
|
|
274
|
+
└── Linkable(Publisher, Subscriber) (ABC)
|
|
275
|
+
├── Link
|
|
276
|
+
└── Chain
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
**`Publisher`** manages a list of subscribers and distributes data to all of them via `publish(datum)`.
|
|
280
|
+
|
|
281
|
+
**`Subscriber`** defines the `push(datum)` interface for receiving data.
|
|
282
|
+
|
|
283
|
+
**`Linkable`** combines both — it can receive data (as a subscriber in one pipeline) and emit data (as a publisher to its own subscribers).
|
|
284
|
+
|
|
285
|
+
**`Link`** implements `Linkable`. Internally it uses an `asyncio.Queue` to decouple the input reader from the subscriber publisher, allowing both to run concurrently.
|
|
286
|
+
|
|
287
|
+
**`Chain`** implements `Linkable`. It wires its internal Links/Chains together, runs them all concurrently via `gather`, and presents a single `await chain()` interface to callers.
|
|
288
|
+
|
|
289
|
+
---
|
|
290
|
+
|
|
291
|
+
## Development
|
|
292
|
+
|
|
293
|
+
```bash
|
|
294
|
+
# Install in editable mode
|
|
295
|
+
pip install -e .
|
|
296
|
+
|
|
297
|
+
# Run unit tests
|
|
298
|
+
python -m pytest test/unit -v
|
|
299
|
+
|
|
300
|
+
# Run user acceptance tests (requires network)
|
|
301
|
+
python -m pytest test/ua -v
|
|
302
|
+
```
|
|
File without changes
|
|
File without changes
|