timedbatch 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- timedbatch-0.2.0/.gitattributes +1 -0
- timedbatch-0.2.0/.github/FUNDING.yml +1 -0
- timedbatch-0.2.0/.gitignore +12 -0
- timedbatch-0.2.0/CHANGELOG.md +36 -0
- timedbatch-0.2.0/CONTRIBUTING.md +24 -0
- timedbatch-0.2.0/LICENSE +21 -0
- timedbatch-0.2.0/PKG-INFO +247 -0
- timedbatch-0.2.0/README.md +212 -0
- timedbatch-0.2.0/pyproject.toml +65 -0
- timedbatch-0.2.0/requirements-dev.txt +3 -0
- timedbatch-0.2.0/scripts/check.ps1 +7 -0
- timedbatch-0.2.0/scripts/release.ps1 +28 -0
- timedbatch-0.2.0/src/timedbatch/__init__.py +21 -0
- timedbatch-0.2.0/src/timedbatch/_abatcher.py +195 -0
- timedbatch-0.2.0/src/timedbatch/_abatching.py +91 -0
- timedbatch-0.2.0/src/timedbatch/_batcher.py +156 -0
- timedbatch-0.2.0/src/timedbatch/_batching.py +60 -0
- timedbatch-0.2.0/src/timedbatch/_core.py +119 -0
- timedbatch-0.2.0/src/timedbatch/_engine.py +116 -0
- timedbatch-0.2.0/src/timedbatch/_errors.py +35 -0
- timedbatch-0.2.0/src/timedbatch/_limits.py +52 -0
- timedbatch-0.2.0/src/timedbatch/py.typed +0 -0
- timedbatch-0.2.0/tests/conftest.py +31 -0
- timedbatch-0.2.0/tests/test_abatcher.py +187 -0
- timedbatch-0.2.0/tests/test_abatcher_edges.py +192 -0
- timedbatch-0.2.0/tests/test_abatching.py +158 -0
- timedbatch-0.2.0/tests/test_batcher_failures.py +88 -0
- timedbatch-0.2.0/tests/test_batcher_sync.py +128 -0
- timedbatch-0.2.0/tests/test_batcher_thread.py +182 -0
- timedbatch-0.2.0/tests/test_batcher_timing.py +40 -0
- timedbatch-0.2.0/tests/test_batching.py +131 -0
- timedbatch-0.2.0/tests/test_contract.py +34 -0
- timedbatch-0.2.0/tests/test_docs.py +75 -0
- timedbatch-0.2.0/tests/test_e2e_wheel.py +106 -0
- timedbatch-0.2.0/tests/test_errors.py +34 -0
- timedbatch-0.2.0/tests/test_exit.py +119 -0
- timedbatch-0.2.0/tests/test_limits.py +94 -0
- timedbatch-0.2.0/tests/test_model.py +99 -0
- timedbatch-0.2.0/tests/test_plain_sinks.py +78 -0
- timedbatch-0.2.0/tests/test_stress.py +139 -0
- timedbatch-0.2.0/tests/test_typing.py +71 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
* text=auto eol=lf
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
buy_me_a_coffee: jlianesglrs
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
|
|
5
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.2.0]
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `TimedBatcher`: a thread-safe push batcher. A background thread delivers batches on
|
|
14
|
+
size, on age (also while idle), on `flush()` and on `close()`, one at a time and in
|
|
15
|
+
order. `on_error` or `FlushError` hands every failed batch back; `max_pending` adds
|
|
16
|
+
back-pressure (`BatcherFull` after `timeout`); a forgotten batcher is closed at
|
|
17
|
+
interpreter exit; `background=False` runs without a thread for deterministic tests.
|
|
18
|
+
- `AsyncTimedBatcher`: the same for one asyncio event loop, with an async flush function.
|
|
19
|
+
- `abatched_by_time`: batch an async iterable, yielding partial batches on time while the
|
|
20
|
+
source is idle without losing the read in flight.
|
|
21
|
+
- `FlushError`, `BatcherClosed` and `BatcherFull`.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
|
|
25
|
+
- `batched_by_time` checks its limits when called instead of on the first step, rejects
|
|
26
|
+
`bool`, floats and other non-integer counts and a NaN window, and accepts `math.inf`
|
|
27
|
+
for count-only batching. Integer types such as NumPy's (anything with `__index__`)
|
|
28
|
+
and `Decimal` windows keep working.
|
|
29
|
+
- When the source raises, `batched_by_time` yields the items already read before the
|
|
30
|
+
exception.
|
|
31
|
+
|
|
32
|
+
## [0.1.0]
|
|
33
|
+
|
|
34
|
+
- First release.
|
|
35
|
+
- `batched_by_time`: batch an iterable by a max count and a max time window,
|
|
36
|
+
flushing when either limit is reached and at the end, with an injectable clock.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Thanks for taking the time. This project keeps a small, predictable shape.
|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
pip install -r requirements-dev.txt
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Before you open a pull request
|
|
12
|
+
|
|
13
|
+
- Keep public API in `src/timedbatch/__init__.py`; internals live in `_`-prefixed modules.
|
|
14
|
+
- Add a test for the behaviour you change. Tests inject time, disk and network where
|
|
15
|
+
they can. Three files use the real clock or real processes on purpose:
|
|
16
|
+
`test_batcher_thread.py` and the asyncio tests (idle flushes need real waiting, with
|
|
17
|
+
wide margins), `test_stress.py` (many threads and tasks), and `test_exit.py` (child
|
|
18
|
+
interpreters prove the exit flush).
|
|
19
|
+
- `TIMEDBATCH_E2E=1 pytest tests/test_e2e_wheel.py` builds the wheel, installs it in a
|
|
20
|
+
fresh virtual environment and drives the installed package.
|
|
21
|
+
- Run `scripts/check.ps1` (format, lint, types, tests) and make sure it passes.
|
|
22
|
+
- One logical change per commit, imperative subject line.
|
|
23
|
+
|
|
24
|
+
Open an issue first if the change is large or shifts the public API.
|
timedbatch-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Javier Lianes
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: timedbatch
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Batch items by a max count and a max age, for iterables, threads and asyncio, without losing items
|
|
5
|
+
Project-URL: Homepage, https://lianes8studio.com
|
|
6
|
+
Project-URL: Repository, https://github.com/JaviLianes8/timedbatch
|
|
7
|
+
Project-URL: Issues, https://github.com/JaviLianes8/timedbatch/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/JaviLianes8/timedbatch/blob/main/CHANGELOG.md
|
|
9
|
+
Project-URL: LinkedIn, https://www.linkedin.com/in/jlianes/
|
|
10
|
+
Project-URL: Funding, https://buymeacoffee.com/jlianesglrs
|
|
11
|
+
Author-email: Javier Lianes <javier@lianes8studio.com>
|
|
12
|
+
License-Expression: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: asyncio,back-pressure,batch,batching,buffer,chunk,flush,iterator,streaming,time-window
|
|
15
|
+
Classifier: Development Status :: 4 - Beta
|
|
16
|
+
Classifier: Framework :: AsyncIO
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Python: >=3.10
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: build; extra == 'dev'
|
|
30
|
+
Requires-Dist: mypy; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
33
|
+
Requires-Dist: twine; extra == 'dev'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# timedbatch
|
|
37
|
+
|
|
38
|
+
Batch items by a max count and a max age, flushing when either limit is hit. Typed, no
|
|
39
|
+
dependencies, for plain iterables, for threads and for asyncio.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
pip install timedbatch
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Which one to use
|
|
48
|
+
|
|
49
|
+
| You have | Use | Flushes while idle |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| An iterable you loop over | `batched_by_time` | No: the age is checked when an item arrives |
|
|
52
|
+
| An async iterable | `abatched_by_time` | Yes |
|
|
53
|
+
| Many places calling `add(item)` from threads | `TimedBatcher` | Yes, from a background thread |
|
|
54
|
+
| Many coroutines calling `await add(item)` | `AsyncTimedBatcher` | Yes, from a background task |
|
|
55
|
+
|
|
56
|
+
A batch is delivered as soon as it holds `max_items` items or its oldest item is
|
|
57
|
+
`max_seconds` old, whichever comes first. `max_seconds` may be `math.inf` to batch by
|
|
58
|
+
count only.
|
|
59
|
+
|
|
60
|
+
## Iterables
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from timedbatch import batched_by_time
|
|
64
|
+
|
|
65
|
+
now = 0.0
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def clock() -> float: # inject a clock and tests never sleep
|
|
69
|
+
return now
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def events():
|
|
73
|
+
global now
|
|
74
|
+
for moment, name in [(0.0, "a"), (0.5, "b"), (0.6, "c"), (9.0, "d"), (9.1, "e")]:
|
|
75
|
+
now = moment
|
|
76
|
+
yield name
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
for batch in batched_by_time(events(), max_items=2, max_seconds=5.0, clock=clock):
|
|
80
|
+
print(batch) # ['a', 'b']
|
|
81
|
+
# ['c', 'd']
|
|
82
|
+
# ['e']
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`['c', 'd']` closes on age: `d` arrives 8.4 seconds after `c`. If the source raises, the
|
|
86
|
+
items already read are yielded first and the exception follows.
|
|
87
|
+
|
|
88
|
+
## Threads: `TimedBatcher`
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from timedbatch import TimedBatcher
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def write_rows(batch: list[int]) -> None:
|
|
95
|
+
print("insert", batch)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
with TimedBatcher(write_rows, max_items=2, max_seconds=1.0) as batcher:
|
|
99
|
+
for row in range(5):
|
|
100
|
+
batcher.add(row) # safe from any number of threads
|
|
101
|
+
# insert [0, 1]
|
|
102
|
+
# insert [2, 3]
|
|
103
|
+
# insert [4]
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
What it promises:
|
|
107
|
+
|
|
108
|
+
- **Size and age.** A full batch goes out at once; a partial one goes out `max_seconds`
|
|
109
|
+
after its oldest item, even if nothing else is added.
|
|
110
|
+
- **Order and no overlap.** One background thread calls your function, one batch at a
|
|
111
|
+
time, in the order the items were accepted. A batch never exceeds `max_items`.
|
|
112
|
+
- **Nothing lost, nothing twice.** Every accepted item is delivered once or handed back
|
|
113
|
+
once. When `add` raises, the item was not accepted (one exception: with
|
|
114
|
+
`background=False`, an interrupt raised inside your function passes through `add`,
|
|
115
|
+
and its batch comes back in the next `FlushError`).
|
|
116
|
+
- **Failures.** Give `on_error(exc, batch)` to decide what to do with a failed batch.
|
|
117
|
+
Without it, failed batches are kept and raised as a `FlushError` (with `.batches` and
|
|
118
|
+
`.errors`) by the next `add`, `poll`, `flush` or `close`.
|
|
119
|
+
- **Back-pressure.** `max_pending` caps items accepted but not yet delivered. `add` waits
|
|
120
|
+
for room, or raises `BatcherFull` (a `TimeoutError`) after `timeout` seconds.
|
|
121
|
+
- **Shutdown.** `close()` (or leaving the `with` block) delivers the rest and stops the
|
|
122
|
+
thread; later calls to `add` raise `BatcherClosed`. A batcher you forget to close, or
|
|
123
|
+
whose `close(timeout=...)` timed out, is closed when the interpreter exits normally,
|
|
124
|
+
through `sys.exit` or an uncaught exception (`flush_at_exit=True`). That exit flush
|
|
125
|
+
waits for your function, so give it its own timeouts. Nothing can flush after
|
|
126
|
+
`os._exit`, a kill signal, or in a child created by `os.fork` (it has no thread).
|
|
127
|
+
- **No deadlocks from inside.** Calling `flush`, `poll` or `close` from the flush
|
|
128
|
+
function raises `RuntimeError`, and an `add` there that would wait raises `BatcherFull`.
|
|
129
|
+
|
|
130
|
+
Failures come back with their items:
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from timedbatch import FlushError, TimedBatcher
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def flaky(batch: list[str]) -> None:
|
|
137
|
+
raise ConnectionError("database is down")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
batcher = TimedBatcher(flaky, max_items=10, max_seconds=60.0)
|
|
141
|
+
batcher.add("order-1")
|
|
142
|
+
batcher.add("order-2")
|
|
143
|
+
try:
|
|
144
|
+
batcher.close()
|
|
145
|
+
except FlushError as error:
|
|
146
|
+
print(error.batches) # [['order-1', 'order-2']]
|
|
147
|
+
print(error) # 1 batch (2 items) failed to flush: ConnectionError('database is down')
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### Deterministic tests
|
|
151
|
+
|
|
152
|
+
`background=False` removes the thread: due batches are delivered by the caller inside
|
|
153
|
+
`add` and `poll()`, so an injected clock drives every age decision (and the `timeout`
|
|
154
|
+
of `flush` and `close` has nothing to wait for).
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
from timedbatch import TimedBatcher
|
|
158
|
+
|
|
159
|
+
now = 0.0
|
|
160
|
+
sent: list[list[str]] = []
|
|
161
|
+
batcher = TimedBatcher(
|
|
162
|
+
sent.append, max_items=100, max_seconds=5.0, clock=lambda: now, background=False
|
|
163
|
+
)
|
|
164
|
+
batcher.add("x")
|
|
165
|
+
now = 4.9
|
|
166
|
+
batcher.poll()
|
|
167
|
+
print(sent) # []
|
|
168
|
+
now = 5.0
|
|
169
|
+
batcher.poll()
|
|
170
|
+
print(sent) # [['x']]
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
## asyncio
|
|
174
|
+
|
|
175
|
+
`abatched_by_time` yields a partial batch on time even while the source waits. It keeps
|
|
176
|
+
the pending read across those flushes (it uses `asyncio.wait`, which does not cancel on
|
|
177
|
+
timeout), so no item is lost. Each read runs in its own task, so context variables the
|
|
178
|
+
source sets do not carry from one item to the next. Closing it early drops the partial
|
|
179
|
+
batch and the read in flight.
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
import asyncio
|
|
183
|
+
from timedbatch import abatched_by_time
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
async def ticks():
|
|
187
|
+
yield 1
|
|
188
|
+
yield 2
|
|
189
|
+
await asyncio.sleep(0.3) # the source goes quiet
|
|
190
|
+
yield 3
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
async def main() -> None:
|
|
194
|
+
async for batch in abatched_by_time(ticks(), max_items=10, max_seconds=0.1):
|
|
195
|
+
print(batch) # [1, 2]
|
|
196
|
+
# [3]
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
asyncio.run(main())
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
`AsyncTimedBatcher` is `TimedBatcher` for one event loop, with an async flush function
|
|
203
|
+
(`on_error` may be sync or async). Close it with `async with` or `await aclose()`:
|
|
204
|
+
coroutines cannot run at interpreter exit. If its flush task is cancelled, for example
|
|
205
|
+
because the loop ended first, the batcher closes and the next call hands back what it
|
|
206
|
+
held. `TimedBatcher` refuses an `async def` flush function instead of dropping batches.
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
import asyncio
|
|
210
|
+
from timedbatch import AsyncTimedBatcher
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
async def send(batch: list[str]) -> None:
|
|
214
|
+
print("send", batch)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
async def main() -> None:
|
|
218
|
+
async with AsyncTimedBatcher(send, max_items=3, max_seconds=1.0, max_pending=100) as out:
|
|
219
|
+
for word in ["a", "b", "c", "d"]:
|
|
220
|
+
await out.add(word) # waits when 100 items are already pending
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
asyncio.run(main())
|
|
224
|
+
# send ['a', 'b', 'c']
|
|
225
|
+
# send ['d']
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
## Why
|
|
229
|
+
|
|
230
|
+
Count-based batching is everywhere (`itertools.batched`, `more-itertools` `chunked`),
|
|
231
|
+
but none of it closes a batch on elapsed time. Code that ships logs, events or metrics
|
|
232
|
+
almost always wants both bounds: cap the size so a batch never grows without limit, and
|
|
233
|
+
cap the latency so a slow stream still drains on schedule. The hard part is the rest:
|
|
234
|
+
flushing an idle buffer, not losing items when the sink fails or the program exits,
|
|
235
|
+
slowing producers down when the sink falls behind. `timedbatch` does that once, with tests
|
|
236
|
+
that run hundreds of random timelines against a reference model and hammer the batchers
|
|
237
|
+
from many threads and tasks.
|
|
238
|
+
|
|
239
|
+
## License
|
|
240
|
+
|
|
241
|
+
MIT. See [LICENSE](LICENSE).
|
|
242
|
+
|
|
243
|
+
## Links
|
|
244
|
+
|
|
245
|
+
- Website: https://lianes8studio.com
|
|
246
|
+
- GitHub: https://github.com/JaviLianes8
|
|
247
|
+
- LinkedIn: https://www.linkedin.com/in/jlianes/
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# timedbatch
|
|
2
|
+
|
|
3
|
+
Batch items by a max count and a max age, flushing when either limit is hit. Typed, no
|
|
4
|
+
dependencies, for plain iterables, for threads and for asyncio.
|
|
5
|
+
|
|
6
|
+
## Install
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
pip install timedbatch
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Which one to use
|
|
13
|
+
|
|
14
|
+
| You have | Use | Flushes while idle |
|
|
15
|
+
|---|---|---|
|
|
16
|
+
| An iterable you loop over | `batched_by_time` | No: the age is checked when an item arrives |
|
|
17
|
+
| An async iterable | `abatched_by_time` | Yes |
|
|
18
|
+
| Many places calling `add(item)` from threads | `TimedBatcher` | Yes, from a background thread |
|
|
19
|
+
| Many coroutines calling `await add(item)` | `AsyncTimedBatcher` | Yes, from a background task |
|
|
20
|
+
|
|
21
|
+
A batch is delivered as soon as it holds `max_items` items or its oldest item is
|
|
22
|
+
`max_seconds` old, whichever comes first. `max_seconds` may be `math.inf` to batch by
|
|
23
|
+
count only.
|
|
24
|
+
|
|
25
|
+
## Iterables
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
from timedbatch import batched_by_time
|
|
29
|
+
|
|
30
|
+
now = 0.0
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def clock() -> float: # inject a clock and tests never sleep
|
|
34
|
+
return now
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def events():
|
|
38
|
+
global now
|
|
39
|
+
for moment, name in [(0.0, "a"), (0.5, "b"), (0.6, "c"), (9.0, "d"), (9.1, "e")]:
|
|
40
|
+
now = moment
|
|
41
|
+
yield name
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
for batch in batched_by_time(events(), max_items=2, max_seconds=5.0, clock=clock):
|
|
45
|
+
print(batch) # ['a', 'b']
|
|
46
|
+
# ['c', 'd']
|
|
47
|
+
# ['e']
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
`['c', 'd']` closes on age: `d` arrives 8.4 seconds after `c`. If the source raises, the
|
|
51
|
+
items already read are yielded first and the exception follows.
|
|
52
|
+
|
|
53
|
+
## Threads: `TimedBatcher`
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
from timedbatch import TimedBatcher
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def write_rows(batch: list[int]) -> None:
|
|
60
|
+
print("insert", batch)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
with TimedBatcher(write_rows, max_items=2, max_seconds=1.0) as batcher:
|
|
64
|
+
for row in range(5):
|
|
65
|
+
batcher.add(row) # safe from any number of threads
|
|
66
|
+
# insert [0, 1]
|
|
67
|
+
# insert [2, 3]
|
|
68
|
+
# insert [4]
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
What it promises:
|
|
72
|
+
|
|
73
|
+
- **Size and age.** A full batch goes out at once; a partial one goes out `max_seconds`
|
|
74
|
+
after its oldest item, even if nothing else is added.
|
|
75
|
+
- **Order and no overlap.** One background thread calls your function, one batch at a
|
|
76
|
+
time, in the order the items were accepted. A batch never exceeds `max_items`.
|
|
77
|
+
- **Nothing lost, nothing twice.** Every accepted item is delivered once or handed back
|
|
78
|
+
once. When `add` raises, the item was not accepted (one exception: with
|
|
79
|
+
`background=False`, an interrupt raised inside your function passes through `add`,
|
|
80
|
+
and its batch comes back in the next `FlushError`).
|
|
81
|
+
- **Failures.** Give `on_error(exc, batch)` to decide what to do with a failed batch.
|
|
82
|
+
Without it, failed batches are kept and raised as a `FlushError` (with `.batches` and
|
|
83
|
+
`.errors`) by the next `add`, `poll`, `flush` or `close`.
|
|
84
|
+
- **Back-pressure.** `max_pending` caps items accepted but not yet delivered. `add` waits
|
|
85
|
+
for room, or raises `BatcherFull` (a `TimeoutError`) after `timeout` seconds.
|
|
86
|
+
- **Shutdown.** `close()` (or leaving the `with` block) delivers the rest and stops the
|
|
87
|
+
thread; later calls to `add` raise `BatcherClosed`. A batcher you forget to close, or
|
|
88
|
+
whose `close(timeout=...)` timed out, is closed when the interpreter exits normally,
|
|
89
|
+
through `sys.exit` or an uncaught exception (`flush_at_exit=True`). That exit flush
|
|
90
|
+
waits for your function, so give it its own timeouts. Nothing can flush after
|
|
91
|
+
`os._exit`, a kill signal, or in a child created by `os.fork` (it has no thread).
|
|
92
|
+
- **No deadlocks from inside.** Calling `flush`, `poll` or `close` from the flush
|
|
93
|
+
function raises `RuntimeError`, and an `add` there that would wait raises `BatcherFull`.
|
|
94
|
+
|
|
95
|
+
Failures come back with their items:
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from timedbatch import FlushError, TimedBatcher
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def flaky(batch: list[str]) -> None:
|
|
102
|
+
raise ConnectionError("database is down")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
batcher = TimedBatcher(flaky, max_items=10, max_seconds=60.0)
|
|
106
|
+
batcher.add("order-1")
|
|
107
|
+
batcher.add("order-2")
|
|
108
|
+
try:
|
|
109
|
+
batcher.close()
|
|
110
|
+
except FlushError as error:
|
|
111
|
+
print(error.batches) # [['order-1', 'order-2']]
|
|
112
|
+
print(error) # 1 batch (2 items) failed to flush: ConnectionError('database is down')
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Deterministic tests
|
|
116
|
+
|
|
117
|
+
`background=False` removes the thread: due batches are delivered by the caller inside
|
|
118
|
+
`add` and `poll()`, so an injected clock drives every age decision (and the `timeout`
|
|
119
|
+
of `flush` and `close` has nothing to wait for).
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
from timedbatch import TimedBatcher
|
|
123
|
+
|
|
124
|
+
now = 0.0
|
|
125
|
+
sent: list[list[str]] = []
|
|
126
|
+
batcher = TimedBatcher(
|
|
127
|
+
sent.append, max_items=100, max_seconds=5.0, clock=lambda: now, background=False
|
|
128
|
+
)
|
|
129
|
+
batcher.add("x")
|
|
130
|
+
now = 4.9
|
|
131
|
+
batcher.poll()
|
|
132
|
+
print(sent) # []
|
|
133
|
+
now = 5.0
|
|
134
|
+
batcher.poll()
|
|
135
|
+
print(sent) # [['x']]
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## asyncio
|
|
139
|
+
|
|
140
|
+
`abatched_by_time` yields a partial batch on time even while the source waits. It keeps
|
|
141
|
+
the pending read across those flushes (it uses `asyncio.wait`, which does not cancel on
|
|
142
|
+
timeout), so no item is lost. Each read runs in its own task, so context variables the
|
|
143
|
+
source sets do not carry from one item to the next. Closing it early drops the partial
|
|
144
|
+
batch and the read in flight.
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
import asyncio
|
|
148
|
+
from timedbatch import abatched_by_time
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
async def ticks():
|
|
152
|
+
yield 1
|
|
153
|
+
yield 2
|
|
154
|
+
await asyncio.sleep(0.3) # the source goes quiet
|
|
155
|
+
yield 3
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
async def main() -> None:
|
|
159
|
+
async for batch in abatched_by_time(ticks(), max_items=10, max_seconds=0.1):
|
|
160
|
+
print(batch) # [1, 2]
|
|
161
|
+
# [3]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
asyncio.run(main())
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
`AsyncTimedBatcher` is `TimedBatcher` for one event loop, with an async flush function
|
|
168
|
+
(`on_error` may be sync or async). Close it with `async with` or `await aclose()`:
|
|
169
|
+
coroutines cannot run at interpreter exit. If its flush task is cancelled, for example
|
|
170
|
+
because the loop ended first, the batcher closes and the next call hands back what it
|
|
171
|
+
held. `TimedBatcher` refuses an `async def` flush function instead of dropping batches.
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
import asyncio
|
|
175
|
+
from timedbatch import AsyncTimedBatcher
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
async def send(batch: list[str]) -> None:
|
|
179
|
+
print("send", batch)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
async def main() -> None:
|
|
183
|
+
async with AsyncTimedBatcher(send, max_items=3, max_seconds=1.0, max_pending=100) as out:
|
|
184
|
+
for word in ["a", "b", "c", "d"]:
|
|
185
|
+
await out.add(word) # waits when 100 items are already pending
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
asyncio.run(main())
|
|
189
|
+
# send ['a', 'b', 'c']
|
|
190
|
+
# send ['d']
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
## Why
|
|
194
|
+
|
|
195
|
+
Count-based batching is everywhere (`itertools.batched`, `more-itertools` `chunked`),
|
|
196
|
+
but none of it closes a batch on elapsed time. Code that ships logs, events or metrics
|
|
197
|
+
almost always wants both bounds: cap the size so a batch never grows without limit, and
|
|
198
|
+
cap the latency so a slow stream still drains on schedule. The hard part is the rest:
|
|
199
|
+
flushing an idle buffer, not losing items when the sink fails or the program exits,
|
|
200
|
+
slowing producers down when the sink falls behind. `timedbatch` does that once, with tests
|
|
201
|
+
that run hundreds of random timelines against a reference model and hammer the batchers
|
|
202
|
+
from many threads and tasks.
|
|
203
|
+
|
|
204
|
+
## License
|
|
205
|
+
|
|
206
|
+
MIT. See [LICENSE](LICENSE).
|
|
207
|
+
|
|
208
|
+
## Links
|
|
209
|
+
|
|
210
|
+
- Website: https://lianes8studio.com
|
|
211
|
+
- GitHub: https://github.com/JaviLianes8
|
|
212
|
+
- LinkedIn: https://www.linkedin.com/in/jlianes/
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "timedbatch"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Batch items by a max count and a max age, for iterables, threads and asyncio, without losing items"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Javier Lianes", email = "javier@lianes8studio.com" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"batch", "batching", "chunk", "iterator", "time-window", "streaming", "flush",
|
|
16
|
+
"buffer", "asyncio", "back-pressure",
|
|
17
|
+
]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 4 - Beta",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"Operating System :: OS Independent",
|
|
22
|
+
"Framework :: AsyncIO",
|
|
23
|
+
"Programming Language :: Python :: 3",
|
|
24
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Programming Language :: Python :: 3.13",
|
|
29
|
+
"Programming Language :: Python :: 3.14",
|
|
30
|
+
"Typing :: Typed",
|
|
31
|
+
]
|
|
32
|
+
dependencies = []
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
dev = ["ruff", "mypy", "pytest", "build", "twine"]
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://lianes8studio.com"
|
|
39
|
+
Repository = "https://github.com/JaviLianes8/timedbatch"
|
|
40
|
+
Issues = "https://github.com/JaviLianes8/timedbatch/issues"
|
|
41
|
+
Changelog = "https://github.com/JaviLianes8/timedbatch/blob/main/CHANGELOG.md"
|
|
42
|
+
LinkedIn = "https://www.linkedin.com/in/jlianes/"
|
|
43
|
+
Funding = "https://buymeacoffee.com/jlianesglrs"
|
|
44
|
+
|
|
45
|
+
[tool.hatch.build.targets.sdist]
|
|
46
|
+
exclude = ["IMPROVEMENTS-round.md"]
|
|
47
|
+
|
|
48
|
+
[tool.hatch.build.targets.wheel]
|
|
49
|
+
packages = ["src/timedbatch"]
|
|
50
|
+
|
|
51
|
+
[tool.ruff]
|
|
52
|
+
line-length = 100
|
|
53
|
+
target-version = "py310"
|
|
54
|
+
|
|
55
|
+
[tool.ruff.lint]
|
|
56
|
+
select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
|
|
57
|
+
|
|
58
|
+
[tool.mypy]
|
|
59
|
+
python_version = "3.10"
|
|
60
|
+
strict = true
|
|
61
|
+
|
|
62
|
+
[tool.pytest.ini_options]
|
|
63
|
+
testpaths = ["tests"]
|
|
64
|
+
pythonpath = ["src"]
|
|
65
|
+
addopts = "-q"
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Build and upload to PyPI. Run scripts/check.ps1 first.
|
|
2
|
+
# The name must not already exist on PyPI; the token is read from a secret store, never git.
|
|
3
|
+
$ErrorActionPreference = "Stop"
|
|
4
|
+
|
|
5
|
+
# Works on both Windows PowerShell 5.1 and pwsh 7 (generic catch, no PS7-only types).
|
|
6
|
+
$taken = $true
|
|
7
|
+
try {
|
|
8
|
+
Invoke-WebRequest -UseBasicParsing "https://pypi.org/pypi/timedbatch/json" | Out-Null
|
|
9
|
+
} catch {
|
|
10
|
+
if ($_.Exception.Response.StatusCode.value__ -eq 404) { $taken = $false } else { throw }
|
|
11
|
+
}
|
|
12
|
+
if ($taken) { throw "The name 'timedbatch' already exists on PyPI. Choose another before publishing." }
|
|
13
|
+
|
|
14
|
+
$token = $env:PYPI_TOKEN
|
|
15
|
+
if (-not $token -and (Test-Path "C:\secrets\pypi-token.txt")) {
|
|
16
|
+
$token = (Get-Content "C:\secrets\pypi-token.txt" -Raw).Trim()
|
|
17
|
+
}
|
|
18
|
+
if (-not $token) {
|
|
19
|
+
throw "No PyPI token. Set `$env:PYPI_TOKEN or save it to C:\secrets\pypi-token.txt (never commit it)."
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
Remove-Item -Recurse -Force dist -ErrorAction SilentlyContinue
|
|
23
|
+
python -m build
|
|
24
|
+
python -m twine check dist/*
|
|
25
|
+
|
|
26
|
+
$env:TWINE_USERNAME = "__token__"
|
|
27
|
+
$env:TWINE_PASSWORD = $token
|
|
28
|
+
try { python -m twine upload dist/* } finally { Remove-Item Env:\TWINE_PASSWORD -ErrorAction SilentlyContinue }
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Batch items by a max count and a max age, flushing when either limit is hit."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from ._abatcher import AsyncTimedBatcher
|
|
6
|
+
from ._abatching import abatched_by_time
|
|
7
|
+
from ._batcher import TimedBatcher
|
|
8
|
+
from ._batching import batched_by_time
|
|
9
|
+
from ._errors import BatcherClosed, BatcherFull, FlushError
|
|
10
|
+
|
|
11
|
+
__version__ = "0.2.0"
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"AsyncTimedBatcher",
|
|
15
|
+
"BatcherClosed",
|
|
16
|
+
"BatcherFull",
|
|
17
|
+
"FlushError",
|
|
18
|
+
"TimedBatcher",
|
|
19
|
+
"abatched_by_time",
|
|
20
|
+
"batched_by_time",
|
|
21
|
+
]
|