timedbatch 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. timedbatch-0.2.0/.gitattributes +1 -0
  2. timedbatch-0.2.0/.github/FUNDING.yml +1 -0
  3. timedbatch-0.2.0/.gitignore +12 -0
  4. timedbatch-0.2.0/CHANGELOG.md +36 -0
  5. timedbatch-0.2.0/CONTRIBUTING.md +24 -0
  6. timedbatch-0.2.0/LICENSE +21 -0
  7. timedbatch-0.2.0/PKG-INFO +247 -0
  8. timedbatch-0.2.0/README.md +212 -0
  9. timedbatch-0.2.0/pyproject.toml +65 -0
  10. timedbatch-0.2.0/requirements-dev.txt +3 -0
  11. timedbatch-0.2.0/scripts/check.ps1 +7 -0
  12. timedbatch-0.2.0/scripts/release.ps1 +28 -0
  13. timedbatch-0.2.0/src/timedbatch/__init__.py +21 -0
  14. timedbatch-0.2.0/src/timedbatch/_abatcher.py +195 -0
  15. timedbatch-0.2.0/src/timedbatch/_abatching.py +91 -0
  16. timedbatch-0.2.0/src/timedbatch/_batcher.py +156 -0
  17. timedbatch-0.2.0/src/timedbatch/_batching.py +60 -0
  18. timedbatch-0.2.0/src/timedbatch/_core.py +119 -0
  19. timedbatch-0.2.0/src/timedbatch/_engine.py +116 -0
  20. timedbatch-0.2.0/src/timedbatch/_errors.py +35 -0
  21. timedbatch-0.2.0/src/timedbatch/_limits.py +52 -0
  22. timedbatch-0.2.0/src/timedbatch/py.typed +0 -0
  23. timedbatch-0.2.0/tests/conftest.py +31 -0
  24. timedbatch-0.2.0/tests/test_abatcher.py +187 -0
  25. timedbatch-0.2.0/tests/test_abatcher_edges.py +192 -0
  26. timedbatch-0.2.0/tests/test_abatching.py +158 -0
  27. timedbatch-0.2.0/tests/test_batcher_failures.py +88 -0
  28. timedbatch-0.2.0/tests/test_batcher_sync.py +128 -0
  29. timedbatch-0.2.0/tests/test_batcher_thread.py +182 -0
  30. timedbatch-0.2.0/tests/test_batcher_timing.py +40 -0
  31. timedbatch-0.2.0/tests/test_batching.py +131 -0
  32. timedbatch-0.2.0/tests/test_contract.py +34 -0
  33. timedbatch-0.2.0/tests/test_docs.py +75 -0
  34. timedbatch-0.2.0/tests/test_e2e_wheel.py +106 -0
  35. timedbatch-0.2.0/tests/test_errors.py +34 -0
  36. timedbatch-0.2.0/tests/test_exit.py +119 -0
  37. timedbatch-0.2.0/tests/test_limits.py +94 -0
  38. timedbatch-0.2.0/tests/test_model.py +99 -0
  39. timedbatch-0.2.0/tests/test_plain_sinks.py +78 -0
  40. timedbatch-0.2.0/tests/test_stress.py +139 -0
  41. timedbatch-0.2.0/tests/test_typing.py +71 -0
@@ -0,0 +1 @@
1
+ * text=auto eol=lf
@@ -0,0 +1 @@
1
+ buy_me_a_coffee: jlianesglrs
@@ -0,0 +1,12 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ venv/
5
+ build/
6
+ dist/
7
+ *.egg-info/
8
+ .pytest_cache/
9
+ .mypy_cache/
10
+ .ruff_cache/
11
+ .coverage
12
+ htmlcov/
@@ -0,0 +1,36 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
5
+ [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
+
7
+ ## [Unreleased]
8
+
9
+ ## [0.2.0]
10
+
11
+ ### Added
12
+
13
+ - `TimedBatcher`: a thread-safe push batcher. A background thread delivers batches on
14
+ size, on age (also while idle), on `flush()` and on `close()`, one at a time and in
15
+ order. `on_error` or `FlushError` hands every failed batch back; `max_pending` adds
16
+ back-pressure (`BatcherFull` after `timeout`); a forgotten batcher is closed at
17
+ interpreter exit; `background=False` runs without a thread for deterministic tests.
18
+ - `AsyncTimedBatcher`: the same for one asyncio event loop, with an async flush function.
19
+ - `abatched_by_time`: batch an async iterable, yielding partial batches on time while the
20
+ source is idle without losing the read in flight.
21
+ - `FlushError`, `BatcherClosed` and `BatcherFull`.
22
+
23
+ ### Changed
24
+
25
+ - `batched_by_time` checks its limits when called instead of on the first step, rejects
26
+ `bool`, floats and other non-integer counts and a NaN window, and accepts `math.inf`
27
+ for count-only batching. Integer types such as NumPy's (anything with `__index__`)
28
+ and `Decimal` windows keep working.
29
+ - When the source raises, `batched_by_time` yields the items already read before the
30
+ exception.
31
+
32
+ ## [0.1.0]
33
+
34
+ - First release.
35
+ - `batched_by_time`: batch an iterable by a max count and a max time window,
36
+ flushing when either limit is reached and at the end, with an injectable clock.
@@ -0,0 +1,24 @@
1
+ # Contributing
2
+
3
+ Thanks for taking the time. This project keeps a small, predictable shape.
4
+
5
+ ## Setup
6
+
7
+ ```
8
+ pip install -r requirements-dev.txt
9
+ ```
10
+
11
+ ## Before you open a pull request
12
+
13
+ - Keep public API in `src/timedbatch/__init__.py`; internals live in `_`-prefixed modules.
14
+ - Add a test for the behaviour you change. Tests inject time, disk and network where
15
+ they can. Three files use the real clock or real processes on purpose:
16
+ `test_batcher_thread.py` and the asyncio tests (idle flushes need real waiting, with
17
+ wide margins), `test_stress.py` (many threads and tasks), and `test_exit.py` (child
18
+ interpreters prove the exit flush).
19
+ - `TIMEDBATCH_E2E=1 pytest tests/test_e2e_wheel.py` builds the wheel, installs it in a
20
+ fresh virtual environment and drives the installed package.
21
+ - Run `scripts/check.ps1` (format, lint, types, tests) and make sure it passes.
22
+ - One logical change per commit, imperative subject line.
23
+
24
+ Open an issue first if the change is large or shifts the public API.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Javier Lianes
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,247 @@
1
+ Metadata-Version: 2.5
2
+ Name: timedbatch
3
+ Version: 0.2.0
4
+ Summary: Batch items by a max count and a max age, for iterables, threads and asyncio, without losing items
5
+ Project-URL: Homepage, https://lianes8studio.com
6
+ Project-URL: Repository, https://github.com/JaviLianes8/timedbatch
7
+ Project-URL: Issues, https://github.com/JaviLianes8/timedbatch/issues
8
+ Project-URL: Changelog, https://github.com/JaviLianes8/timedbatch/blob/main/CHANGELOG.md
9
+ Project-URL: LinkedIn, https://www.linkedin.com/in/jlianes/
10
+ Project-URL: Funding, https://buymeacoffee.com/jlianesglrs
11
+ Author-email: Javier Lianes <javier@lianes8studio.com>
12
+ License-Expression: MIT
13
+ License-File: LICENSE
14
+ Keywords: asyncio,back-pressure,batch,batching,buffer,chunk,flush,iterator,streaming,time-window
15
+ Classifier: Development Status :: 4 - Beta
16
+ Classifier: Framework :: AsyncIO
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3 :: Only
21
+ Classifier: Programming Language :: Python :: 3.10
22
+ Classifier: Programming Language :: Python :: 3.11
23
+ Classifier: Programming Language :: Python :: 3.12
24
+ Classifier: Programming Language :: Python :: 3.13
25
+ Classifier: Programming Language :: Python :: 3.14
26
+ Classifier: Typing :: Typed
27
+ Requires-Python: >=3.10
28
+ Provides-Extra: dev
29
+ Requires-Dist: build; extra == 'dev'
30
+ Requires-Dist: mypy; extra == 'dev'
31
+ Requires-Dist: pytest; extra == 'dev'
32
+ Requires-Dist: ruff; extra == 'dev'
33
+ Requires-Dist: twine; extra == 'dev'
34
+ Description-Content-Type: text/markdown
35
+
36
+ # timedbatch
37
+
38
+ Batch items by a max count and a max age, flushing when either limit is hit. Typed, no
39
+ dependencies, for plain iterables, for threads and for asyncio.
40
+
41
+ ## Install
42
+
43
+ ```
44
+ pip install timedbatch
45
+ ```
46
+
47
+ ## Which one to use
48
+
49
+ | You have | Use | Flushes while idle |
50
+ |---|---|---|
51
+ | An iterable you loop over | `batched_by_time` | No: the age is checked when an item arrives |
52
+ | An async iterable | `abatched_by_time` | Yes |
53
+ | Many places calling `add(item)` from threads | `TimedBatcher` | Yes, from a background thread |
54
+ | Many coroutines calling `await add(item)` | `AsyncTimedBatcher` | Yes, from a background task |
55
+
56
+ A batch is delivered as soon as it holds `max_items` items or its oldest item is
57
+ `max_seconds` old, whichever comes first. `max_seconds` may be `math.inf` to batch by
58
+ count only.
59
+
60
+ ## Iterables
61
+
62
+ ```python
63
+ from timedbatch import batched_by_time
64
+
65
+ now = 0.0
66
+
67
+
68
+ def clock() -> float: # inject a clock and tests never sleep
69
+ return now
70
+
71
+
72
+ def events():
73
+ global now
74
+ for moment, name in [(0.0, "a"), (0.5, "b"), (0.6, "c"), (9.0, "d"), (9.1, "e")]:
75
+ now = moment
76
+ yield name
77
+
78
+
79
+ for batch in batched_by_time(events(), max_items=2, max_seconds=5.0, clock=clock):
80
+ print(batch) # ['a', 'b']
81
+ # ['c', 'd']
82
+ # ['e']
83
+ ```
84
+
85
+ `['c', 'd']` closes on age: `d` arrives 8.4 seconds after `c`. If the source raises, the
86
+ items already read are yielded first and the exception follows.
87
+
88
+ ## Threads: `TimedBatcher`
89
+
90
+ ```python
91
+ from timedbatch import TimedBatcher
92
+
93
+
94
+ def write_rows(batch: list[int]) -> None:
95
+ print("insert", batch)
96
+
97
+
98
+ with TimedBatcher(write_rows, max_items=2, max_seconds=1.0) as batcher:
99
+ for row in range(5):
100
+ batcher.add(row) # safe from any number of threads
101
+ # insert [0, 1]
102
+ # insert [2, 3]
103
+ # insert [4]
104
+ ```
105
+
106
+ What it promises:
107
+
108
+ - **Size and age.** A full batch goes out at once; a partial one goes out `max_seconds`
109
+ after its oldest item, even if nothing else is added.
110
+ - **Order and no overlap.** One background thread calls your function, one batch at a
111
+ time, in the order the items were accepted. A batch never exceeds `max_items`.
112
+ - **Nothing lost, nothing twice.** Every accepted item is delivered once or handed back
113
+ once. When `add` raises, the item was not accepted (one exception: with
114
+ `background=False`, an interrupt raised inside your function passes through `add`,
115
+ and its batch comes back in the next `FlushError`).
116
+ - **Failures.** Give `on_error(exc, batch)` to decide what to do with a failed batch.
117
+ Without it, failed batches are kept and raised as a `FlushError` (with `.batches` and
118
+ `.errors`) by the next `add`, `poll`, `flush` or `close`.
119
+ - **Back-pressure.** `max_pending` caps items accepted but not yet delivered. `add` waits
120
+ for room, or raises `BatcherFull` (a `TimeoutError`) after `timeout` seconds.
121
+ - **Shutdown.** `close()` (or leaving the `with` block) delivers the rest and stops the
122
+ thread; later calls to `add` raise `BatcherClosed`. A batcher you forget to close, or
123
+ whose `close(timeout=...)` timed out, is closed when the interpreter exits normally,
124
+ through `sys.exit` or an uncaught exception (`flush_at_exit=True`). That exit flush
125
+ waits for your function, so give it its own timeouts. Nothing can flush after
126
+ `os._exit`, a kill signal, or in a child created by `os.fork` (it has no thread).
127
+ - **No deadlocks from inside.** Calling `flush`, `poll` or `close` from the flush
128
+ function raises `RuntimeError`, and an `add` there that would wait raises `BatcherFull`.
129
+
130
+ Failures come back with their items:
131
+
132
+ ```python
133
+ from timedbatch import FlushError, TimedBatcher
134
+
135
+
136
+ def flaky(batch: list[str]) -> None:
137
+ raise ConnectionError("database is down")
138
+
139
+
140
+ batcher = TimedBatcher(flaky, max_items=10, max_seconds=60.0)
141
+ batcher.add("order-1")
142
+ batcher.add("order-2")
143
+ try:
144
+ batcher.close()
145
+ except FlushError as error:
146
+ print(error.batches) # [['order-1', 'order-2']]
147
+ print(error) # 1 batch (2 items) failed to flush: ConnectionError('database is down')
148
+ ```
149
+
150
+ ### Deterministic tests
151
+
152
+ `background=False` removes the thread: due batches are delivered by the caller inside
153
+ `add` and `poll()`, so an injected clock drives every age decision (and the `timeout`
154
+ of `flush` and `close` has nothing to wait for).
155
+
156
+ ```python
157
+ from timedbatch import TimedBatcher
158
+
159
+ now = 0.0
160
+ sent: list[list[str]] = []
161
+ batcher = TimedBatcher(
162
+ sent.append, max_items=100, max_seconds=5.0, clock=lambda: now, background=False
163
+ )
164
+ batcher.add("x")
165
+ now = 4.9
166
+ batcher.poll()
167
+ print(sent) # []
168
+ now = 5.0
169
+ batcher.poll()
170
+ print(sent) # [['x']]
171
+ ```
172
+
173
+ ## asyncio
174
+
175
+ `abatched_by_time` yields a partial batch on time even while the source waits. It keeps
176
+ the pending read across those flushes (it uses `asyncio.wait`, which does not cancel on
177
+ timeout), so no item is lost. Each read runs in its own task, so context variables the
178
+ source sets do not carry from one item to the next. Closing it early drops the partial
179
+ batch and the read in flight.
180
+
181
+ ```python
182
+ import asyncio
183
+ from timedbatch import abatched_by_time
184
+
185
+
186
+ async def ticks():
187
+ yield 1
188
+ yield 2
189
+ await asyncio.sleep(0.3) # the source goes quiet
190
+ yield 3
191
+
192
+
193
+ async def main() -> None:
194
+ async for batch in abatched_by_time(ticks(), max_items=10, max_seconds=0.1):
195
+ print(batch) # [1, 2]
196
+ # [3]
197
+
198
+
199
+ asyncio.run(main())
200
+ ```
201
+
202
+ `AsyncTimedBatcher` is `TimedBatcher` for one event loop, with an async flush function
203
+ (`on_error` may be sync or async). Close it with `async with` or `await aclose()`:
204
+ coroutines cannot run at interpreter exit. If its flush task is cancelled, for example
205
+ because the loop ended first, the batcher closes and the next call hands back what it
206
+ held. `TimedBatcher` refuses an `async def` flush function instead of dropping batches.
207
+
208
+ ```python
209
+ import asyncio
210
+ from timedbatch import AsyncTimedBatcher
211
+
212
+
213
+ async def send(batch: list[str]) -> None:
214
+ print("send", batch)
215
+
216
+
217
+ async def main() -> None:
218
+ async with AsyncTimedBatcher(send, max_items=3, max_seconds=1.0, max_pending=100) as out:
219
+ for word in ["a", "b", "c", "d"]:
220
+ await out.add(word) # waits when 100 items are already pending
221
+
222
+
223
+ asyncio.run(main())
224
+ # send ['a', 'b', 'c']
225
+ # send ['d']
226
+ ```
227
+
228
+ ## Why
229
+
230
+ Count-based batching is everywhere (`itertools.batched`, `more-itertools` `chunked`),
231
+ but none of it closes a batch on elapsed time. Code that ships logs, events or metrics
232
+ almost always wants both bounds: cap the size so a batch never grows without limit, and
233
+ cap the latency so a slow stream still drains on schedule. The hard part is the rest:
234
+ flushing an idle buffer, not losing items when the sink fails or the program exits,
235
+ slowing producers down when the sink falls behind. `timedbatch` does that once, with tests
236
+ that run hundreds of random timelines against a reference model and hammer the batchers
237
+ from many threads and tasks.
238
+
239
+ ## License
240
+
241
+ MIT. See [LICENSE](LICENSE).
242
+
243
+ ## Links
244
+
245
+ - Website: https://lianes8studio.com
246
+ - GitHub: https://github.com/JaviLianes8
247
+ - LinkedIn: https://www.linkedin.com/in/jlianes/
@@ -0,0 +1,212 @@
1
+ # timedbatch
2
+
3
+ Batch items by a max count and a max age, flushing when either limit is hit. Typed, no
4
+ dependencies, for plain iterables, for threads and for asyncio.
5
+
6
+ ## Install
7
+
8
+ ```
9
+ pip install timedbatch
10
+ ```
11
+
12
+ ## Which one to use
13
+
14
+ | You have | Use | Flushes while idle |
15
+ |---|---|---|
16
+ | An iterable you loop over | `batched_by_time` | No: the age is checked when an item arrives |
17
+ | An async iterable | `abatched_by_time` | Yes |
18
+ | Many places calling `add(item)` from threads | `TimedBatcher` | Yes, from a background thread |
19
+ | Many coroutines calling `await add(item)` | `AsyncTimedBatcher` | Yes, from a background task |
20
+
21
+ A batch is delivered as soon as it holds `max_items` items or its oldest item is
22
+ `max_seconds` old, whichever comes first. `max_seconds` may be `math.inf` to batch by
23
+ count only.
24
+
25
+ ## Iterables
26
+
27
+ ```python
28
+ from timedbatch import batched_by_time
29
+
30
+ now = 0.0
31
+
32
+
33
+ def clock() -> float: # inject a clock and tests never sleep
34
+ return now
35
+
36
+
37
+ def events():
38
+ global now
39
+ for moment, name in [(0.0, "a"), (0.5, "b"), (0.6, "c"), (9.0, "d"), (9.1, "e")]:
40
+ now = moment
41
+ yield name
42
+
43
+
44
+ for batch in batched_by_time(events(), max_items=2, max_seconds=5.0, clock=clock):
45
+ print(batch) # ['a', 'b']
46
+ # ['c', 'd']
47
+ # ['e']
48
+ ```
49
+
50
+ `['c', 'd']` closes on age: `d` arrives 8.4 seconds after `c`. If the source raises, the
51
+ items already read are yielded first and the exception follows.
52
+
53
+ ## Threads: `TimedBatcher`
54
+
55
+ ```python
56
+ from timedbatch import TimedBatcher
57
+
58
+
59
+ def write_rows(batch: list[int]) -> None:
60
+ print("insert", batch)
61
+
62
+
63
+ with TimedBatcher(write_rows, max_items=2, max_seconds=1.0) as batcher:
64
+ for row in range(5):
65
+ batcher.add(row) # safe from any number of threads
66
+ # insert [0, 1]
67
+ # insert [2, 3]
68
+ # insert [4]
69
+ ```
70
+
71
+ What it promises:
72
+
73
+ - **Size and age.** A full batch goes out at once; a partial one goes out `max_seconds`
74
+ after its oldest item, even if nothing else is added.
75
+ - **Order and no overlap.** One background thread calls your function, one batch at a
76
+ time, in the order the items were accepted. A batch never exceeds `max_items`.
77
+ - **Nothing lost, nothing twice.** Every accepted item is delivered once or handed back
78
+ once. When `add` raises, the item was not accepted (one exception: with
79
+ `background=False`, an interrupt raised inside your function passes through `add`,
80
+ and its batch comes back in the next `FlushError`).
81
+ - **Failures.** Give `on_error(exc, batch)` to decide what to do with a failed batch.
82
+ Without it, failed batches are kept and raised as a `FlushError` (with `.batches` and
83
+ `.errors`) by the next `add`, `poll`, `flush` or `close`.
84
+ - **Back-pressure.** `max_pending` caps items accepted but not yet delivered. `add` waits
85
+ for room, or raises `BatcherFull` (a `TimeoutError`) after `timeout` seconds.
86
+ - **Shutdown.** `close()` (or leaving the `with` block) delivers the rest and stops the
87
+ thread; later calls to `add` raise `BatcherClosed`. A batcher you forget to close, or
88
+ whose `close(timeout=...)` timed out, is closed when the interpreter exits normally,
89
+ through `sys.exit` or an uncaught exception (`flush_at_exit=True`). That exit flush
90
+ waits for your function, so give it its own timeouts. Nothing can flush after
91
+ `os._exit`, a kill signal, or in a child created by `os.fork` (it has no thread).
92
+ - **No deadlocks from inside.** Calling `flush`, `poll` or `close` from the flush
93
+ function raises `RuntimeError`, and an `add` there that would wait raises `BatcherFull`.
94
+
95
+ Failures come back with their items:
96
+
97
+ ```python
98
+ from timedbatch import FlushError, TimedBatcher
99
+
100
+
101
+ def flaky(batch: list[str]) -> None:
102
+ raise ConnectionError("database is down")
103
+
104
+
105
+ batcher = TimedBatcher(flaky, max_items=10, max_seconds=60.0)
106
+ batcher.add("order-1")
107
+ batcher.add("order-2")
108
+ try:
109
+ batcher.close()
110
+ except FlushError as error:
111
+ print(error.batches) # [['order-1', 'order-2']]
112
+ print(error) # 1 batch (2 items) failed to flush: ConnectionError('database is down')
113
+ ```
114
+
115
+ ### Deterministic tests
116
+
117
+ `background=False` removes the thread: due batches are delivered by the caller inside
118
+ `add` and `poll()`, so an injected clock drives every age decision (and the `timeout`
119
+ of `flush` and `close` has nothing to wait for).
120
+
121
+ ```python
122
+ from timedbatch import TimedBatcher
123
+
124
+ now = 0.0
125
+ sent: list[list[str]] = []
126
+ batcher = TimedBatcher(
127
+ sent.append, max_items=100, max_seconds=5.0, clock=lambda: now, background=False
128
+ )
129
+ batcher.add("x")
130
+ now = 4.9
131
+ batcher.poll()
132
+ print(sent) # []
133
+ now = 5.0
134
+ batcher.poll()
135
+ print(sent) # [['x']]
136
+ ```
137
+
138
+ ## asyncio
139
+
140
+ `abatched_by_time` yields a partial batch on time even while the source waits. It keeps
141
+ the pending read across those flushes (it uses `asyncio.wait`, which does not cancel on
142
+ timeout), so no item is lost. Each read runs in its own task, so context variables the
143
+ source sets do not carry from one item to the next. Closing it early drops the partial
144
+ batch and the read in flight.
145
+
146
+ ```python
147
+ import asyncio
148
+ from timedbatch import abatched_by_time
149
+
150
+
151
+ async def ticks():
152
+ yield 1
153
+ yield 2
154
+ await asyncio.sleep(0.3) # the source goes quiet
155
+ yield 3
156
+
157
+
158
+ async def main() -> None:
159
+ async for batch in abatched_by_time(ticks(), max_items=10, max_seconds=0.1):
160
+ print(batch) # [1, 2]
161
+ # [3]
162
+
163
+
164
+ asyncio.run(main())
165
+ ```
166
+
167
+ `AsyncTimedBatcher` is `TimedBatcher` for one event loop, with an async flush function
168
+ (`on_error` may be sync or async). Close it with `async with` or `await aclose()`:
169
+ coroutines cannot run at interpreter exit. If its flush task is cancelled, for example
170
+ because the loop ended first, the batcher closes and the next call hands back what it
171
+ held. `TimedBatcher` refuses an `async def` flush function instead of dropping batches.
172
+
173
+ ```python
174
+ import asyncio
175
+ from timedbatch import AsyncTimedBatcher
176
+
177
+
178
+ async def send(batch: list[str]) -> None:
179
+ print("send", batch)
180
+
181
+
182
+ async def main() -> None:
183
+ async with AsyncTimedBatcher(send, max_items=3, max_seconds=1.0, max_pending=100) as out:
184
+ for word in ["a", "b", "c", "d"]:
185
+ await out.add(word) # waits when 100 items are already pending
186
+
187
+
188
+ asyncio.run(main())
189
+ # send ['a', 'b', 'c']
190
+ # send ['d']
191
+ ```
192
+
193
+ ## Why
194
+
195
+ Count-based batching is everywhere (`itertools.batched`, `more-itertools` `chunked`),
196
+ but none of it closes a batch on elapsed time. Code that ships logs, events or metrics
197
+ almost always wants both bounds: cap the size so a batch never grows without limit, and
198
+ cap the latency so a slow stream still drains on schedule. The hard part is the rest:
199
+ flushing an idle buffer, not losing items when the sink fails or the program exits,
200
+ slowing producers down when the sink falls behind. `timedbatch` does that once, with tests
201
+ that run hundreds of random timelines against a reference model and hammer the batchers
202
+ from many threads and tasks.
203
+
204
+ ## License
205
+
206
+ MIT. See [LICENSE](LICENSE).
207
+
208
+ ## Links
209
+
210
+ - Website: https://lianes8studio.com
211
+ - GitHub: https://github.com/JaviLianes8
212
+ - LinkedIn: https://www.linkedin.com/in/jlianes/
@@ -0,0 +1,65 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "timedbatch"
7
+ version = "0.2.0"
8
+ description = "Batch items by a max count and a max age, for iterables, threads and asyncio, without losing items"
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Javier Lianes", email = "javier@lianes8studio.com" }]
14
+ keywords = [
15
+ "batch", "batching", "chunk", "iterator", "time-window", "streaming", "flush",
16
+ "buffer", "asyncio", "back-pressure",
17
+ ]
18
+ classifiers = [
19
+ "Development Status :: 4 - Beta",
20
+ "Intended Audience :: Developers",
21
+ "Operating System :: OS Independent",
22
+ "Framework :: AsyncIO",
23
+ "Programming Language :: Python :: 3",
24
+ "Programming Language :: Python :: 3 :: Only",
25
+ "Programming Language :: Python :: 3.10",
26
+ "Programming Language :: Python :: 3.11",
27
+ "Programming Language :: Python :: 3.12",
28
+ "Programming Language :: Python :: 3.13",
29
+ "Programming Language :: Python :: 3.14",
30
+ "Typing :: Typed",
31
+ ]
32
+ dependencies = []
33
+
34
+ [project.optional-dependencies]
35
+ dev = ["ruff", "mypy", "pytest", "build", "twine"]
36
+
37
+ [project.urls]
38
+ Homepage = "https://lianes8studio.com"
39
+ Repository = "https://github.com/JaviLianes8/timedbatch"
40
+ Issues = "https://github.com/JaviLianes8/timedbatch/issues"
41
+ Changelog = "https://github.com/JaviLianes8/timedbatch/blob/main/CHANGELOG.md"
42
+ LinkedIn = "https://www.linkedin.com/in/jlianes/"
43
+ Funding = "https://buymeacoffee.com/jlianesglrs"
44
+
45
+ [tool.hatch.build.targets.sdist]
46
+ exclude = ["IMPROVEMENTS-round.md"]
47
+
48
+ [tool.hatch.build.targets.wheel]
49
+ packages = ["src/timedbatch"]
50
+
51
+ [tool.ruff]
52
+ line-length = 100
53
+ target-version = "py310"
54
+
55
+ [tool.ruff.lint]
56
+ select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
57
+
58
+ [tool.mypy]
59
+ python_version = "3.10"
60
+ strict = true
61
+
62
+ [tool.pytest.ini_options]
63
+ testpaths = ["tests"]
64
+ pythonpath = ["src"]
65
+ addopts = "-q"
@@ -0,0 +1,3 @@
1
+ # Development dependencies: the package in editable mode plus its dev extras
2
+ # (ruff, mypy, pytest, build, twine). Install with: pip install -r requirements-dev.txt
3
+ -e .[dev]
@@ -0,0 +1,7 @@
1
+ # Local quality gate. Run before every commit. No CI runs in the cloud.
2
+ $ErrorActionPreference = "Stop"
3
+
4
+ ruff format --check .
5
+ ruff check .
6
+ mypy --strict src tests
7
+ pytest
@@ -0,0 +1,28 @@
1
+ # Build and upload to PyPI. Run scripts/check.ps1 first.
2
+ # The name must not already exist on PyPI; the token is read from a secret store, never git.
3
+ $ErrorActionPreference = "Stop"
4
+
5
+ # Works on both Windows PowerShell 5.1 and pwsh 7 (generic catch, no PS7-only types).
6
+ $taken = $true
7
+ try {
8
+ Invoke-WebRequest -UseBasicParsing "https://pypi.org/pypi/timedbatch/json" | Out-Null
9
+ } catch {
10
+ if ($_.Exception.Response.StatusCode.value__ -eq 404) { $taken = $false } else { throw }
11
+ }
12
+ if ($taken) { throw "The name 'timedbatch' already exists on PyPI. Choose another before publishing." }
13
+
14
+ $token = $env:PYPI_TOKEN
15
+ if (-not $token -and (Test-Path "C:\secrets\pypi-token.txt")) {
16
+ $token = (Get-Content "C:\secrets\pypi-token.txt" -Raw).Trim()
17
+ }
18
+ if (-not $token) {
19
+ throw "No PyPI token. Set `$env:PYPI_TOKEN or save it to C:\secrets\pypi-token.txt (never commit it)."
20
+ }
21
+
22
+ Remove-Item -Recurse -Force dist -ErrorAction SilentlyContinue
23
+ python -m build
24
+ python -m twine check dist/*
25
+
26
+ $env:TWINE_USERNAME = "__token__"
27
+ $env:TWINE_PASSWORD = $token
28
+ try { python -m twine upload dist/* } finally { Remove-Item Env:\TWINE_PASSWORD -ErrorAction SilentlyContinue }
@@ -0,0 +1,21 @@
1
+ """Batch items by a max count and a max age, flushing when either limit is hit."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from ._abatcher import AsyncTimedBatcher
6
+ from ._abatching import abatched_by_time
7
+ from ._batcher import TimedBatcher
8
+ from ._batching import batched_by_time
9
+ from ._errors import BatcherClosed, BatcherFull, FlushError
10
+
11
+ __version__ = "0.2.0"
12
+
13
+ __all__ = [
14
+ "AsyncTimedBatcher",
15
+ "BatcherClosed",
16
+ "BatcherFull",
17
+ "FlushError",
18
+ "TimedBatcher",
19
+ "abatched_by_time",
20
+ "batched_by_time",
21
+ ]