dataloom-engine 0.5.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/PKG-INFO +47 -1
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/README.md +45 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/config.py +11 -6
- dataloom_engine-0.7.0/dataloom_engine/exceptions.py +41 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/hooks.py +5 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/sinks.py +42 -9
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/weaver.py +6 -2
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/PKG-INFO +47 -1
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/pyproject.toml +7 -6
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_config.py +12 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_core.py +38 -1
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_optional.py +2 -1
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_sinks.py +53 -1
- dataloom_engine-0.5.0/dataloom_engine/exceptions.py +0 -25
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/LICENSE +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/__init__.py +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/_optional.py +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/logs.py +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/loom.py +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/processors.py +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/py.typed +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/sources.py +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/types.py +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/SOURCES.txt +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/dependency_links.txt +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/requires.txt +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/top_level.txt +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/setup.cfg +0 -0
- {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_loom.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataloom-engine
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
|
|
5
5
|
Author-email: Dioni Padilha <dionipdl@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
21
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
22
|
Classifier: Typing :: Typed
|
|
22
23
|
Requires-Python: >=3.9
|
|
@@ -188,6 +189,51 @@ DataLoom is built around a weaving metaphor:
|
|
|
188
189
|
- **Processor:** the business logic. Turns raw data into information.
|
|
189
190
|
- **Sink:** the final destination. Where the finished product is deposited (e.g. `JsonFileSink`, `CsvFileSink`, or any destination via `CallbackSink`).
|
|
190
191
|
|
|
192
|
+
## 🧶 Threading model
|
|
193
|
+
|
|
194
|
+
Knowing which thread runs each extension point is what makes writing
|
|
195
|
+
safe Processors, Sinks and Hooks straightforward:
|
|
196
|
+
|
|
197
|
+
| Your code | Runs on |
|
|
198
|
+
| ---------------------------------------------- | ------------------------------------------------ |
|
|
199
|
+
| `Source.__iter__` | the thread that called `loom.start()` (producer) |
|
|
200
|
+
| `Processor.process` / `Sink.send` | the Weaver threads, **concurrently** |
|
|
201
|
+
| `hooks.on_error` / `hooks.on_batch_processed` | the Weaver threads, **concurrently** |
|
|
202
|
+
| `hooks.on_start` / `hooks.on_stop` / `Sink.close` | the thread that called `start()` / `stop()` |
|
|
203
|
+
|
|
204
|
+
Practical consequences:
|
|
205
|
+
|
|
206
|
+
- **Processors and Sinks must be thread-safe** if they touch shared
|
|
207
|
+
state (the built-in sinks are; `CallbackSink` callables must be).
|
|
208
|
+
- **Hooks run on the hot path**: keep `on_batch_processed` fast and
|
|
209
|
+
thread-safe.
|
|
210
|
+
- **Shutdown semantics:** `stop()` (or leaving the `with` block) drains
|
|
211
|
+
the items already queued, then joins the Weavers — pass
|
|
212
|
+
`stop(timeout=...)` to bound the wait. A pipeline whose source was
|
|
213
|
+
exhausted ends as `COMPLETED`; an interrupted one (external `stop()`,
|
|
214
|
+
`Ctrl+C`) ends as `STOPPED`; a source error ends as `FAILED`.
|
|
215
|
+
|
|
216
|
+
## 📊 Benchmarks
|
|
217
|
+
|
|
218
|
+
Reproducible, zero-dependency benchmarks live in
|
|
219
|
+
[`benchmarks/throughput.py`](benchmarks/throughput.py):
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
python benchmarks/throughput.py
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Reference numbers from a 4-vCPU Linux container (treat them as an order
|
|
226
|
+
of magnitude — run it on your own hardware):
|
|
227
|
+
|
|
228
|
+
| Scenario | Result |
|
|
229
|
+
| ------------------------------------------ | ---------------- |
|
|
230
|
+
| I/O-bound, 200 items × 10ms, 8 weavers | **7.8x** speedup over sequential |
|
|
231
|
+
| No-op processor (pure engine overhead) | ~51,000 items/s (~20µs per item) |
|
|
232
|
+
|
|
233
|
+
The rule of thumb the second number gives you: if your per-item work
|
|
234
|
+
costs less than ~20µs, a plain loop beats any orchestration — DataLoom
|
|
235
|
+
pays off when each item does real I/O or meaningful work.
|
|
236
|
+
|
|
191
237
|
## 🛠️ Development and Testing
|
|
192
238
|
|
|
193
239
|
To contribute to the project or run the test suite:
|
|
@@ -154,6 +154,51 @@ DataLoom is built around a weaving metaphor:
|
|
|
154
154
|
- **Processor:** the business logic. Turns raw data into information.
|
|
155
155
|
- **Sink:** the final destination. Where the finished product is deposited (e.g. `JsonFileSink`, `CsvFileSink`, or any destination via `CallbackSink`).
|
|
156
156
|
|
|
157
|
+
## 🧶 Threading model
|
|
158
|
+
|
|
159
|
+
Knowing which thread runs each extension point is what makes writing
|
|
160
|
+
safe Processors, Sinks and Hooks straightforward:
|
|
161
|
+
|
|
162
|
+
| Your code | Runs on |
|
|
163
|
+
| ---------------------------------------------- | ------------------------------------------------ |
|
|
164
|
+
| `Source.__iter__` | the thread that called `loom.start()` (producer) |
|
|
165
|
+
| `Processor.process` / `Sink.send` | the Weaver threads, **concurrently** |
|
|
166
|
+
| `hooks.on_error` / `hooks.on_batch_processed` | the Weaver threads, **concurrently** |
|
|
167
|
+
| `hooks.on_start` / `hooks.on_stop` / `Sink.close` | the thread that called `start()` / `stop()` |
|
|
168
|
+
|
|
169
|
+
Practical consequences:
|
|
170
|
+
|
|
171
|
+
- **Processors and Sinks must be thread-safe** if they touch shared
|
|
172
|
+
state (the built-in sinks are; `CallbackSink` callables must be).
|
|
173
|
+
- **Hooks run on the hot path**: keep `on_batch_processed` fast and
|
|
174
|
+
thread-safe.
|
|
175
|
+
- **Shutdown semantics:** `stop()` (or leaving the `with` block) drains
|
|
176
|
+
the items already queued, then joins the Weavers — pass
|
|
177
|
+
`stop(timeout=...)` to bound the wait. A pipeline whose source was
|
|
178
|
+
exhausted ends as `COMPLETED`; an interrupted one (external `stop()`,
|
|
179
|
+
`Ctrl+C`) ends as `STOPPED`; a source error ends as `FAILED`.
|
|
180
|
+
|
|
181
|
+
## 📊 Benchmarks
|
|
182
|
+
|
|
183
|
+
Reproducible, zero-dependency benchmarks live in
|
|
184
|
+
[`benchmarks/throughput.py`](benchmarks/throughput.py):
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
python benchmarks/throughput.py
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Reference numbers from a 4-vCPU Linux container (treat them as an order
|
|
191
|
+
of magnitude — run it on your own hardware):
|
|
192
|
+
|
|
193
|
+
| Scenario | Result |
|
|
194
|
+
| ------------------------------------------ | ---------------- |
|
|
195
|
+
| I/O-bound, 200 items × 10ms, 8 weavers | **7.8x** speedup over sequential |
|
|
196
|
+
| No-op processor (pure engine overhead) | ~51,000 items/s (~20µs per item) |
|
|
197
|
+
|
|
198
|
+
The rule of thumb the second number gives you: if your per-item work
|
|
199
|
+
costs less than ~20µs, a plain loop beats any orchestration — DataLoom
|
|
200
|
+
pays off when each item does real I/O or meaningful work.
|
|
201
|
+
|
|
157
202
|
## 🛠️ Development and Testing
|
|
158
203
|
|
|
159
204
|
To contribute to the project or run the test suite:
|
|
@@ -18,10 +18,14 @@ class LoomConfig:
|
|
|
18
18
|
DataLoom's main configuration object.
|
|
19
19
|
|
|
20
20
|
Args:
|
|
21
|
-
output_dir (Path): Base directory
|
|
22
|
-
Strings are converted to Path automatically.
|
|
23
|
-
|
|
24
|
-
|
|
21
|
+
output_dir (Optional[Path]): Base directory for file-based Sinks.
|
|
22
|
+
Strings are converted to Path automatically. Optional:
|
|
23
|
+
pipelines that don't write local files (CallbackSink, custom
|
|
24
|
+
sinks) don't need it — the engine core never reads it.
|
|
25
|
+
batch_size (int): Number of items produced per processing cycle
|
|
26
|
+
(used by the built-in demo Source).
|
|
27
|
+
interval_seconds (float): Time interval between task generations
|
|
28
|
+
(used by the built-in demo Source).
|
|
25
29
|
queue_maxsize (Optional[int]): Maximum capacity of the task queue
|
|
26
30
|
(backpressure). None uses the Loom default (num_weavers * 4);
|
|
27
31
|
0 means an unbounded queue.
|
|
@@ -30,13 +34,14 @@ class LoomConfig:
|
|
|
30
34
|
ConfigurationError: if any parameter is invalid.
|
|
31
35
|
"""
|
|
32
36
|
|
|
33
|
-
output_dir: Union[str, Path]
|
|
37
|
+
output_dir: Optional[Union[str, Path]] = None
|
|
34
38
|
batch_size: int = 10
|
|
35
39
|
interval_seconds: float = 1.0
|
|
36
40
|
queue_maxsize: Optional[int] = None
|
|
37
41
|
|
|
38
42
|
def __post_init__(self) -> None:
|
|
39
|
-
self.output_dir
|
|
43
|
+
if self.output_dir is not None:
|
|
44
|
+
self.output_dir = Path(self.output_dir)
|
|
40
45
|
|
|
41
46
|
if self.batch_size <= 0:
|
|
42
47
|
raise ConfigurationError(
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# dataloom_engine/exceptions.py
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Custom exceptions for DataLoom.
|
|
5
|
+
Lets consumers catch library-specific errors without relying on
|
|
6
|
+
generic Python exceptions.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from typing import Any, Optional
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class LoomError(Exception):
|
|
13
|
+
"""Base exception for every DataLoom error."""
|
|
14
|
+
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ConfigurationError(LoomError):
|
|
19
|
+
"""Raised when LoomConfig validation fails."""
|
|
20
|
+
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class WeaverError(LoomError):
|
|
25
|
+
"""
|
|
26
|
+
Raised when a Weaver fails to handle a batch.
|
|
27
|
+
|
|
28
|
+
Carries the failure context so hooks.on_error can implement retry,
|
|
29
|
+
quarantine or dead-letter logic without parsing error strings:
|
|
30
|
+
|
|
31
|
+
Attributes:
|
|
32
|
+
batch: The batch that failed, exactly as yielded by the Source.
|
|
33
|
+
stage: Where the failure happened: "process" (Processor.process)
|
|
34
|
+
or "send" (Sink.send). None when the error was constructed
|
|
35
|
+
without context.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, message: str, batch: Any = None, stage: Optional[str] = None):
|
|
39
|
+
super().__init__(message)
|
|
40
|
+
self.batch = batch
|
|
41
|
+
self.stage = stage
|
|
@@ -29,6 +29,11 @@ class LoomHooks:
|
|
|
29
29
|
Called when an exception occurs in the Loom's main loop or while
|
|
30
30
|
processing a batch inside a Weaver (WeaverError).
|
|
31
31
|
|
|
32
|
+
Batch-level failures arrive as WeaverError carrying the failed
|
|
33
|
+
batch (`error.batch`) and the stage where it broke
|
|
34
|
+
(`error.stage`: "process" or "send"), enabling retry, quarantine
|
|
35
|
+
or dead-letter handling without parsing error messages.
|
|
36
|
+
|
|
32
37
|
Note: it may be invoked from multiple Weaver threads at the same
|
|
33
38
|
time — implementations must be thread-safe.
|
|
34
39
|
"""
|
|
@@ -12,7 +12,7 @@ import queue
|
|
|
12
12
|
import threading
|
|
13
13
|
from abc import ABC, abstractmethod
|
|
14
14
|
from pathlib import Path
|
|
15
|
-
from typing import Any, Callable, Dict, Optional
|
|
15
|
+
from typing import Any, Callable, Dict, Optional, TextIO
|
|
16
16
|
|
|
17
17
|
from dataloom_engine.exceptions import LoomError
|
|
18
18
|
|
|
@@ -42,6 +42,11 @@ class JsonFileSink(Sink):
|
|
|
42
42
|
"""
|
|
43
43
|
Default sink that appends results to a local JSON-lines file.
|
|
44
44
|
Uses a threading.Lock to keep concurrent writes consistent.
|
|
45
|
+
|
|
46
|
+
The file is opened lazily on the first send() and kept open until
|
|
47
|
+
close(), avoiding an open/close syscall pair per result. Every line
|
|
48
|
+
is flushed, so results are visible to readers immediately. A send()
|
|
49
|
+
after close() transparently reopens the file in append mode.
|
|
45
50
|
"""
|
|
46
51
|
|
|
47
52
|
def __init__(self, output_dir: Path, filename: str = "results.json"):
|
|
@@ -50,12 +55,21 @@ class JsonFileSink(Sink):
|
|
|
50
55
|
self._path = self.output_dir / filename
|
|
51
56
|
# The lock ensures only one Weaver writes to the file at a time
|
|
52
57
|
self._lock = threading.Lock()
|
|
58
|
+
self._file: Optional[TextIO] = None
|
|
53
59
|
|
|
54
60
|
def send(self, result: Dict[str, Any]) -> None:
|
|
55
61
|
with self._lock:
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
62
|
+
if self._file is None:
|
|
63
|
+
self._file = open(self._path, "a")
|
|
64
|
+
json.dump(result, self._file)
|
|
65
|
+
self._file.write("\n")
|
|
66
|
+
self._file.flush()
|
|
67
|
+
|
|
68
|
+
def close(self) -> None:
|
|
69
|
+
with self._lock:
|
|
70
|
+
if self._file is not None:
|
|
71
|
+
self._file.close()
|
|
72
|
+
self._file = None
|
|
59
73
|
|
|
60
74
|
|
|
61
75
|
class CsvFileSink(Sink):
|
|
@@ -66,6 +80,12 @@ class CsvFileSink(Sink):
|
|
|
66
80
|
In subsequent results, extra keys are ignored and missing keys are
|
|
67
81
|
left empty. Uses a threading.Lock to keep concurrent writes
|
|
68
82
|
consistent.
|
|
83
|
+
|
|
84
|
+
The file is opened lazily on the first send() and kept open until
|
|
85
|
+
close(), avoiding an open/close syscall pair per row. Every row is
|
|
86
|
+
flushed, so results are visible to readers immediately. A send()
|
|
87
|
+
after close() transparently reopens the file in append mode (the
|
|
88
|
+
header is not repeated).
|
|
69
89
|
"""
|
|
70
90
|
|
|
71
91
|
def __init__(self, output_dir: Path, filename: str = "results.csv"):
|
|
@@ -74,6 +94,8 @@ class CsvFileSink(Sink):
|
|
|
74
94
|
self._path = self.output_dir / filename
|
|
75
95
|
self._lock = threading.Lock()
|
|
76
96
|
self._fieldnames: Optional[list] = None
|
|
97
|
+
self._file: Optional[TextIO] = None
|
|
98
|
+
self._writer: Optional[csv.DictWriter] = None
|
|
77
99
|
|
|
78
100
|
def send(self, result: Dict[str, Any]) -> None:
|
|
79
101
|
with self._lock:
|
|
@@ -82,11 +104,22 @@ class CsvFileSink(Sink):
|
|
|
82
104
|
if fieldnames is None:
|
|
83
105
|
fieldnames = list(result.keys())
|
|
84
106
|
self._fieldnames = fieldnames
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
writer
|
|
107
|
+
writer = self._writer
|
|
108
|
+
if self._file is None or writer is None:
|
|
109
|
+
self._file = open(self._path, "a", newline="")
|
|
110
|
+
writer = csv.DictWriter(self._file, fieldnames=fieldnames, extrasaction="ignore")
|
|
111
|
+
self._writer = writer
|
|
112
|
+
if write_header:
|
|
113
|
+
writer.writeheader()
|
|
114
|
+
writer.writerow(result)
|
|
115
|
+
self._file.flush()
|
|
116
|
+
|
|
117
|
+
def close(self) -> None:
|
|
118
|
+
with self._lock:
|
|
119
|
+
if self._file is not None:
|
|
120
|
+
self._file.close()
|
|
121
|
+
self._file = None
|
|
122
|
+
self._writer = None
|
|
90
123
|
|
|
91
124
|
|
|
92
125
|
class CallbackSink(Sink):
|
|
@@ -61,15 +61,19 @@ class Weaver(threading.Thread):
|
|
|
61
61
|
self.task_queue.task_done()
|
|
62
62
|
|
|
63
63
|
def _process_batch(self, batch: Any) -> None:
|
|
64
|
+
stage = "process"
|
|
64
65
|
try:
|
|
65
66
|
started = time.monotonic()
|
|
66
67
|
result = self.processor.process(batch)
|
|
68
|
+
stage = "send"
|
|
67
69
|
self.sink.send(result)
|
|
68
70
|
duration = time.monotonic() - started
|
|
69
71
|
except Exception as exc:
|
|
70
|
-
logger.exception("Weaver failed to
|
|
72
|
+
logger.exception("Weaver failed to %s a batch; the batch was dropped.", stage)
|
|
71
73
|
if self.on_error is not None:
|
|
72
|
-
|
|
74
|
+
# The failed batch and the stage travel with the error so
|
|
75
|
+
# hooks can retry or quarantine without parsing strings
|
|
76
|
+
error = WeaverError(f"Failed to {stage} batch: {exc}", batch=batch, stage=stage)
|
|
73
77
|
error.__cause__ = exc
|
|
74
78
|
try:
|
|
75
79
|
self.on_error(error)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataloom-engine
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
|
|
5
5
|
Author-email: Dioni Padilha <dionipdl@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
21
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
22
|
Classifier: Typing :: Typed
|
|
22
23
|
Requires-Python: >=3.9
|
|
@@ -188,6 +189,51 @@ DataLoom is built around a weaving metaphor:
|
|
|
188
189
|
- **Processor:** the business logic. Turns raw data into information.
|
|
189
190
|
- **Sink:** the final destination. Where the finished product is deposited (e.g. `JsonFileSink`, `CsvFileSink`, or any destination via `CallbackSink`).
|
|
190
191
|
|
|
192
|
+
## 🧶 Threading model
|
|
193
|
+
|
|
194
|
+
Knowing which thread runs each extension point is what makes writing
|
|
195
|
+
safe Processors, Sinks and Hooks straightforward:
|
|
196
|
+
|
|
197
|
+
| Your code | Runs on |
|
|
198
|
+
| ---------------------------------------------- | ------------------------------------------------ |
|
|
199
|
+
| `Source.__iter__` | the thread that called `loom.start()` (producer) |
|
|
200
|
+
| `Processor.process` / `Sink.send` | the Weaver threads, **concurrently** |
|
|
201
|
+
| `hooks.on_error` / `hooks.on_batch_processed` | the Weaver threads, **concurrently** |
|
|
202
|
+
| `hooks.on_start` / `hooks.on_stop` / `Sink.close` | the thread that called `start()` / `stop()` |
|
|
203
|
+
|
|
204
|
+
Practical consequences:
|
|
205
|
+
|
|
206
|
+
- **Processors and Sinks must be thread-safe** if they touch shared
|
|
207
|
+
state (the built-in sinks are; `CallbackSink` callables must be).
|
|
208
|
+
- **Hooks run on the hot path**: keep `on_batch_processed` fast and
|
|
209
|
+
thread-safe.
|
|
210
|
+
- **Shutdown semantics:** `stop()` (or leaving the `with` block) drains
|
|
211
|
+
the items already queued, then joins the Weavers — pass
|
|
212
|
+
`stop(timeout=...)` to bound the wait. A pipeline whose source was
|
|
213
|
+
exhausted ends as `COMPLETED`; an interrupted one (external `stop()`,
|
|
214
|
+
`Ctrl+C`) ends as `STOPPED`; a source error ends as `FAILED`.
|
|
215
|
+
|
|
216
|
+
## 📊 Benchmarks
|
|
217
|
+
|
|
218
|
+
Reproducible, zero-dependency benchmarks live in
|
|
219
|
+
[`benchmarks/throughput.py`](benchmarks/throughput.py):
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
python benchmarks/throughput.py
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Reference numbers from a 4-vCPU Linux container (treat them as an order
|
|
226
|
+
of magnitude — run it on your own hardware):
|
|
227
|
+
|
|
228
|
+
| Scenario | Result |
|
|
229
|
+
| ------------------------------------------ | ---------------- |
|
|
230
|
+
| I/O-bound, 200 items × 10ms, 8 weavers | **7.8x** speedup over sequential |
|
|
231
|
+
| No-op processor (pure engine overhead) | ~51,000 items/s (~20µs per item) |
|
|
232
|
+
|
|
233
|
+
The rule of thumb the second number gives you: if your per-item work
|
|
234
|
+
costs less than ~20µs, a plain loop beats any orchestration — DataLoom
|
|
235
|
+
pays off when each item does real I/O or meaningful work.
|
|
236
|
+
|
|
191
237
|
## 🛠️ Development and Testing
|
|
192
238
|
|
|
193
239
|
To contribute to the project or run the test suite:
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "dataloom-engine"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.7.0"
|
|
8
8
|
description = "DataLoom: a lightweight and efficient thread orchestration engine for data pipelines."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -23,6 +23,7 @@ classifiers = [
|
|
|
23
23
|
"Programming Language :: Python :: 3.11",
|
|
24
24
|
"Programming Language :: Python :: 3.12",
|
|
25
25
|
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Programming Language :: Python :: 3.14",
|
|
26
27
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
27
28
|
"Typing :: Typed",
|
|
28
29
|
]
|
|
@@ -68,9 +69,9 @@ select = [
|
|
|
68
69
|
"B", # flake8-bugbear
|
|
69
70
|
]
|
|
70
71
|
|
|
71
|
-
#
|
|
72
|
-
#
|
|
73
|
-
#
|
|
72
|
+
# No pinned python_version: recent mypy releases no longer accept a 3.9
|
|
73
|
+
# target, and the numpy stubs require >= 3.12. Compatibility with 3.9 is
|
|
74
|
+
# guaranteed by the CI test matrix, which runs on that version.
|
|
74
75
|
[tool.mypy]
|
|
75
76
|
files = ["dataloom_engine"]
|
|
76
77
|
check_untyped_defs = true
|
|
@@ -78,7 +79,7 @@ warn_redundant_casts = true
|
|
|
78
79
|
no_implicit_optional = true
|
|
79
80
|
|
|
80
81
|
[[tool.mypy.overrides]]
|
|
81
|
-
# numpy
|
|
82
|
-
#
|
|
82
|
+
# numpy is only used by the demo implementations; ignoring missing stubs
|
|
83
|
+
# keeps the check stable in environments without resolved type stubs
|
|
83
84
|
module = "numpy.*"
|
|
84
85
|
ignore_missing_imports = true
|
|
@@ -19,6 +19,18 @@ def test_config_coerces_output_dir_to_path():
|
|
|
19
19
|
assert isinstance(config.output_dir, Path)
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
def test_config_output_dir_is_optional():
|
|
23
|
+
"""Pipelines without file-based sinks don't need to invent a directory."""
|
|
24
|
+
config = LoomConfig()
|
|
25
|
+
assert config.output_dir is None
|
|
26
|
+
assert config.batch_size == 10 # remaining defaults still apply
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_config_validation_still_runs_without_output_dir():
|
|
30
|
+
with pytest.raises(ConfigurationError):
|
|
31
|
+
LoomConfig(batch_size=0)
|
|
32
|
+
|
|
33
|
+
|
|
22
34
|
def test_config_accepts_float_interval():
|
|
23
35
|
config = LoomConfig(output_dir=".", interval_seconds=0.25)
|
|
24
36
|
assert config.interval_seconds == 0.25
|
|
@@ -181,7 +181,44 @@ def test_weaver_handles_processor_error():
|
|
|
181
181
|
# The valid items were processed despite the error in the middle
|
|
182
182
|
assert sorted(r["sum"] for r in mock_sink.results) == [1.0, 3.0]
|
|
183
183
|
|
|
184
|
-
# The error was typed and reported
|
|
184
|
+
# The error was typed and reported, with full failure context
|
|
185
185
|
assert len(errors) == 1
|
|
186
186
|
assert isinstance(errors[0], WeaverError)
|
|
187
187
|
assert isinstance(errors[0].__cause__, ValueError)
|
|
188
|
+
assert errors[0].stage == "process"
|
|
189
|
+
assert list(errors[0].batch) == [2] # the exact batch that failed
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def test_weaver_reports_sink_failure_with_send_stage():
|
|
193
|
+
"""A Sink failure is reported as WeaverError with stage='send' and the original batch."""
|
|
194
|
+
|
|
195
|
+
class BrokenSink(Sink):
|
|
196
|
+
def send(self, result):
|
|
197
|
+
raise IOError("disk full!")
|
|
198
|
+
|
|
199
|
+
task_queue = queue.Queue()
|
|
200
|
+
errors = []
|
|
201
|
+
errors_lock = threading.Lock()
|
|
202
|
+
|
|
203
|
+
def on_error(exc):
|
|
204
|
+
with errors_lock:
|
|
205
|
+
errors.append(exc)
|
|
206
|
+
|
|
207
|
+
task_queue.put(np.array([7]))
|
|
208
|
+
task_queue.put(STOP_SENTINEL)
|
|
209
|
+
|
|
210
|
+
weaver = Weaver(
|
|
211
|
+
task_queue=task_queue,
|
|
212
|
+
processor=SimpleProcessor(),
|
|
213
|
+
sink=BrokenSink(),
|
|
214
|
+
on_error=on_error,
|
|
215
|
+
)
|
|
216
|
+
weaver.start()
|
|
217
|
+
weaver.join(timeout=2)
|
|
218
|
+
assert not weaver.is_alive()
|
|
219
|
+
|
|
220
|
+
assert len(errors) == 1
|
|
221
|
+
assert isinstance(errors[0], WeaverError)
|
|
222
|
+
assert errors[0].stage == "send"
|
|
223
|
+
assert list(errors[0].batch) == [7]
|
|
224
|
+
assert isinstance(errors[0].__cause__, IOError)
|
|
@@ -68,7 +68,8 @@ def test_pipeline_runs_without_numpy():
|
|
|
68
68
|
self.results.append(result)
|
|
69
69
|
|
|
70
70
|
sink = CollectSink()
|
|
71
|
-
|
|
71
|
+
# No file sink involved: output_dir can simply be omitted
|
|
72
|
+
config = LoomConfig(batch_size=1, interval_seconds=0)
|
|
72
73
|
with Loom(config=config, processor=SumProcessor(), sink=sink, source=ListSource()) as loom:
|
|
73
74
|
loom.start()
|
|
74
75
|
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
# tests/test_sinks.py
|
|
2
2
|
|
|
3
3
|
import csv
|
|
4
|
+
import json
|
|
4
5
|
import threading
|
|
5
6
|
|
|
6
7
|
import pytest
|
|
7
8
|
|
|
8
9
|
from dataloom_engine.exceptions import LoomError
|
|
9
|
-
from dataloom_engine.sinks import
|
|
10
|
+
from dataloom_engine.sinks import (
|
|
11
|
+
CallbackSink,
|
|
12
|
+
CsvFileSink,
|
|
13
|
+
JsonFileSink,
|
|
14
|
+
Sink,
|
|
15
|
+
ThreadedBufferedSink,
|
|
16
|
+
)
|
|
10
17
|
|
|
11
18
|
|
|
12
19
|
class MockSink(Sink):
|
|
@@ -148,6 +155,51 @@ def test_csv_sink_concurrent_writes(tmp_path):
|
|
|
148
155
|
assert ids == list(range(100))
|
|
149
156
|
|
|
150
157
|
|
|
158
|
+
def test_file_sinks_flush_results_before_close(tmp_path):
|
|
159
|
+
"""With persistent handles, every result must still be readable immediately."""
|
|
160
|
+
json_sink = JsonFileSink(output_dir=tmp_path)
|
|
161
|
+
json_sink.send({"id": 1})
|
|
162
|
+
# No close() yet: the line must already be on disk
|
|
163
|
+
assert json.loads((tmp_path / "results.json").read_text()) == {"id": 1}
|
|
164
|
+
|
|
165
|
+
csv_sink = CsvFileSink(output_dir=tmp_path)
|
|
166
|
+
csv_sink.send({"id": 1})
|
|
167
|
+
with open(tmp_path / "results.csv", newline="") as f:
|
|
168
|
+
assert list(csv.DictReader(f)) == [{"id": "1"}]
|
|
169
|
+
|
|
170
|
+
json_sink.close()
|
|
171
|
+
csv_sink.close()
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def test_file_sinks_reopen_after_close(tmp_path):
|
|
175
|
+
"""send() after close() appends transparently; the CSV header is not repeated."""
|
|
176
|
+
sink = CsvFileSink(output_dir=tmp_path)
|
|
177
|
+
sink.send({"id": 1})
|
|
178
|
+
sink.close()
|
|
179
|
+
sink.send({"id": 2})
|
|
180
|
+
sink.close()
|
|
181
|
+
|
|
182
|
+
lines = (tmp_path / "results.csv").read_text().strip().splitlines()
|
|
183
|
+
assert lines == ["id", "1", "2"] # a single header, both rows
|
|
184
|
+
|
|
185
|
+
json_sink = JsonFileSink(output_dir=tmp_path)
|
|
186
|
+
json_sink.send({"id": 1})
|
|
187
|
+
json_sink.close()
|
|
188
|
+
json_sink.send({"id": 2})
|
|
189
|
+
json_sink.close()
|
|
190
|
+
|
|
191
|
+
rows = [json.loads(line) for line in (tmp_path / "results.json").read_text().splitlines()]
|
|
192
|
+
assert rows == [{"id": 1}, {"id": 2}]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def test_file_sink_close_before_any_send_is_noop(tmp_path):
|
|
196
|
+
"""close() without any send() must not fail nor create files."""
|
|
197
|
+
JsonFileSink(output_dir=tmp_path, filename="a.json").close()
|
|
198
|
+
CsvFileSink(output_dir=tmp_path, filename="a.csv").close()
|
|
199
|
+
assert not (tmp_path / "a.json").exists()
|
|
200
|
+
assert not (tmp_path / "a.csv").exists()
|
|
201
|
+
|
|
202
|
+
|
|
151
203
|
def test_callback_sink_delegates_send_and_close():
|
|
152
204
|
received = []
|
|
153
205
|
closed = []
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
# dataloom_engine/exceptions.py
|
|
2
|
-
|
|
3
|
-
"""
|
|
4
|
-
Custom exceptions for DataLoom.
|
|
5
|
-
Lets consumers catch library-specific errors without relying on
|
|
6
|
-
generic Python exceptions.
|
|
7
|
-
"""
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class LoomError(Exception):
|
|
11
|
-
"""Base exception for every DataLoom error."""
|
|
12
|
-
|
|
13
|
-
pass
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class ConfigurationError(LoomError):
|
|
17
|
-
"""Raised when LoomConfig validation fails."""
|
|
18
|
-
|
|
19
|
-
pass
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class WeaverError(LoomError):
|
|
23
|
-
"""Raised when a Weaver fails to process a batch."""
|
|
24
|
-
|
|
25
|
-
pass
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|