dataloom-engine 0.5.0__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/PKG-INFO +47 -1
  2. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/README.md +45 -0
  3. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/config.py +11 -6
  4. dataloom_engine-0.7.0/dataloom_engine/exceptions.py +41 -0
  5. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/hooks.py +5 -0
  6. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/sinks.py +42 -9
  7. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/weaver.py +6 -2
  8. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/PKG-INFO +47 -1
  9. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/pyproject.toml +7 -6
  10. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_config.py +12 -0
  11. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_core.py +38 -1
  12. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_optional.py +2 -1
  13. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_sinks.py +53 -1
  14. dataloom_engine-0.5.0/dataloom_engine/exceptions.py +0 -25
  15. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/LICENSE +0 -0
  16. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/__init__.py +0 -0
  17. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/_optional.py +0 -0
  18. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/logs.py +0 -0
  19. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/loom.py +0 -0
  20. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/processors.py +0 -0
  21. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/py.typed +0 -0
  22. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/sources.py +0 -0
  23. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine/types.py +0 -0
  24. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/SOURCES.txt +0 -0
  25. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/dependency_links.txt +0 -0
  26. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/requires.txt +0 -0
  27. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/dataloom_engine.egg-info/top_level.txt +0 -0
  28. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/setup.cfg +0 -0
  29. {dataloom_engine-0.5.0 → dataloom_engine-0.7.0}/tests/test_loom.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataloom-engine
3
- Version: 0.5.0
3
+ Version: 0.7.0
4
4
  Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
5
5
  Author-email: Dioni Padilha <dionipdl@gmail.com>
6
6
  License: MIT
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.10
17
17
  Classifier: Programming Language :: Python :: 3.11
18
18
  Classifier: Programming Language :: Python :: 3.12
19
19
  Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
20
21
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
21
22
  Classifier: Typing :: Typed
22
23
  Requires-Python: >=3.9
@@ -188,6 +189,51 @@ DataLoom is built around a weaving metaphor:
188
189
  - **Processor:** the business logic. Turns raw data into information.
189
190
  - **Sink:** the final destination. Where the finished product is deposited (e.g. `JsonFileSink`, `CsvFileSink`, or any destination via `CallbackSink`).
190
191
 
192
+ ## 🧶 Threading model
193
+
194
+ Knowing which thread runs each extension point is what makes writing
195
+ safe Processors, Sinks and Hooks straightforward:
196
+
197
+ | Your code | Runs on |
198
+ | ---------------------------------------------- | ------------------------------------------------ |
199
+ | `Source.__iter__` | the thread that called `loom.start()` (producer) |
200
+ | `Processor.process` / `Sink.send` | the Weaver threads, **concurrently** |
201
+ | `hooks.on_error` / `hooks.on_batch_processed` | the Weaver threads, **concurrently** |
202
+ | `hooks.on_start` / `hooks.on_stop` / `Sink.close` | the thread that called `start()` / `stop()` |
203
+
204
+ Practical consequences:
205
+
206
+ - **Processors and Sinks must be thread-safe** if they touch shared
207
+ state (the built-in sinks are; `CallbackSink` callables must be).
208
+ - **Hooks run on the hot path**: keep `on_batch_processed` fast and
209
+ thread-safe.
210
+ - **Shutdown semantics:** `stop()` (or leaving the `with` block) drains
211
+ the items already queued, then joins the Weavers — pass
212
+ `stop(timeout=...)` to bound the wait. A pipeline whose source was
213
+ exhausted ends as `COMPLETED`; an interrupted one (external `stop()`,
214
+ `Ctrl+C`) ends as `STOPPED`; a source error ends as `FAILED`.
215
+
216
+ ## 📊 Benchmarks
217
+
218
+ Reproducible, zero-dependency benchmarks live in
219
+ [`benchmarks/throughput.py`](benchmarks/throughput.py):
220
+
221
+ ```bash
222
+ python benchmarks/throughput.py
223
+ ```
224
+
225
+ Reference numbers from a 4-vCPU Linux container (treat them as an order
226
+ of magnitude — run it on your own hardware):
227
+
228
+ | Scenario | Result |
229
+ | ------------------------------------------ | ---------------- |
230
+ | I/O-bound, 200 items × 10ms, 8 weavers | **7.8x** speedup over sequential |
231
+ | No-op processor (pure engine overhead) | ~51,000 items/s (~20µs per item) |
232
+
233
+ The rule of thumb the second number gives you: if your per-item work
234
+ costs less than ~20µs, a plain loop beats any orchestration — DataLoom
235
+ pays off when each item does real I/O or meaningful work.
236
+
191
237
  ## 🛠️ Development and Testing
192
238
 
193
239
  To contribute to the project or run the test suite:
@@ -154,6 +154,51 @@ DataLoom is built around a weaving metaphor:
154
154
  - **Processor:** the business logic. Turns raw data into information.
155
155
  - **Sink:** the final destination. Where the finished product is deposited (e.g. `JsonFileSink`, `CsvFileSink`, or any destination via `CallbackSink`).
156
156
 
157
+ ## 🧶 Threading model
158
+
159
+ Knowing which thread runs each extension point is what makes writing
160
+ safe Processors, Sinks and Hooks straightforward:
161
+
162
+ | Your code | Runs on |
163
+ | ---------------------------------------------- | ------------------------------------------------ |
164
+ | `Source.__iter__` | the thread that called `loom.start()` (producer) |
165
+ | `Processor.process` / `Sink.send` | the Weaver threads, **concurrently** |
166
+ | `hooks.on_error` / `hooks.on_batch_processed` | the Weaver threads, **concurrently** |
167
+ | `hooks.on_start` / `hooks.on_stop` / `Sink.close` | the thread that called `start()` / `stop()` |
168
+
169
+ Practical consequences:
170
+
171
+ - **Processors and Sinks must be thread-safe** if they touch shared
172
+ state (the built-in sinks are; `CallbackSink` callables must be).
173
+ - **Hooks run on the hot path**: keep `on_batch_processed` fast and
174
+ thread-safe.
175
+ - **Shutdown semantics:** `stop()` (or leaving the `with` block) drains
176
+ the items already queued, then joins the Weavers — pass
177
+ `stop(timeout=...)` to bound the wait. A pipeline whose source was
178
+ exhausted ends as `COMPLETED`; an interrupted one (external `stop()`,
179
+ `Ctrl+C`) ends as `STOPPED`; a source error ends as `FAILED`.
180
+
181
+ ## 📊 Benchmarks
182
+
183
+ Reproducible, zero-dependency benchmarks live in
184
+ [`benchmarks/throughput.py`](benchmarks/throughput.py):
185
+
186
+ ```bash
187
+ python benchmarks/throughput.py
188
+ ```
189
+
190
+ Reference numbers from a 4-vCPU Linux container (treat them as an order
191
+ of magnitude — run it on your own hardware):
192
+
193
+ | Scenario | Result |
194
+ | ------------------------------------------ | ---------------- |
195
+ | I/O-bound, 200 items × 10ms, 8 weavers | **7.8x** speedup over sequential |
196
+ | No-op processor (pure engine overhead) | ~51,000 items/s (~20µs per item) |
197
+
198
+ The rule of thumb the second number gives you: if your per-item work
199
+ costs less than ~20µs, a plain loop beats any orchestration — DataLoom
200
+ pays off when each item does real I/O or meaningful work.
201
+
157
202
  ## 🛠️ Development and Testing
158
203
 
159
204
  To contribute to the project or run the test suite:
@@ -18,10 +18,14 @@ class LoomConfig:
18
18
  DataLoom's main configuration object.
19
19
 
20
20
  Args:
21
- output_dir (Path): Base directory where the built-in Sinks save data.
22
- Strings are converted to Path automatically.
23
- batch_size (int): Number of items produced per processing cycle.
24
- interval_seconds (float): Time interval between task generations.
21
+ output_dir (Optional[Path]): Base directory for file-based Sinks.
22
+ Strings are converted to Path automatically. Optional:
23
+ pipelines that don't write local files (CallbackSink, custom
24
+ sinks) don't need it the engine core never reads it.
25
+ batch_size (int): Number of items produced per processing cycle
26
+ (used by the built-in demo Source).
27
+ interval_seconds (float): Time interval between task generations
28
+ (used by the built-in demo Source).
25
29
  queue_maxsize (Optional[int]): Maximum capacity of the task queue
26
30
  (backpressure). None uses the Loom default (num_weavers * 4);
27
31
  0 means an unbounded queue.
@@ -30,13 +34,14 @@ class LoomConfig:
30
34
  ConfigurationError: if any parameter is invalid.
31
35
  """
32
36
 
33
- output_dir: Union[str, Path]
37
+ output_dir: Optional[Union[str, Path]] = None
34
38
  batch_size: int = 10
35
39
  interval_seconds: float = 1.0
36
40
  queue_maxsize: Optional[int] = None
37
41
 
38
42
  def __post_init__(self) -> None:
39
- self.output_dir = Path(self.output_dir)
43
+ if self.output_dir is not None:
44
+ self.output_dir = Path(self.output_dir)
40
45
 
41
46
  if self.batch_size <= 0:
42
47
  raise ConfigurationError(
@@ -0,0 +1,41 @@
1
+ # dataloom_engine/exceptions.py
2
+
3
+ """
4
+ Custom exceptions for DataLoom.
5
+ Lets consumers catch library-specific errors without relying on
6
+ generic Python exceptions.
7
+ """
8
+
9
+ from typing import Any, Optional
10
+
11
+
12
+ class LoomError(Exception):
13
+ """Base exception for every DataLoom error."""
14
+
15
+ pass
16
+
17
+
18
+ class ConfigurationError(LoomError):
19
+ """Raised when LoomConfig validation fails."""
20
+
21
+ pass
22
+
23
+
24
+ class WeaverError(LoomError):
25
+ """
26
+ Raised when a Weaver fails to handle a batch.
27
+
28
+ Carries the failure context so hooks.on_error can implement retry,
29
+ quarantine or dead-letter logic without parsing error strings:
30
+
31
+ Attributes:
32
+ batch: The batch that failed, exactly as yielded by the Source.
33
+ stage: Where the failure happened: "process" (Processor.process)
34
+ or "send" (Sink.send). None when the error was constructed
35
+ without context.
36
+ """
37
+
38
+ def __init__(self, message: str, batch: Any = None, stage: Optional[str] = None):
39
+ super().__init__(message)
40
+ self.batch = batch
41
+ self.stage = stage
@@ -29,6 +29,11 @@ class LoomHooks:
29
29
  Called when an exception occurs in the Loom's main loop or while
30
30
  processing a batch inside a Weaver (WeaverError).
31
31
 
32
+ Batch-level failures arrive as WeaverError carrying the failed
33
+ batch (`error.batch`) and the stage where it broke
34
+ (`error.stage`: "process" or "send"), enabling retry, quarantine
35
+ or dead-letter handling without parsing error messages.
36
+
32
37
  Note: it may be invoked from multiple Weaver threads at the same
33
38
  time — implementations must be thread-safe.
34
39
  """
@@ -12,7 +12,7 @@ import queue
12
12
  import threading
13
13
  from abc import ABC, abstractmethod
14
14
  from pathlib import Path
15
- from typing import Any, Callable, Dict, Optional
15
+ from typing import Any, Callable, Dict, Optional, TextIO
16
16
 
17
17
  from dataloom_engine.exceptions import LoomError
18
18
 
@@ -42,6 +42,11 @@ class JsonFileSink(Sink):
42
42
  """
43
43
  Default sink that appends results to a local JSON-lines file.
44
44
  Uses a threading.Lock to keep concurrent writes consistent.
45
+
46
+ The file is opened lazily on the first send() and kept open until
47
+ close(), avoiding an open/close syscall pair per result. Every line
48
+ is flushed, so results are visible to readers immediately. A send()
49
+ after close() transparently reopens the file in append mode.
45
50
  """
46
51
 
47
52
  def __init__(self, output_dir: Path, filename: str = "results.json"):
@@ -50,12 +55,21 @@ class JsonFileSink(Sink):
50
55
  self._path = self.output_dir / filename
51
56
  # The lock ensures only one Weaver writes to the file at a time
52
57
  self._lock = threading.Lock()
58
+ self._file: Optional[TextIO] = None
53
59
 
54
60
  def send(self, result: Dict[str, Any]) -> None:
55
61
  with self._lock:
56
- with open(self._path, "a") as f:
57
- json.dump(result, f)
58
- f.write("\n")
62
+ if self._file is None:
63
+ self._file = open(self._path, "a")
64
+ json.dump(result, self._file)
65
+ self._file.write("\n")
66
+ self._file.flush()
67
+
68
+ def close(self) -> None:
69
+ with self._lock:
70
+ if self._file is not None:
71
+ self._file.close()
72
+ self._file = None
59
73
 
60
74
 
61
75
  class CsvFileSink(Sink):
@@ -66,6 +80,12 @@ class CsvFileSink(Sink):
66
80
  In subsequent results, extra keys are ignored and missing keys are
67
81
  left empty. Uses a threading.Lock to keep concurrent writes
68
82
  consistent.
83
+
84
+ The file is opened lazily on the first send() and kept open until
85
+ close(), avoiding an open/close syscall pair per row. Every row is
86
+ flushed, so results are visible to readers immediately. A send()
87
+ after close() transparently reopens the file in append mode (the
88
+ header is not repeated).
69
89
  """
70
90
 
71
91
  def __init__(self, output_dir: Path, filename: str = "results.csv"):
@@ -74,6 +94,8 @@ class CsvFileSink(Sink):
74
94
  self._path = self.output_dir / filename
75
95
  self._lock = threading.Lock()
76
96
  self._fieldnames: Optional[list] = None
97
+ self._file: Optional[TextIO] = None
98
+ self._writer: Optional[csv.DictWriter] = None
77
99
 
78
100
  def send(self, result: Dict[str, Any]) -> None:
79
101
  with self._lock:
@@ -82,11 +104,22 @@ class CsvFileSink(Sink):
82
104
  if fieldnames is None:
83
105
  fieldnames = list(result.keys())
84
106
  self._fieldnames = fieldnames
85
- with open(self._path, "a", newline="") as f:
86
- writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
87
- if write_header:
88
- writer.writeheader()
89
- writer.writerow(result)
107
+ writer = self._writer
108
+ if self._file is None or writer is None:
109
+ self._file = open(self._path, "a", newline="")
110
+ writer = csv.DictWriter(self._file, fieldnames=fieldnames, extrasaction="ignore")
111
+ self._writer = writer
112
+ if write_header:
113
+ writer.writeheader()
114
+ writer.writerow(result)
115
+ self._file.flush()
116
+
117
+ def close(self) -> None:
118
+ with self._lock:
119
+ if self._file is not None:
120
+ self._file.close()
121
+ self._file = None
122
+ self._writer = None
90
123
 
91
124
 
92
125
  class CallbackSink(Sink):
@@ -61,15 +61,19 @@ class Weaver(threading.Thread):
61
61
  self.task_queue.task_done()
62
62
 
63
63
  def _process_batch(self, batch: Any) -> None:
64
+ stage = "process"
64
65
  try:
65
66
  started = time.monotonic()
66
67
  result = self.processor.process(batch)
68
+ stage = "send"
67
69
  self.sink.send(result)
68
70
  duration = time.monotonic() - started
69
71
  except Exception as exc:
70
- logger.exception("Weaver failed to process a batch; the batch was dropped.")
72
+ logger.exception("Weaver failed to %s a batch; the batch was dropped.", stage)
71
73
  if self.on_error is not None:
72
- error = WeaverError(f"Failed to process batch: {exc}")
74
+ # The failed batch and the stage travel with the error so
75
+ # hooks can retry or quarantine without parsing strings
76
+ error = WeaverError(f"Failed to {stage} batch: {exc}", batch=batch, stage=stage)
73
77
  error.__cause__ = exc
74
78
  try:
75
79
  self.on_error(error)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataloom-engine
3
- Version: 0.5.0
3
+ Version: 0.7.0
4
4
  Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
5
5
  Author-email: Dioni Padilha <dionipdl@gmail.com>
6
6
  License: MIT
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.10
17
17
  Classifier: Programming Language :: Python :: 3.11
18
18
  Classifier: Programming Language :: Python :: 3.12
19
19
  Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
20
21
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
21
22
  Classifier: Typing :: Typed
22
23
  Requires-Python: >=3.9
@@ -188,6 +189,51 @@ DataLoom is built around a weaving metaphor:
188
189
  - **Processor:** the business logic. Turns raw data into information.
189
190
  - **Sink:** the final destination. Where the finished product is deposited (e.g. `JsonFileSink`, `CsvFileSink`, or any destination via `CallbackSink`).
190
191
 
192
+ ## 🧶 Threading model
193
+
194
+ Knowing which thread runs each extension point is what makes writing
195
+ safe Processors, Sinks and Hooks straightforward:
196
+
197
+ | Your code | Runs on |
198
+ | ---------------------------------------------- | ------------------------------------------------ |
199
+ | `Source.__iter__` | the thread that called `loom.start()` (producer) |
200
+ | `Processor.process` / `Sink.send` | the Weaver threads, **concurrently** |
201
+ | `hooks.on_error` / `hooks.on_batch_processed` | the Weaver threads, **concurrently** |
202
+ | `hooks.on_start` / `hooks.on_stop` / `Sink.close` | the thread that called `start()` / `stop()` |
203
+
204
+ Practical consequences:
205
+
206
+ - **Processors and Sinks must be thread-safe** if they touch shared
207
+ state (the built-in sinks are; `CallbackSink` callables must be).
208
+ - **Hooks run on the hot path**: keep `on_batch_processed` fast and
209
+ thread-safe.
210
+ - **Shutdown semantics:** `stop()` (or leaving the `with` block) drains
211
+ the items already queued, then joins the Weavers — pass
212
+ `stop(timeout=...)` to bound the wait. A pipeline whose source was
213
+ exhausted ends as `COMPLETED`; an interrupted one (external `stop()`,
214
+ `Ctrl+C`) ends as `STOPPED`; a source error ends as `FAILED`.
215
+
216
+ ## 📊 Benchmarks
217
+
218
+ Reproducible, zero-dependency benchmarks live in
219
+ [`benchmarks/throughput.py`](benchmarks/throughput.py):
220
+
221
+ ```bash
222
+ python benchmarks/throughput.py
223
+ ```
224
+
225
+ Reference numbers from a 4-vCPU Linux container (treat them as an order
226
+ of magnitude — run it on your own hardware):
227
+
228
+ | Scenario | Result |
229
+ | ------------------------------------------ | ---------------- |
230
+ | I/O-bound, 200 items × 10ms, 8 weavers | **7.8x** speedup over sequential |
231
+ | No-op processor (pure engine overhead) | ~51,000 items/s (~20µs per item) |
232
+
233
+ The rule of thumb the second number gives you: if your per-item work
234
+ costs less than ~20µs, a plain loop beats any orchestration — DataLoom
235
+ pays off when each item does real I/O or meaningful work.
236
+
191
237
  ## 🛠️ Development and Testing
192
238
 
193
239
  To contribute to the project or run the test suite:
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "dataloom-engine"
7
- version = "0.5.0"
7
+ version = "0.7.0"
8
8
  description = "DataLoom: a lightweight and efficient thread orchestration engine for data pipelines."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -23,6 +23,7 @@ classifiers = [
23
23
  "Programming Language :: Python :: 3.11",
24
24
  "Programming Language :: Python :: 3.12",
25
25
  "Programming Language :: Python :: 3.13",
26
+ "Programming Language :: Python :: 3.14",
26
27
  "Topic :: Software Development :: Libraries :: Python Modules",
27
28
  "Typing :: Typed",
28
29
  ]
@@ -68,9 +69,9 @@ select = [
68
69
  "B", # flake8-bugbear
69
70
  ]
70
71
 
71
- # Sem python_version fixo: versões novas do mypy não aceitam mais alvo 3.9,
72
- # e os stubs do numpy exigem alvo >= 3.12. A compatibilidade com 3.9 é
73
- # garantida pela matriz de testes do CI, que executa nessa versão.
72
+ # No pinned python_version: recent mypy releases no longer accept a 3.9
73
+ # target, and the numpy stubs require >= 3.12. Compatibility with 3.9 is
74
+ # guaranteed by the CI test matrix, which runs on that version.
74
75
  [tool.mypy]
75
76
  files = ["dataloom_engine"]
76
77
  check_untyped_defs = true
@@ -78,7 +79,7 @@ warn_redundant_casts = true
78
79
  no_implicit_optional = true
79
80
 
80
81
  [[tool.mypy.overrides]]
81
- # numpy é usado apenas nas implementações de demonstração; ignorar stubs
82
- # ausentes mantém o check estável em ambientes sem os type stubs resolvidos
82
+ # numpy is only used by the demo implementations; ignoring missing stubs
83
+ # keeps the check stable in environments without resolved type stubs
83
84
  module = "numpy.*"
84
85
  ignore_missing_imports = true
@@ -19,6 +19,18 @@ def test_config_coerces_output_dir_to_path():
19
19
  assert isinstance(config.output_dir, Path)
20
20
 
21
21
 
22
+ def test_config_output_dir_is_optional():
23
+ """Pipelines without file-based sinks don't need to invent a directory."""
24
+ config = LoomConfig()
25
+ assert config.output_dir is None
26
+ assert config.batch_size == 10 # remaining defaults still apply
27
+
28
+
29
+ def test_config_validation_still_runs_without_output_dir():
30
+ with pytest.raises(ConfigurationError):
31
+ LoomConfig(batch_size=0)
32
+
33
+
22
34
  def test_config_accepts_float_interval():
23
35
  config = LoomConfig(output_dir=".", interval_seconds=0.25)
24
36
  assert config.interval_seconds == 0.25
@@ -181,7 +181,44 @@ def test_weaver_handles_processor_error():
181
181
  # The valid items were processed despite the error in the middle
182
182
  assert sorted(r["sum"] for r in mock_sink.results) == [1.0, 3.0]
183
183
 
184
- # The error was typed and reported
184
+ # The error was typed and reported, with full failure context
185
185
  assert len(errors) == 1
186
186
  assert isinstance(errors[0], WeaverError)
187
187
  assert isinstance(errors[0].__cause__, ValueError)
188
+ assert errors[0].stage == "process"
189
+ assert list(errors[0].batch) == [2] # the exact batch that failed
190
+
191
+
192
+ def test_weaver_reports_sink_failure_with_send_stage():
193
+ """A Sink failure is reported as WeaverError with stage='send' and the original batch."""
194
+
195
+ class BrokenSink(Sink):
196
+ def send(self, result):
197
+ raise IOError("disk full!")
198
+
199
+ task_queue = queue.Queue()
200
+ errors = []
201
+ errors_lock = threading.Lock()
202
+
203
+ def on_error(exc):
204
+ with errors_lock:
205
+ errors.append(exc)
206
+
207
+ task_queue.put(np.array([7]))
208
+ task_queue.put(STOP_SENTINEL)
209
+
210
+ weaver = Weaver(
211
+ task_queue=task_queue,
212
+ processor=SimpleProcessor(),
213
+ sink=BrokenSink(),
214
+ on_error=on_error,
215
+ )
216
+ weaver.start()
217
+ weaver.join(timeout=2)
218
+ assert not weaver.is_alive()
219
+
220
+ assert len(errors) == 1
221
+ assert isinstance(errors[0], WeaverError)
222
+ assert errors[0].stage == "send"
223
+ assert list(errors[0].batch) == [7]
224
+ assert isinstance(errors[0].__cause__, IOError)
@@ -68,7 +68,8 @@ def test_pipeline_runs_without_numpy():
68
68
  self.results.append(result)
69
69
 
70
70
  sink = CollectSink()
71
- config = LoomConfig(output_dir=".", batch_size=1, interval_seconds=0)
71
+ # No file sink involved: output_dir can simply be omitted
72
+ config = LoomConfig(batch_size=1, interval_seconds=0)
72
73
  with Loom(config=config, processor=SumProcessor(), sink=sink, source=ListSource()) as loom:
73
74
  loom.start()
74
75
 
@@ -1,12 +1,19 @@
1
1
  # tests/test_sinks.py
2
2
 
3
3
  import csv
4
+ import json
4
5
  import threading
5
6
 
6
7
  import pytest
7
8
 
8
9
  from dataloom_engine.exceptions import LoomError
9
- from dataloom_engine.sinks import CallbackSink, CsvFileSink, Sink, ThreadedBufferedSink
10
+ from dataloom_engine.sinks import (
11
+ CallbackSink,
12
+ CsvFileSink,
13
+ JsonFileSink,
14
+ Sink,
15
+ ThreadedBufferedSink,
16
+ )
10
17
 
11
18
 
12
19
  class MockSink(Sink):
@@ -148,6 +155,51 @@ def test_csv_sink_concurrent_writes(tmp_path):
148
155
  assert ids == list(range(100))
149
156
 
150
157
 
158
+ def test_file_sinks_flush_results_before_close(tmp_path):
159
+ """With persistent handles, every result must still be readable immediately."""
160
+ json_sink = JsonFileSink(output_dir=tmp_path)
161
+ json_sink.send({"id": 1})
162
+ # No close() yet: the line must already be on disk
163
+ assert json.loads((tmp_path / "results.json").read_text()) == {"id": 1}
164
+
165
+ csv_sink = CsvFileSink(output_dir=tmp_path)
166
+ csv_sink.send({"id": 1})
167
+ with open(tmp_path / "results.csv", newline="") as f:
168
+ assert list(csv.DictReader(f)) == [{"id": "1"}]
169
+
170
+ json_sink.close()
171
+ csv_sink.close()
172
+
173
+
174
+ def test_file_sinks_reopen_after_close(tmp_path):
175
+ """send() after close() appends transparently; the CSV header is not repeated."""
176
+ sink = CsvFileSink(output_dir=tmp_path)
177
+ sink.send({"id": 1})
178
+ sink.close()
179
+ sink.send({"id": 2})
180
+ sink.close()
181
+
182
+ lines = (tmp_path / "results.csv").read_text().strip().splitlines()
183
+ assert lines == ["id", "1", "2"] # a single header, both rows
184
+
185
+ json_sink = JsonFileSink(output_dir=tmp_path)
186
+ json_sink.send({"id": 1})
187
+ json_sink.close()
188
+ json_sink.send({"id": 2})
189
+ json_sink.close()
190
+
191
+ rows = [json.loads(line) for line in (tmp_path / "results.json").read_text().splitlines()]
192
+ assert rows == [{"id": 1}, {"id": 2}]
193
+
194
+
195
+ def test_file_sink_close_before_any_send_is_noop(tmp_path):
196
+ """close() without any send() must not fail nor create files."""
197
+ JsonFileSink(output_dir=tmp_path, filename="a.json").close()
198
+ CsvFileSink(output_dir=tmp_path, filename="a.csv").close()
199
+ assert not (tmp_path / "a.json").exists()
200
+ assert not (tmp_path / "a.csv").exists()
201
+
202
+
151
203
  def test_callback_sink_delegates_send_and_close():
152
204
  received = []
153
205
  closed = []
@@ -1,25 +0,0 @@
1
- # dataloom_engine/exceptions.py
2
-
3
- """
4
- Custom exceptions for DataLoom.
5
- Lets consumers catch library-specific errors without relying on
6
- generic Python exceptions.
7
- """
8
-
9
-
10
- class LoomError(Exception):
11
- """Base exception for every DataLoom error."""
12
-
13
- pass
14
-
15
-
16
- class ConfigurationError(LoomError):
17
- """Raised when LoomConfig validation fails."""
18
-
19
- pass
20
-
21
-
22
- class WeaverError(LoomError):
23
- """Raised when a Weaver fails to process a batch."""
24
-
25
- pass
File without changes