dataloom-engine 0.5.0__tar.gz → 0.6.0__tar.gz

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.5.0
+Version: 0.6.0
 Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/dataloom_engine/config.py

@@ -18,10 +18,14 @@ class LoomConfig:
     DataLoom's main configuration object.
 
     Args:
-        output_dir (Path): Base directory where the built-in Sinks save data.
-            Strings are converted to Path automatically.
-        batch_size (int): Number of items produced per processing cycle.
-        interval_seconds (float): Time interval between task generations.
+        output_dir (Optional[Path]): Base directory for file-based Sinks.
+            Strings are converted to Path automatically. Optional:
+            pipelines that don't write local files (CallbackSink, custom
+            sinks) don't need it — the engine core never reads it.
+        batch_size (int): Number of items produced per processing cycle
+            (used by the built-in demo Source).
+        interval_seconds (float): Time interval between task generations
+            (used by the built-in demo Source).
         queue_maxsize (Optional[int]): Maximum capacity of the task queue
             (backpressure). None uses the Loom default (num_weavers * 4);
             0 means an unbounded queue.
@@ -30,13 +34,14 @@ class LoomConfig:
         ConfigurationError: if any parameter is invalid.
     """
 
-    output_dir: Union[str, Path]
+    output_dir: Optional[Union[str, Path]] = None
     batch_size: int = 10
     interval_seconds: float = 1.0
     queue_maxsize: Optional[int] = None
 
     def __post_init__(self) -> None:
-        self.output_dir = Path(self.output_dir)
+        if self.output_dir is not None:
+            self.output_dir = Path(self.output_dir)
 
         if self.batch_size <= 0:
             raise ConfigurationError(

dataloom_engine-0.6.0/dataloom_engine/exceptions.py

@@ -0,0 +1,41 @@
+# dataloom_engine/exceptions.py
+
+"""
+Custom exceptions for DataLoom.
+Lets consumers catch library-specific errors without relying on
+generic Python exceptions.
+"""
+
+from typing import Any, Optional
+
+
+class LoomError(Exception):
+    """Base exception for every DataLoom error."""
+
+    pass
+
+
+class ConfigurationError(LoomError):
+    """Raised when LoomConfig validation fails."""
+
+    pass
+
+
+class WeaverError(LoomError):
+    """
+    Raised when a Weaver fails to handle a batch.
+
+    Carries the failure context so hooks.on_error can implement retry,
+    quarantine or dead-letter logic without parsing error strings:
+
+    Attributes:
+        batch: The batch that failed, exactly as yielded by the Source.
+        stage: Where the failure happened: "process" (Processor.process)
+            or "send" (Sink.send). None when the error was constructed
+            without context.
+    """
+
+    def __init__(self, message: str, batch: Any = None, stage: Optional[str] = None):
+        super().__init__(message)
+        self.batch = batch
+        self.stage = stage

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/dataloom_engine/hooks.py

@@ -29,6 +29,11 @@ class LoomHooks:
         Called when an exception occurs in the Loom's main loop or while
         processing a batch inside a Weaver (WeaverError).
 
+        Batch-level failures arrive as WeaverError carrying the failed
+        batch (`error.batch`) and the stage where it broke
+        (`error.stage`: "process" or "send"), enabling retry, quarantine
+        or dead-letter handling without parsing error messages.
+
         Note: it may be invoked from multiple Weaver threads at the same
         time — implementations must be thread-safe.
         """

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/dataloom_engine/sinks.py

@@ -12,7 +12,7 @@ import queue
 import threading
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict, Optional, TextIO
 
 from dataloom_engine.exceptions import LoomError
 
@@ -42,6 +42,11 @@ class JsonFileSink(Sink):
     """
     Default sink that appends results to a local JSON-lines file.
     Uses a threading.Lock to keep concurrent writes consistent.
+
+    The file is opened lazily on the first send() and kept open until
+    close(), avoiding an open/close syscall pair per result. Every line
+    is flushed, so results are visible to readers immediately. A send()
+    after close() transparently reopens the file in append mode.
     """
 
     def __init__(self, output_dir: Path, filename: str = "results.json"):
@@ -50,12 +55,21 @@ class JsonFileSink(Sink):
         self._path = self.output_dir / filename
         # The lock ensures only one Weaver writes to the file at a time
         self._lock = threading.Lock()
+        self._file: Optional[TextIO] = None
 
     def send(self, result: Dict[str, Any]) -> None:
         with self._lock:
-            with open(self._path, "a") as f:
-                json.dump(result, f)
-                f.write("\n")
+            if self._file is None:
+                self._file = open(self._path, "a")
+            json.dump(result, self._file)
+            self._file.write("\n")
+            self._file.flush()
+
+    def close(self) -> None:
+        with self._lock:
+            if self._file is not None:
+                self._file.close()
+                self._file = None
 
 
 class CsvFileSink(Sink):
@@ -66,6 +80,12 @@ class CsvFileSink(Sink):
     In subsequent results, extra keys are ignored and missing keys are
     left empty. Uses a threading.Lock to keep concurrent writes
     consistent.
+
+    The file is opened lazily on the first send() and kept open until
+    close(), avoiding an open/close syscall pair per row. Every row is
+    flushed, so results are visible to readers immediately. A send()
+    after close() transparently reopens the file in append mode (the
+    header is not repeated).
     """
 
     def __init__(self, output_dir: Path, filename: str = "results.csv"):
@@ -74,6 +94,8 @@ class CsvFileSink(Sink):
         self._path = self.output_dir / filename
         self._lock = threading.Lock()
         self._fieldnames: Optional[list] = None
+        self._file: Optional[TextIO] = None
+        self._writer: Optional[csv.DictWriter] = None
 
     def send(self, result: Dict[str, Any]) -> None:
         with self._lock:
@@ -82,11 +104,22 @@ class CsvFileSink(Sink):
             if fieldnames is None:
                 fieldnames = list(result.keys())
                 self._fieldnames = fieldnames
-            with open(self._path, "a", newline="") as f:
-                writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
-                if write_header:
-                    writer.writeheader()
-                writer.writerow(result)
+            writer = self._writer
+            if self._file is None or writer is None:
+                self._file = open(self._path, "a", newline="")
+                writer = csv.DictWriter(self._file, fieldnames=fieldnames, extrasaction="ignore")
+                self._writer = writer
+            if write_header:
+                writer.writeheader()
+            writer.writerow(result)
+            self._file.flush()
+
+    def close(self) -> None:
+        with self._lock:
+            if self._file is not None:
+                self._file.close()
+                self._file = None
+                self._writer = None
 
 
 class CallbackSink(Sink):

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/dataloom_engine/weaver.py

@@ -61,15 +61,19 @@ class Weaver(threading.Thread):
                 self.task_queue.task_done()
 
     def _process_batch(self, batch: Any) -> None:
+        stage = "process"
         try:
             started = time.monotonic()
             result = self.processor.process(batch)
+            stage = "send"
             self.sink.send(result)
             duration = time.monotonic() - started
         except Exception as exc:
-            logger.exception("Weaver failed to process a batch; the batch was dropped.")
+            logger.exception("Weaver failed to %s a batch; the batch was dropped.", stage)
             if self.on_error is not None:
-                error = WeaverError(f"Failed to process batch: {exc}")
+                # The failed batch and the stage travel with the error so
+                # hooks can retry or quarantine without parsing strings
+                error = WeaverError(f"Failed to {stage} batch: {exc}", batch=batch, stage=stage)
                 error.__cause__ = exc
                 try:
                     self.on_error(error)

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/dataloom_engine.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.5.0
+Version: 0.6.0
 Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataloom-engine"
-version = "0.5.0"
+version = "0.6.0"
 description = "DataLoom: a lightweight and efficient thread orchestration engine for data pipelines."
 readme = "README.md"
 requires-python = ">=3.9"

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/tests/test_config.py

@@ -19,6 +19,18 @@ def test_config_coerces_output_dir_to_path():
     assert isinstance(config.output_dir, Path)
 
 
+def test_config_output_dir_is_optional():
+    """Pipelines without file-based sinks don't need to invent a directory."""
+    config = LoomConfig()
+    assert config.output_dir is None
+    assert config.batch_size == 10  # remaining defaults still apply
+
+
+def test_config_validation_still_runs_without_output_dir():
+    with pytest.raises(ConfigurationError):
+        LoomConfig(batch_size=0)
+
+
 def test_config_accepts_float_interval():
     config = LoomConfig(output_dir=".", interval_seconds=0.25)
     assert config.interval_seconds == 0.25

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/tests/test_core.py

@@ -181,7 +181,44 @@ def test_weaver_handles_processor_error():
     # The valid items were processed despite the error in the middle
     assert sorted(r["sum"] for r in mock_sink.results) == [1.0, 3.0]
 
-    # The error was typed and reported
+    # The error was typed and reported, with full failure context
     assert len(errors) == 1
     assert isinstance(errors[0], WeaverError)
     assert isinstance(errors[0].__cause__, ValueError)
+    assert errors[0].stage == "process"
+    assert list(errors[0].batch) == [2]  # the exact batch that failed
+
+
+def test_weaver_reports_sink_failure_with_send_stage():
+    """A Sink failure is reported as WeaverError with stage='send' and the original batch."""
+
+    class BrokenSink(Sink):
+        def send(self, result):
+            raise IOError("disk full!")
+
+    task_queue = queue.Queue()
+    errors = []
+    errors_lock = threading.Lock()
+
+    def on_error(exc):
+        with errors_lock:
+            errors.append(exc)
+
+    task_queue.put(np.array([7]))
+    task_queue.put(STOP_SENTINEL)
+
+    weaver = Weaver(
+        task_queue=task_queue,
+        processor=SimpleProcessor(),
+        sink=BrokenSink(),
+        on_error=on_error,
+    )
+    weaver.start()
+    weaver.join(timeout=2)
+    assert not weaver.is_alive()
+
+    assert len(errors) == 1
+    assert isinstance(errors[0], WeaverError)
+    assert errors[0].stage == "send"
+    assert list(errors[0].batch) == [7]
+    assert isinstance(errors[0].__cause__, IOError)

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/tests/test_optional.py

@@ -68,7 +68,8 @@ def test_pipeline_runs_without_numpy():
             self.results.append(result)
 
     sink = CollectSink()
-    config = LoomConfig(output_dir=".", batch_size=1, interval_seconds=0)
+    # No file sink involved: output_dir can simply be omitted
+    config = LoomConfig(batch_size=1, interval_seconds=0)
     with Loom(config=config, processor=SumProcessor(), sink=sink, source=ListSource()) as loom:
         loom.start()
 

{dataloom_engine-0.5.0 → dataloom_engine-0.6.0}/tests/test_sinks.py

@@ -1,12 +1,19 @@
 # tests/test_sinks.py
 
 import csv
+import json
 import threading
 
 import pytest
 
 from dataloom_engine.exceptions import LoomError
-from dataloom_engine.sinks import CallbackSink, CsvFileSink, Sink, ThreadedBufferedSink
+from dataloom_engine.sinks import (
+    CallbackSink,
+    CsvFileSink,
+    JsonFileSink,
+    Sink,
+    ThreadedBufferedSink,
+)
 
 
 class MockSink(Sink):
@@ -148,6 +155,51 @@ def test_csv_sink_concurrent_writes(tmp_path):
     assert ids == list(range(100))
 
 
+def test_file_sinks_flush_results_before_close(tmp_path):
+    """With persistent handles, every result must still be readable immediately."""
+    json_sink = JsonFileSink(output_dir=tmp_path)
+    json_sink.send({"id": 1})
+    # No close() yet: the line must already be on disk
+    assert json.loads((tmp_path / "results.json").read_text()) == {"id": 1}
+
+    csv_sink = CsvFileSink(output_dir=tmp_path)
+    csv_sink.send({"id": 1})
+    with open(tmp_path / "results.csv", newline="") as f:
+        assert list(csv.DictReader(f)) == [{"id": "1"}]
+
+    json_sink.close()
+    csv_sink.close()
+
+
+def test_file_sinks_reopen_after_close(tmp_path):
+    """send() after close() appends transparently; the CSV header is not repeated."""
+    sink = CsvFileSink(output_dir=tmp_path)
+    sink.send({"id": 1})
+    sink.close()
+    sink.send({"id": 2})
+    sink.close()
+
+    lines = (tmp_path / "results.csv").read_text().strip().splitlines()
+    assert lines == ["id", "1", "2"]  # a single header, both rows
+
+    json_sink = JsonFileSink(output_dir=tmp_path)
+    json_sink.send({"id": 1})
+    json_sink.close()
+    json_sink.send({"id": 2})
+    json_sink.close()
+
+    rows = [json.loads(line) for line in (tmp_path / "results.json").read_text().splitlines()]
+    assert rows == [{"id": 1}, {"id": 2}]
+
+
+def test_file_sink_close_before_any_send_is_noop(tmp_path):
+    """close() without any send() must not fail nor create files."""
+    JsonFileSink(output_dir=tmp_path, filename="a.json").close()
+    CsvFileSink(output_dir=tmp_path, filename="a.csv").close()
+    assert not (tmp_path / "a.json").exists()
+    assert not (tmp_path / "a.csv").exists()
+
+
 def test_callback_sink_delegates_send_and_close():
     received = []
     closed = []

dataloom_engine-0.5.0/dataloom_engine/exceptions.py

@@ -1,25 +0,0 @@
-# dataloom_engine/exceptions.py
-
-"""
-Custom exceptions for DataLoom.
-Lets consumers catch library-specific errors without relying on
-generic Python exceptions.
-"""
-
-
-class LoomError(Exception):
-    """Base exception for every DataLoom error."""
-
-    pass
-
-
-class ConfigurationError(LoomError):
-    """Raised when LoomConfig validation fails."""
-
-    pass
-
-
-class WeaverError(LoomError):
-    """Raised when a Weaver fails to process a batch."""
-
-    pass