dataloom-engine 0.4.1__tar.gz → 0.6.0__tar.gz

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.4.1
+Version: 0.6.0
 Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/dataloom_engine/config.py

@@ -18,10 +18,14 @@ class LoomConfig:
     DataLoom's main configuration object.
 
     Args:
-        output_dir (Path): Base directory where the built-in Sinks save data.
-            Strings are converted to Path automatically.
-        batch_size (int): Number of items produced per processing cycle.
-        interval_seconds (float): Time interval between task generations.
+        output_dir (Optional[Path]): Base directory for file-based Sinks.
+            Strings are converted to Path automatically. Optional:
+            pipelines that don't write local files (CallbackSink, custom
+            sinks) don't need it — the engine core never reads it.
+        batch_size (int): Number of items produced per processing cycle
+            (used by the built-in demo Source).
+        interval_seconds (float): Time interval between task generations
+            (used by the built-in demo Source).
         queue_maxsize (Optional[int]): Maximum capacity of the task queue
             (backpressure). None uses the Loom default (num_weavers * 4);
             0 means an unbounded queue.
@@ -30,13 +34,14 @@ class LoomConfig:
         ConfigurationError: if any parameter is invalid.
     """
 
-    output_dir: Union[str, Path]
+    output_dir: Optional[Union[str, Path]] = None
     batch_size: int = 10
     interval_seconds: float = 1.0
     queue_maxsize: Optional[int] = None
 
     def __post_init__(self) -> None:
-        self.output_dir = Path(self.output_dir)
+        if self.output_dir is not None:
+            self.output_dir = Path(self.output_dir)
 
         if self.batch_size <= 0:
             raise ConfigurationError(

dataloom_engine-0.6.0/dataloom_engine/exceptions.py

@@ -0,0 +1,41 @@
+# dataloom_engine/exceptions.py
+
+"""
+Custom exceptions for DataLoom.
+Lets consumers catch library-specific errors without relying on
+generic Python exceptions.
+"""
+
+from typing import Any, Optional
+
+
+class LoomError(Exception):
+    """Base exception for every DataLoom error."""
+
+    pass
+
+
+class ConfigurationError(LoomError):
+    """Raised when LoomConfig validation fails."""
+
+    pass
+
+
+class WeaverError(LoomError):
+    """
+    Raised when a Weaver fails to handle a batch.
+
+    Carries the failure context so hooks.on_error can implement retry,
+    quarantine or dead-letter logic without parsing error strings:
+
+    Attributes:
+        batch: The batch that failed, exactly as yielded by the Source.
+        stage: Where the failure happened: "process" (Processor.process)
+            or "send" (Sink.send). None when the error was constructed
+            without context.
+    """
+
+    def __init__(self, message: str, batch: Any = None, stage: Optional[str] = None):
+        super().__init__(message)
+        self.batch = batch
+        self.stage = stage

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/dataloom_engine/hooks.py

@@ -29,6 +29,11 @@ class LoomHooks:
         Called when an exception occurs in the Loom's main loop or while
         processing a batch inside a Weaver (WeaverError).
 
+        Batch-level failures arrive as WeaverError carrying the failed
+        batch (`error.batch`) and the stage where it broke
+        (`error.stage`: "process" or "send"), enabling retry, quarantine
+        or dead-letter handling without parsing error messages.
+
         Note: it may be invoked from multiple Weaver threads at the same
         time — implementations must be thread-safe.
         """

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/dataloom_engine/loom.py

@@ -6,8 +6,10 @@ Defines the main class responsible for managing the lifecycle of the
 worker threads (Weavers) and the distribution of tasks.
 """
 
+import logging
 import queue
 import threading
+import time
 from typing import TYPE_CHECKING, Optional
 
 from dataloom_engine.config import LoomConfig
@@ -21,6 +23,8 @@ from dataloom_engine.weaver import STOP_SENTINEL, Weaver
 if TYPE_CHECKING:
     from dataloom_engine.sources import Source
 
+logger = logging.getLogger(__name__)
+
 
 class Loom:
     """
@@ -80,6 +84,8 @@ class Loom:
         self.weavers: list[Weaver] = []
         self._stop_lock = threading.Lock()
         self._stopped = False
+        # Distinguishes COMPLETED (source exhausted) from STOPPED (interrupted)
+        self._source_exhausted = False
 
     def __enter__(self) -> "Loom":
         return self
@@ -127,6 +133,11 @@ class Loom:
                 if self.stop_event.is_set():
                     break
                 self._enqueue(batch)
+            else:
+                # The for/else only runs when the loop ended without a
+                # break: the source was exhausted naturally, so stop()
+                # may report COMPLETED instead of STOPPED.
+                self._source_exhausted = True
         except Exception as e:
             self.state = LoomState.FAILED
             self.hooks.on_error(e)
@@ -146,13 +157,21 @@ class Loom:
             except queue.Full:
                 continue
 
-    def stop(self) -> None:
+    def stop(self, timeout: Optional[float] = None) -> None:
         """
         Signals every component to stop and waits for cleanup.
         Safe to call multiple times or from finally blocks.
 
         Items already queued are processed before shutdown: each Weaver
         drains the queue until it finds its stop sentinel.
+
+        Args:
+            timeout: Maximum time in seconds to wait for the Weavers to
+                finish. None (default) waits indefinitely. When the
+                deadline passes, still-running Weavers are reported via
+                hooks.on_error (as LoomError), the sink is closed anyway
+                and stop() returns; the leftover daemon threads do not
+                block process exit.
         """
         with self._stop_lock:
             if self._stopped:
@@ -161,17 +180,42 @@ class Loom:
 
         self.stop_event.set()
 
+        deadline = None if timeout is None else time.monotonic() + timeout
+
         # One sentinel per Weaver: each thread drains the queue and exits
         # upon consuming its own. This replaces the queue join(), which
         # could block forever if a Weaver died before emptying it.
         for _ in self.weavers:
-            self.task_queue.put(STOP_SENTINEL)
-        for weaver in self.weavers:
-            weaver.join()
+            if not self._put_sentinel(deadline):
+                break
 
-        # Never overwrite a FAILED state set by start() on error
+        stuck = []
+        for weaver in self.weavers:
+            if deadline is None:
+                weaver.join()
+            else:
+                weaver.join(timeout=max(deadline - time.monotonic(), 0))
+            if weaver.is_alive():
+                stuck.append(weaver)
+
+        if stuck:
+            logger.warning(
+                "%d weaver(s) still running after the stop timeout; they are "
+                "daemon threads and will not block process exit.",
+                len(stuck),
+            )
+            try:
+                self.hooks.on_error(
+                    LoomError(f"{len(stuck)} weaver(s) did not finish within the stop timeout.")
+                )
+            except Exception:
+                logger.exception("The on_error callback raised an exception.")
+
+        # Natural exhaustion of the source becomes COMPLETED; an external
+        # stop or interruption becomes STOPPED. A FAILED state set by
+        # start() on a source error is never overwritten.
         if self.state is LoomState.RUNNING:
-            self.state = LoomState.COMPLETED
+            self.state = LoomState.COMPLETED if self._source_exhausted else LoomState.STOPPED
 
         try:
             self.sink.close()
@@ -180,3 +224,23 @@ class Loom:
             self.hooks.on_error(e)
 
         self.hooks.on_stop()
+
+    def _put_sentinel(self, deadline: Optional[float]) -> bool:
+        """
+        Enqueues one stop sentinel, giving up when the deadline passes or
+        when no Weaver is alive to drain a full queue — a plain blocking
+        put() would hang stop() forever in that scenario.
+        """
+        while True:
+            if deadline is None:
+                wait = 0.1
+            else:
+                wait = min(0.1, max(deadline - time.monotonic(), 0.0))
+            try:
+                self.task_queue.put(STOP_SENTINEL, timeout=wait)
+                return True
+            except queue.Full:
+                if deadline is not None and time.monotonic() >= deadline:
+                    return False
+                if not any(weaver.is_alive() for weaver in self.weavers):
+                    return False

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/dataloom_engine/sinks.py

@@ -12,7 +12,7 @@ import queue
 import threading
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict, Optional, TextIO
 
 from dataloom_engine.exceptions import LoomError
 
@@ -42,6 +42,11 @@ class JsonFileSink(Sink):
     """
     Default sink that appends results to a local JSON-lines file.
     Uses a threading.Lock to keep concurrent writes consistent.
+
+    The file is opened lazily on the first send() and kept open until
+    close(), avoiding an open/close syscall pair per result. Every line
+    is flushed, so results are visible to readers immediately. A send()
+    after close() transparently reopens the file in append mode.
     """
 
     def __init__(self, output_dir: Path, filename: str = "results.json"):
@@ -50,12 +55,21 @@ class JsonFileSink(Sink):
         self._path = self.output_dir / filename
         # The lock ensures only one Weaver writes to the file at a time
         self._lock = threading.Lock()
+        self._file: Optional[TextIO] = None
 
     def send(self, result: Dict[str, Any]) -> None:
         with self._lock:
-            with open(self._path, "a") as f:
-                json.dump(result, f)
-                f.write("\n")
+            if self._file is None:
+                self._file = open(self._path, "a")
+            json.dump(result, self._file)
+            self._file.write("\n")
+            self._file.flush()
+
+    def close(self) -> None:
+        with self._lock:
+            if self._file is not None:
+                self._file.close()
+                self._file = None
 
 
 class CsvFileSink(Sink):
@@ -66,6 +80,12 @@ class CsvFileSink(Sink):
     In subsequent results, extra keys are ignored and missing keys are
     left empty. Uses a threading.Lock to keep concurrent writes
     consistent.
+
+    The file is opened lazily on the first send() and kept open until
+    close(), avoiding an open/close syscall pair per row. Every row is
+    flushed, so results are visible to readers immediately. A send()
+    after close() transparently reopens the file in append mode (the
+    header is not repeated).
     """
 
     def __init__(self, output_dir: Path, filename: str = "results.csv"):
@@ -74,6 +94,8 @@ class CsvFileSink(Sink):
         self._path = self.output_dir / filename
         self._lock = threading.Lock()
         self._fieldnames: Optional[list] = None
+        self._file: Optional[TextIO] = None
+        self._writer: Optional[csv.DictWriter] = None
 
     def send(self, result: Dict[str, Any]) -> None:
         with self._lock:
@@ -82,11 +104,22 @@ class CsvFileSink(Sink):
             if fieldnames is None:
                 fieldnames = list(result.keys())
                 self._fieldnames = fieldnames
-            with open(self._path, "a", newline="") as f:
-                writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
-                if write_header:
-                    writer.writeheader()
-                writer.writerow(result)
+            writer = self._writer
+            if self._file is None or writer is None:
+                self._file = open(self._path, "a", newline="")
+                writer = csv.DictWriter(self._file, fieldnames=fieldnames, extrasaction="ignore")
+                self._writer = writer
+            if write_header:
+                writer.writeheader()
+            writer.writerow(result)
+            self._file.flush()
+
+    def close(self) -> None:
+        with self._lock:
+            if self._file is not None:
+                self._file.close()
+                self._file = None
+                self._writer = None
 
 
 class CallbackSink(Sink):

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/dataloom_engine/types.py

@@ -14,5 +14,6 @@ class LoomState(Enum):
 
     PENDING = "pending"
     RUNNING = "running"
-    COMPLETED = "completed"
-    FAILED = "failed"
+    COMPLETED = "completed"  # the source was exhausted naturally
+    STOPPED = "stopped"  # interrupted (external stop() or KeyboardInterrupt) before exhaustion
+    FAILED = "failed"  # the source raised an error

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/dataloom_engine/weaver.py

@@ -61,15 +61,19 @@ class Weaver(threading.Thread):
                 self.task_queue.task_done()
 
     def _process_batch(self, batch: Any) -> None:
+        stage = "process"
         try:
             started = time.monotonic()
             result = self.processor.process(batch)
+            stage = "send"
             self.sink.send(result)
             duration = time.monotonic() - started
         except Exception as exc:
-            logger.exception("Weaver failed to process a batch; the batch was dropped.")
+            logger.exception("Weaver failed to %s a batch; the batch was dropped.", stage)
             if self.on_error is not None:
-                error = WeaverError(f"Failed to process batch: {exc}")
+                # The failed batch and the stage travel with the error so
+                # hooks can retry or quarantine without parsing strings
+                error = WeaverError(f"Failed to {stage} batch: {exc}", batch=batch, stage=stage)
                 error.__cause__ = exc
                 try:
                     self.on_error(error)

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/dataloom_engine.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.4.1
+Version: 0.6.0
 Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataloom-engine"
-version = "0.4.1"
+version = "0.6.0"
 description = "DataLoom: a lightweight and efficient thread orchestration engine for data pipelines."
 readme = "README.md"
 requires-python = ">=3.9"

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/tests/test_config.py

@@ -19,6 +19,18 @@ def test_config_coerces_output_dir_to_path():
     assert isinstance(config.output_dir, Path)
 
 
+def test_config_output_dir_is_optional():
+    """Pipelines without file-based sinks don't need to invent a directory."""
+    config = LoomConfig()
+    assert config.output_dir is None
+    assert config.batch_size == 10  # remaining defaults still apply
+
+
+def test_config_validation_still_runs_without_output_dir():
+    with pytest.raises(ConfigurationError):
+        LoomConfig(batch_size=0)
+
+
 def test_config_accepts_float_interval():
     config = LoomConfig(output_dir=".", interval_seconds=0.25)
     assert config.interval_seconds == 0.25

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/tests/test_core.py

@@ -181,7 +181,44 @@ def test_weaver_handles_processor_error():
     # The valid items were processed despite the error in the middle
     assert sorted(r["sum"] for r in mock_sink.results) == [1.0, 3.0]
 
-    # The error was typed and reported
+    # The error was typed and reported, with full failure context
     assert len(errors) == 1
     assert isinstance(errors[0], WeaverError)
     assert isinstance(errors[0].__cause__, ValueError)
+    assert errors[0].stage == "process"
+    assert list(errors[0].batch) == [2]  # the exact batch that failed
+
+
+def test_weaver_reports_sink_failure_with_send_stage():
+    """A Sink failure is reported as WeaverError with stage='send' and the original batch."""
+
+    class BrokenSink(Sink):
+        def send(self, result):
+            raise IOError("disk full!")
+
+    task_queue = queue.Queue()
+    errors = []
+    errors_lock = threading.Lock()
+
+    def on_error(exc):
+        with errors_lock:
+            errors.append(exc)
+
+    task_queue.put(np.array([7]))
+    task_queue.put(STOP_SENTINEL)
+
+    weaver = Weaver(
+        task_queue=task_queue,
+        processor=SimpleProcessor(),
+        sink=BrokenSink(),
+        on_error=on_error,
+    )
+    weaver.start()
+    weaver.join(timeout=2)
+    assert not weaver.is_alive()
+
+    assert len(errors) == 1
+    assert isinstance(errors[0], WeaverError)
+    assert errors[0].stage == "send"
+    assert list(errors[0].batch) == [7]
+    assert isinstance(errors[0].__cause__, IOError)

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/tests/test_loom.py

@@ -1,6 +1,7 @@
 # tests/test_loom.py
 
 import threading
+import time
 from typing import Any, Iterator
 
 import numpy as np
@@ -275,6 +276,105 @@ def test_loom_survives_broken_metrics_hook():
     assert loom.state is LoomState.COMPLETED
 
 
+def test_loom_external_stop_sets_stopped_state():
+    """stop() before the source is exhausted must end as STOPPED, not COMPLETED."""
+
+    class InfiniteSource(Source):
+        def __iter__(self) -> Iterator[Any]:
+            i = 0
+            while True:
+                yield np.array([i])
+                i += 1
+
+    hooks = RecordingHooks()
+    sink = InMemorySink()
+    loom = _make_loom(InfiniteSource(), hooks=hooks, sink=sink)
+
+    runner = threading.Thread(target=loom.start, daemon=True)
+    runner.start()
+    # Wait until the pipeline is demonstrably flowing before stopping it
+    for _ in range(500):
+        with sink._lock:
+            if sink.results:
+                break
+        time.sleep(0.01)
+    assert sink.results, "pipeline never produced a result"
+
+    loom.stop()
+    runner.join(timeout=5)
+
+    assert not runner.is_alive(), "start() did not return after stop()"
+    assert loom.state is LoomState.STOPPED
+    assert hooks.stopped
+
+
+def test_loom_keyboard_interrupt_sets_stopped_state():
+    """Ctrl+C is an interruption: the state must be STOPPED, not COMPLETED or FAILED."""
+
+    class InterruptingSource(Source):
+        def __iter__(self) -> Iterator[Any]:
+            yield np.array([1])
+            raise KeyboardInterrupt
+
+    hooks = RecordingHooks()
+    loom = _make_loom(InterruptingSource(), hooks=hooks)
+
+    with pytest.raises(KeyboardInterrupt):
+        loom.start()
+
+    assert loom.state is LoomState.STOPPED
+    assert hooks.stopped  # cleanup still ran
+    assert hooks.errors == []  # an interruption is not an error
+
+
+def test_loom_stop_timeout_reports_stuck_weaver():
+    """stop(timeout=...) returns even with a hung Processor and reports it via on_error."""
+    entered = threading.Event()
+    release = threading.Event()
+
+    class BlockingProcessor(Processor):
+        def process(self, batch):
+            entered.set()
+            release.wait(timeout=10)
+            return {"data": batch[0]}
+
+    class GatedSource(Source):
+        """Yields one batch, then keeps the producer inside the loop until released."""
+
+        def __iter__(self) -> Iterator[Any]:
+            yield np.array([1])
+            release.wait(timeout=10)
+
+    hooks = RecordingHooks()
+    config = LoomConfig(output_dir=".", batch_size=1, interval_seconds=0)
+    loom = Loom(
+        config=config,
+        processor=BlockingProcessor(),
+        sink=InMemorySink(),
+        source=GatedSource(),
+        hooks=hooks,
+        num_weavers=1,
+    )
+
+    runner = threading.Thread(target=loom.start, daemon=True)
+    runner.start()
+    assert entered.wait(timeout=5), "the weaver never picked up the batch"
+
+    started = time.monotonic()
+    loom.stop(timeout=0.2)
+    elapsed = time.monotonic() - started
+
+    assert elapsed < 5, "stop(timeout) blocked far beyond its deadline"
+    assert loom.state is LoomState.STOPPED
+    assert any("stop timeout" in str(e) for e in hooks.errors)
+    assert hooks.stopped  # the sink was closed and on_stop fired despite the stuck weaver
+
+    # Cleanup: unblock everything so the test leaves no lingering threads
+    release.set()
+    runner.join(timeout=5)
+    assert not runner.is_alive()
+
+
 @pytest.mark.parametrize("num_weavers", [0, -1])
 def test_loom_rejects_non_positive_num_weavers(num_weavers):
     """

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/tests/test_optional.py

@@ -68,7 +68,8 @@ def test_pipeline_runs_without_numpy():
             self.results.append(result)
 
     sink = CollectSink()
-    config = LoomConfig(output_dir=".", batch_size=1, interval_seconds=0)
+    # No file sink involved: output_dir can simply be omitted
+    config = LoomConfig(batch_size=1, interval_seconds=0)
     with Loom(config=config, processor=SumProcessor(), sink=sink, source=ListSource()) as loom:
         loom.start()
 

{dataloom_engine-0.4.1 → dataloom_engine-0.6.0}/tests/test_sinks.py

@@ -1,12 +1,19 @@
 # tests/test_sinks.py
 
 import csv
+import json
 import threading
 
 import pytest
 
 from dataloom_engine.exceptions import LoomError
-from dataloom_engine.sinks import CallbackSink, CsvFileSink, Sink, ThreadedBufferedSink
+from dataloom_engine.sinks import (
+    CallbackSink,
+    CsvFileSink,
+    JsonFileSink,
+    Sink,
+    ThreadedBufferedSink,
+)
 
 
 class MockSink(Sink):
@@ -148,6 +155,51 @@ def test_csv_sink_concurrent_writes(tmp_path):
     assert ids == list(range(100))
 
 
+def test_file_sinks_flush_results_before_close(tmp_path):
+    """With persistent handles, every result must still be readable immediately."""
+    json_sink = JsonFileSink(output_dir=tmp_path)
+    json_sink.send({"id": 1})
+    # No close() yet: the line must already be on disk
+    assert json.loads((tmp_path / "results.json").read_text()) == {"id": 1}
+
+    csv_sink = CsvFileSink(output_dir=tmp_path)
+    csv_sink.send({"id": 1})
+    with open(tmp_path / "results.csv", newline="") as f:
+        assert list(csv.DictReader(f)) == [{"id": "1"}]
+
+    json_sink.close()
+    csv_sink.close()
+
+
+def test_file_sinks_reopen_after_close(tmp_path):
+    """send() after close() appends transparently; the CSV header is not repeated."""
+    sink = CsvFileSink(output_dir=tmp_path)
+    sink.send({"id": 1})
+    sink.close()
+    sink.send({"id": 2})
+    sink.close()
+
+    lines = (tmp_path / "results.csv").read_text().strip().splitlines()
+    assert lines == ["id", "1", "2"]  # a single header, both rows
+
+    json_sink = JsonFileSink(output_dir=tmp_path)
+    json_sink.send({"id": 1})
+    json_sink.close()
+    json_sink.send({"id": 2})
+    json_sink.close()
+
+    rows = [json.loads(line) for line in (tmp_path / "results.json").read_text().splitlines()]
+    assert rows == [{"id": 1}, {"id": 2}]
+
+
+def test_file_sink_close_before_any_send_is_noop(tmp_path):
+    """close() without any send() must not fail nor create files."""
+    JsonFileSink(output_dir=tmp_path, filename="a.json").close()
+    CsvFileSink(output_dir=tmp_path, filename="a.csv").close()
+    assert not (tmp_path / "a.json").exists()
+    assert not (tmp_path / "a.csv").exists()
+
+
 def test_callback_sink_delegates_send_and_close():
     received = []
     closed = []

dataloom_engine-0.4.1/dataloom_engine/exceptions.py

@@ -1,25 +0,0 @@
-# dataloom_engine/exceptions.py
-
-"""
-Custom exceptions for DataLoom.
-Lets consumers catch library-specific errors without relying on
-generic Python exceptions.
-"""
-
-
-class LoomError(Exception):
-    """Base exception for every DataLoom error."""
-
-    pass
-
-
-class ConfigurationError(LoomError):
-    """Raised when LoomConfig validation fails."""
-
-    pass
-
-
-class WeaverError(LoomError):
-    """Raised when a Weaver fails to process a batch."""
-
-    pass