dataloom-engine 0.4.1__tar.gz → 0.5.0__tar.gz

{dataloom_engine-0.4.1 → dataloom_engine-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.4.1
+Version: 0.5.0
 Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT

{dataloom_engine-0.4.1 → dataloom_engine-0.5.0}/dataloom_engine/loom.py

@@ -6,8 +6,10 @@ Defines the main class responsible for managing the lifecycle of the
 worker threads (Weavers) and the distribution of tasks.
 """
 
+import logging
 import queue
 import threading
+import time
 from typing import TYPE_CHECKING, Optional
 
 from dataloom_engine.config import LoomConfig
@@ -21,6 +23,8 @@ from dataloom_engine.weaver import STOP_SENTINEL, Weaver
 if TYPE_CHECKING:
     from dataloom_engine.sources import Source
 
+logger = logging.getLogger(__name__)
+
 
 class Loom:
     """
@@ -80,6 +84,8 @@ class Loom:
         self.weavers: list[Weaver] = []
         self._stop_lock = threading.Lock()
         self._stopped = False
+        # Distinguishes COMPLETED (source exhausted) from STOPPED (interrupted)
+        self._source_exhausted = False
 
     def __enter__(self) -> "Loom":
         return self
@@ -127,6 +133,11 @@ class Loom:
                 if self.stop_event.is_set():
                     break
                 self._enqueue(batch)
+            else:
+                # The for/else only runs when the loop ended without a
+                # break: the source was exhausted naturally, so stop()
+                # may report COMPLETED instead of STOPPED.
+                self._source_exhausted = True
         except Exception as e:
             self.state = LoomState.FAILED
             self.hooks.on_error(e)
@@ -146,13 +157,21 @@ class Loom:
             except queue.Full:
                 continue
 
-    def stop(self) -> None:
+    def stop(self, timeout: Optional[float] = None) -> None:
         """
         Signals every component to stop and waits for cleanup.
         Safe to call multiple times or from finally blocks.
 
         Items already queued are processed before shutdown: each Weaver
         drains the queue until it finds its stop sentinel.
+
+        Args:
+            timeout: Maximum time in seconds to wait for the Weavers to
+                finish. None (default) waits indefinitely. When the
+                deadline passes, still-running Weavers are reported via
+                hooks.on_error (as LoomError), the sink is closed anyway
+                and stop() returns; the leftover daemon threads do not
+                block process exit.
         """
         with self._stop_lock:
             if self._stopped:
@@ -161,17 +180,42 @@ class Loom:
 
         self.stop_event.set()
 
+        deadline = None if timeout is None else time.monotonic() + timeout
+
         # One sentinel per Weaver: each thread drains the queue and exits
         # upon consuming its own. This replaces the queue join(), which
         # could block forever if a Weaver died before emptying it.
         for _ in self.weavers:
-            self.task_queue.put(STOP_SENTINEL)
-        for weaver in self.weavers:
-            weaver.join()
+            if not self._put_sentinel(deadline):
+                break
 
-        # Never overwrite a FAILED state set by start() on error
+        stuck = []
+        for weaver in self.weavers:
+            if deadline is None:
+                weaver.join()
+            else:
+                weaver.join(timeout=max(deadline - time.monotonic(), 0))
+            if weaver.is_alive():
+                stuck.append(weaver)
+
+        if stuck:
+            logger.warning(
+                "%d weaver(s) still running after the stop timeout; they are "
+                "daemon threads and will not block process exit.",
+                len(stuck),
+            )
+            try:
+                self.hooks.on_error(
+                    LoomError(f"{len(stuck)} weaver(s) did not finish within the stop timeout.")
+                )
+            except Exception:
+                logger.exception("The on_error callback raised an exception.")
+
+        # Natural exhaustion of the source becomes COMPLETED; an external
+        # stop or interruption becomes STOPPED. A FAILED state set by
+        # start() on a source error is never overwritten.
         if self.state is LoomState.RUNNING:
-            self.state = LoomState.COMPLETED
+            self.state = LoomState.COMPLETED if self._source_exhausted else LoomState.STOPPED
 
         try:
             self.sink.close()
@@ -180,3 +224,23 @@ class Loom:
             self.hooks.on_error(e)
 
         self.hooks.on_stop()
+
+    def _put_sentinel(self, deadline: Optional[float]) -> bool:
+        """
+        Enqueues one stop sentinel, giving up when the deadline passes or
+        when no Weaver is alive to drain a full queue — a plain blocking
+        put() would hang stop() forever in that scenario.
+        """
+        while True:
+            if deadline is None:
+                wait = 0.1
+            else:
+                wait = min(0.1, max(deadline - time.monotonic(), 0.0))
+            try:
+                self.task_queue.put(STOP_SENTINEL, timeout=wait)
+                return True
+            except queue.Full:
+                if deadline is not None and time.monotonic() >= deadline:
+                    return False
+                if not any(weaver.is_alive() for weaver in self.weavers):
+                    return False

{dataloom_engine-0.4.1 → dataloom_engine-0.5.0}/dataloom_engine/types.py

@@ -14,5 +14,6 @@ class LoomState(Enum):
 
     PENDING = "pending"
     RUNNING = "running"
-    COMPLETED = "completed"
-    FAILED = "failed"
+    COMPLETED = "completed"  # the source was exhausted naturally
+    STOPPED = "stopped"  # interrupted (external stop() or KeyboardInterrupt) before exhaustion
+    FAILED = "failed"  # the source raised an error

{dataloom_engine-0.4.1 → dataloom_engine-0.5.0}/dataloom_engine.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.4.1
+Version: 0.5.0
 Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT

{dataloom_engine-0.4.1 → dataloom_engine-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataloom-engine"
-version = "0.4.1"
+version = "0.5.0"
 description = "DataLoom: a lightweight and efficient thread orchestration engine for data pipelines."
 readme = "README.md"
 requires-python = ">=3.9"

{dataloom_engine-0.4.1 → dataloom_engine-0.5.0}/tests/test_loom.py

@@ -1,6 +1,7 @@
 # tests/test_loom.py
 
 import threading
+import time
 from typing import Any, Iterator
 
 import numpy as np
@@ -275,6 +276,105 @@ def test_loom_survives_broken_metrics_hook():
     assert loom.state is LoomState.COMPLETED
 
 
+def test_loom_external_stop_sets_stopped_state():
+    """stop() before the source is exhausted must end as STOPPED, not COMPLETED."""
+
+    class InfiniteSource(Source):
+        def __iter__(self) -> Iterator[Any]:
+            i = 0
+            while True:
+                yield np.array([i])
+                i += 1
+
+    hooks = RecordingHooks()
+    sink = InMemorySink()
+    loom = _make_loom(InfiniteSource(), hooks=hooks, sink=sink)
+
+    runner = threading.Thread(target=loom.start, daemon=True)
+    runner.start()
+    # Wait until the pipeline is demonstrably flowing before stopping it
+    for _ in range(500):
+        with sink._lock:
+            if sink.results:
+                break
+        time.sleep(0.01)
+    assert sink.results, "pipeline never produced a result"
+
+    loom.stop()
+    runner.join(timeout=5)
+
+    assert not runner.is_alive(), "start() did not return after stop()"
+    assert loom.state is LoomState.STOPPED
+    assert hooks.stopped
+
+
+def test_loom_keyboard_interrupt_sets_stopped_state():
+    """Ctrl+C is an interruption: the state must be STOPPED, not COMPLETED or FAILED."""
+
+    class InterruptingSource(Source):
+        def __iter__(self) -> Iterator[Any]:
+            yield np.array([1])
+            raise KeyboardInterrupt
+
+    hooks = RecordingHooks()
+    loom = _make_loom(InterruptingSource(), hooks=hooks)
+
+    with pytest.raises(KeyboardInterrupt):
+        loom.start()
+
+    assert loom.state is LoomState.STOPPED
+    assert hooks.stopped  # cleanup still ran
+    assert hooks.errors == []  # an interruption is not an error
+
+
+def test_loom_stop_timeout_reports_stuck_weaver():
+    """stop(timeout=...) returns even with a hung Processor and reports it via on_error."""
+    entered = threading.Event()
+    release = threading.Event()
+
+    class BlockingProcessor(Processor):
+        def process(self, batch):
+            entered.set()
+            release.wait(timeout=10)
+            return {"data": batch[0]}
+
+    class GatedSource(Source):
+        """Yields one batch, then keeps the producer inside the loop until released."""
+
+        def __iter__(self) -> Iterator[Any]:
+            yield np.array([1])
+            release.wait(timeout=10)
+
+    hooks = RecordingHooks()
+    config = LoomConfig(output_dir=".", batch_size=1, interval_seconds=0)
+    loom = Loom(
+        config=config,
+        processor=BlockingProcessor(),
+        sink=InMemorySink(),
+        source=GatedSource(),
+        hooks=hooks,
+        num_weavers=1,
+    )
+
+    runner = threading.Thread(target=loom.start, daemon=True)
+    runner.start()
+    assert entered.wait(timeout=5), "the weaver never picked up the batch"
+
+    started = time.monotonic()
+    loom.stop(timeout=0.2)
+    elapsed = time.monotonic() - started
+
+    assert elapsed < 5, "stop(timeout) blocked far beyond its deadline"
+    assert loom.state is LoomState.STOPPED
+    assert any("stop timeout" in str(e) for e in hooks.errors)
+    assert hooks.stopped  # the sink was closed and on_stop fired despite the stuck weaver
+
+    # Cleanup: unblock everything so the test leaves no lingering threads
+    release.set()
+    runner.join(timeout=5)
+    assert not runner.is_alive()
+
+
 @pytest.mark.parametrize("num_weavers", [0, -1])
 def test_loom_rejects_non_positive_num_weavers(num_weavers):
     """