dataloom-engine 0.4.0__tar.gz → 0.5.0__tar.gz

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.4.0
-Summary: DataLoom: Um orquestrador de threads leve e eficiente para dados.
+Version: 0.5.0
+Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT
 Project-URL: Homepage, https://github.com/dionipadilha/dataloom
@@ -175,6 +175,10 @@ if __name__ == "__main__":
         print("\n🛑 Loom stopped.")
 ```
 
+More runnable, real-world examples — API enrichment with parallel speedup,
+a resilient file-processing pipeline, a buffered sensor stream — live in the
+[`examples/`](examples/) directory.
+
 ## 🏗️ Architecture
 
 DataLoom is built around a weaving metaphor:

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/README.md

@@ -141,6 +141,10 @@ if __name__ == "__main__":
         print("\n🛑 Loom stopped.")
 ```
 
+More runnable, real-world examples — API enrichment with parallel speedup,
+a resilient file-processing pipeline, a buffered sensor stream — live in the
+[`examples/`](examples/) directory.
+
 ## 🏗️ Architecture
 
 DataLoom is built around a weaving metaphor:

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/dataloom_engine/__init__.py

@@ -8,6 +8,8 @@ such as 'Weaver' are deliberately not exposed, keeping the usage surface
 clean and safe for consumers.
 """
 
+from importlib.metadata import PackageNotFoundError, version
+
 from dataloom_engine.config import LoomConfig
 from dataloom_engine.exceptions import ConfigurationError, LoomError, WeaverError
 from dataloom_engine.hooks import LoomHooks
@@ -24,7 +26,13 @@ from dataloom_engine.sinks import (
 from dataloom_engine.sources import Source
 from dataloom_engine.types import LoomState
 
+try:
+    __version__ = version("dataloom-engine")
+except PackageNotFoundError:  # pragma: no cover -- running from an uninstalled source tree
+    __version__ = "0+unknown"
+
 __all__ = [
+    "__version__",
     "Loom",
     "LoomConfig",
     "LoomState",

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/dataloom_engine/loom.py

@@ -6,11 +6,14 @@ Defines the main class responsible for managing the lifecycle of the
 worker threads (Weavers) and the distribution of tasks.
 """
 
+import logging
 import queue
 import threading
+import time
 from typing import TYPE_CHECKING, Optional
 
 from dataloom_engine.config import LoomConfig
+from dataloom_engine.exceptions import ConfigurationError, LoomError
 from dataloom_engine.hooks import LoomHooks
 from dataloom_engine.processors import Processor
 from dataloom_engine.sinks import Sink
@@ -20,6 +23,8 @@ from dataloom_engine.weaver import STOP_SENTINEL, Weaver
 if TYPE_CHECKING:
     from dataloom_engine.sources import Source
 
+logger = logging.getLogger(__name__)
+
 
 class Loom:
     """
@@ -45,6 +50,9 @@ class Loom:
         hooks: Optional[LoomHooks] = None,
         num_weavers: int = 2,
     ):
+        if num_weavers < 1:
+            raise ConfigurationError(f"num_weavers must be at least 1 (got: {num_weavers}).")
+
         self.config = config
         self.processor = processor
         self.sink = sink
@@ -76,6 +84,8 @@ class Loom:
         self.weavers: list[Weaver] = []
         self._stop_lock = threading.Lock()
         self._stopped = False
+        # Distinguishes COMPLETED (source exhausted) from STOPPED (interrupted)
+        self._source_exhausted = False
 
     def __enter__(self) -> "Loom":
         return self
@@ -90,7 +100,18 @@ class Loom:
         """
         Starts the Weavers and begins the task production loop.
         This method blocks until an error occurs or the loom is stopped.
+
+        A Loom instance is single-use: calling start() again after it has
+        run (or after stop()) raises LoomError.
         """
+        # Restarting a stopped instance would spawn Weavers that never
+        # receive a stop sentinel (stop() is idempotent), leaking threads.
+        if self.state is not LoomState.PENDING or self._stopped:
+            raise LoomError(
+                "This Loom has already been started or stopped; "
+                "create a new instance to run another pipeline."
+            )
+
         self.state = LoomState.RUNNING
         self.hooks.on_start()
 
@@ -112,6 +133,11 @@ class Loom:
                 if self.stop_event.is_set():
                     break
                 self._enqueue(batch)
+            else:
+                # The for/else only runs when the loop ended without a
+                # break: the source was exhausted naturally, so stop()
+                # may report COMPLETED instead of STOPPED.
+                self._source_exhausted = True
         except Exception as e:
             self.state = LoomState.FAILED
             self.hooks.on_error(e)
@@ -131,13 +157,21 @@ class Loom:
             except queue.Full:
                 continue
 
-    def stop(self) -> None:
+    def stop(self, timeout: Optional[float] = None) -> None:
         """
         Signals every component to stop and waits for cleanup.
         Safe to call multiple times or from finally blocks.
 
         Items already queued are processed before shutdown: each Weaver
         drains the queue until it finds its stop sentinel.
+
+        Args:
+            timeout: Maximum time in seconds to wait for the Weavers to
+                finish. None (default) waits indefinitely. When the
+                deadline passes, still-running Weavers are reported via
+                hooks.on_error (as LoomError), the sink is closed anyway
+                and stop() returns; the leftover daemon threads do not
+                block process exit.
         """
         with self._stop_lock:
             if self._stopped:
@@ -146,17 +180,42 @@ class Loom:
 
         self.stop_event.set()
 
+        deadline = None if timeout is None else time.monotonic() + timeout
+
         # One sentinel per Weaver: each thread drains the queue and exits
         # upon consuming its own. This replaces the queue join(), which
         # could block forever if a Weaver died before emptying it.
         for _ in self.weavers:
-            self.task_queue.put(STOP_SENTINEL)
-        for weaver in self.weavers:
-            weaver.join()
+            if not self._put_sentinel(deadline):
+                break
 
-        # Never overwrite a FAILED state set by start() on error
+        stuck = []
+        for weaver in self.weavers:
+            if deadline is None:
+                weaver.join()
+            else:
+                weaver.join(timeout=max(deadline - time.monotonic(), 0))
+            if weaver.is_alive():
+                stuck.append(weaver)
+
+        if stuck:
+            logger.warning(
+                "%d weaver(s) still running after the stop timeout; they are "
+                "daemon threads and will not block process exit.",
+                len(stuck),
+            )
+            try:
+                self.hooks.on_error(
+                    LoomError(f"{len(stuck)} weaver(s) did not finish within the stop timeout.")
+                )
+            except Exception:
+                logger.exception("The on_error callback raised an exception.")
+
+        # Natural exhaustion of the source becomes COMPLETED; an external
+        # stop or interruption becomes STOPPED. A FAILED state set by
+        # start() on a source error is never overwritten.
         if self.state is LoomState.RUNNING:
-            self.state = LoomState.COMPLETED
+            self.state = LoomState.COMPLETED if self._source_exhausted else LoomState.STOPPED
 
         try:
             self.sink.close()
@@ -165,3 +224,23 @@ class Loom:
             self.hooks.on_error(e)
 
         self.hooks.on_stop()
+
+    def _put_sentinel(self, deadline: Optional[float]) -> bool:
+        """
+        Enqueues one stop sentinel, giving up when the deadline passes or
+        when no Weaver is alive to drain a full queue — a plain blocking
+        put() would hang stop() forever in that scenario.
+        """
+        while True:
+            if deadline is None:
+                wait = 0.1
+            else:
+                wait = min(0.1, max(deadline - time.monotonic(), 0.0))
+            try:
+                self.task_queue.put(STOP_SENTINEL, timeout=wait)
+                return True
+            except queue.Full:
+                if deadline is not None and time.monotonic() >= deadline:
+                    return False
+                if not any(weaver.is_alive() for weaver in self.weavers):
+                    return False

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/dataloom_engine/sinks.py

@@ -44,17 +44,16 @@ class JsonFileSink(Sink):
     Uses a threading.Lock to keep concurrent writes consistent.
     """
 
-    def __init__(self, output_dir: Path):
-        self.output_dir = output_dir
+    def __init__(self, output_dir: Path, filename: str = "results.json"):
+        self.output_dir = Path(output_dir)
         self.output_dir.mkdir(parents=True, exist_ok=True)
+        self._path = self.output_dir / filename
         # The lock ensures only one Weaver writes to the file at a time
         self._lock = threading.Lock()
 
     def send(self, result: Dict[str, Any]) -> None:
-        filename = self.output_dir / "results.json"
-
         with self._lock:
-            with open(filename, "a") as f:
+            with open(self._path, "a") as f:
                 json.dump(result, f)
                 f.write("\n")
 
@@ -138,9 +137,14 @@ class ThreadedBufferedSink(Sink):
         self.worker_thread.start()
 
     def send(self, result: Dict[str, Any]) -> None:
-        if self._closed:
-            raise LoomError("ThreadedBufferedSink is already closed; send() is not allowed.")
-        self.queue.put(result)
+        # The closed check and the put must be atomic with respect to
+        # close(): otherwise a send() racing with close() could pass the
+        # check, lose the CPU while close() drains the buffer, and then
+        # enqueue the item behind the stop sentinel — silently lost.
+        with self._close_lock:
+            if self._closed:
+                raise LoomError("ThreadedBufferedSink is already closed; send() is not allowed.")
+            self.queue.put(result)
 
     def _worker(self) -> None:
         while True:
@@ -157,15 +161,16 @@ class ThreadedBufferedSink(Sink):
                 self.queue.task_done()
 
     def close(self) -> None:
-        # Idempotent: only the first call performs the shutdown
+        # Idempotent: only the first call performs the shutdown. The
+        # sentinel is enqueued under the same lock as send(), so it is
+        # guaranteed to land behind every accepted item — the worker
+        # drains all of them before exiting.
         with self._close_lock:
             if self._closed:
                 return
             self._closed = True
+            self.queue.put(self._STOP)
 
-        # The sentinel goes in behind any pending items: the worker
-        # drains everything before exiting
-        self.queue.put(self._STOP)
         self.worker_thread.join()
 
         # Propagate the close

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/dataloom_engine/types.py

@@ -14,5 +14,6 @@ class LoomState(Enum):
 
     PENDING = "pending"
     RUNNING = "running"
-    COMPLETED = "completed"
-    FAILED = "failed"
+    COMPLETED = "completed"  # the source was exhausted naturally
+    STOPPED = "stopped"  # interrupted (external stop() or KeyboardInterrupt) before exhaustion
+    FAILED = "failed"  # the source raised an error

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/dataloom_engine.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.4.0
-Summary: DataLoom: Um orquestrador de threads leve e eficiente para dados.
+Version: 0.5.0
+Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT
 Project-URL: Homepage, https://github.com/dionipadilha/dataloom
@@ -175,6 +175,10 @@ if __name__ == "__main__":
         print("\n🛑 Loom stopped.")
 ```
 
+More runnable, real-world examples — API enrichment with parallel speedup,
+a resilient file-processing pipeline, a buffered sensor stream — live in the
+[`examples/`](examples/) directory.
+
 ## 🏗️ Architecture
 
 DataLoom is built around a weaving metaphor:

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataloom-engine"
-version = "0.4.0"
-description = "DataLoom: Um orquestrador de threads leve e eficiente para dados."
+version = "0.5.0"
+description = "DataLoom: a lightweight and efficient thread orchestration engine for data pipelines."
 readme = "README.md"
 requires-python = ">=3.9"
 license = {text = "MIT"}

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/tests/test_core.py

@@ -11,11 +11,12 @@ import threading
 
 import numpy as np
 
-from dataloom_engine import JsonFileSink, Processor, Sink
+from dataloom_engine import JsonFileSink, LoomConfig, Processor, Sink
 
 # Importing internal classes explicitly for testing
 from dataloom_engine.exceptions import WeaverError
 from dataloom_engine.processors import StatisticsProcessor
+from dataloom_engine.sources import RandomNumPySource
 from dataloom_engine.weaver import STOP_SENTINEL, Weaver
 
 # --- Mocks and helpers ---
@@ -76,6 +77,30 @@ def test_json_sink_writes_file(tmp_path):
     assert loaded_json == data
 
 
+def test_json_sink_accepts_custom_filename(tmp_path):
+    """The output filename is configurable (parity with CsvFileSink)."""
+    sink = JsonFileSink(output_dir=tmp_path, filename="custom.jsonl")
+    sink.send({"id": 7})
+
+    assert (tmp_path / "custom.jsonl").exists()
+    assert not (tmp_path / "results.json").exists()
+    assert json.loads((tmp_path / "custom.jsonl").read_text()) == {"id": 7}
+
+
+def test_random_numpy_source_respects_limit_and_batch_size():
+    """The demo source yields exactly `limit` batches of `batch_size` values in [0, 1)."""
+    config = LoomConfig(output_dir=".", batch_size=5, interval_seconds=0)
+    source = RandomNumPySource(config, limit=3)
+
+    batches = list(source)
+
+    assert len(batches) == 3
+    for batch in batches:
+        assert len(batch) == 5
+        assert float(batch.min()) >= 0.0
+        assert float(batch.max()) < 1.0
+
+
 # --- Integration tests (Weaver/flow) ---
 
 

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/tests/test_loom.py

@@ -1,13 +1,14 @@
 # tests/test_loom.py
 
 import threading
+import time
 from typing import Any, Iterator
 
 import numpy as np
 import pytest
 
 from dataloom_engine import Loom, LoomConfig, LoomHooks, LoomState, Processor, Sink
-from dataloom_engine.exceptions import WeaverError
+from dataloom_engine.exceptions import ConfigurationError, LoomError, WeaverError
 from dataloom_engine.sources import Source
 
 # --- Mocks ---
@@ -61,11 +62,8 @@ def test_loom_uses_custom_source():
 
     loom = Loom(config=config, processor=processor, sink=sink, source=source, num_weavers=2)
 
-    # Run Loom (start blocks until source is exhausted or error)
-    # But wait, Loom.start() blocks until source is exhausted AND then calls stop().
-    # Since our source is finite, it should finish naturally.
-    if hasattr(loom, "start"):
-        loom.start()
+    # start() blocks until the finite source is exhausted, then stops itself
+    loom.start()
 
     # Verify results
     assert len(sink.results) == 3
@@ -278,6 +276,161 @@ def test_loom_survives_broken_metrics_hook():
     assert loom.state is LoomState.COMPLETED
 
 
+def test_loom_external_stop_sets_stopped_state():
+    """stop() before the source is exhausted must end as STOPPED, not COMPLETED."""
+
+    class InfiniteSource(Source):
+        def __iter__(self) -> Iterator[Any]:
+            i = 0
+            while True:
+                yield np.array([i])
+                i += 1
+
+    hooks = RecordingHooks()
+    sink = InMemorySink()
+    loom = _make_loom(InfiniteSource(), hooks=hooks, sink=sink)
+
+    runner = threading.Thread(target=loom.start, daemon=True)
+    runner.start()
+    # Wait until the pipeline is demonstrably flowing before stopping it
+    for _ in range(500):
+        with sink._lock:
+            if sink.results:
+                break
+        time.sleep(0.01)
+    assert sink.results, "pipeline never produced a result"
+
+    loom.stop()
+    runner.join(timeout=5)
+
+    assert not runner.is_alive(), "start() did not return after stop()"
+    assert loom.state is LoomState.STOPPED
+    assert hooks.stopped
+
+
+def test_loom_keyboard_interrupt_sets_stopped_state():
+    """Ctrl+C is an interruption: the state must be STOPPED, not COMPLETED or FAILED."""
+
+    class InterruptingSource(Source):
+        def __iter__(self) -> Iterator[Any]:
+            yield np.array([1])
+            raise KeyboardInterrupt
+
+    hooks = RecordingHooks()
+    loom = _make_loom(InterruptingSource(), hooks=hooks)
+
+    with pytest.raises(KeyboardInterrupt):
+        loom.start()
+
+    assert loom.state is LoomState.STOPPED
+    assert hooks.stopped  # cleanup still ran
+    assert hooks.errors == []  # an interruption is not an error
+
+
+def test_loom_stop_timeout_reports_stuck_weaver():
+    """stop(timeout=...) returns even with a hung Processor and reports it via on_error."""
+    entered = threading.Event()
+    release = threading.Event()
+
+    class BlockingProcessor(Processor):
+        def process(self, batch):
+            entered.set()
+            release.wait(timeout=10)
+            return {"data": batch[0]}
+
+    class GatedSource(Source):
+        """Yields one batch, then keeps the producer inside the loop until released."""
+
+        def __iter__(self) -> Iterator[Any]:
+            yield np.array([1])
+            release.wait(timeout=10)
+
+    hooks = RecordingHooks()
+    config = LoomConfig(output_dir=".", batch_size=1, interval_seconds=0)
+    loom = Loom(
+        config=config,
+        processor=BlockingProcessor(),
+        sink=InMemorySink(),
+        source=GatedSource(),
+        hooks=hooks,
+        num_weavers=1,
+    )
+
+    runner = threading.Thread(target=loom.start, daemon=True)
+    runner.start()
+    assert entered.wait(timeout=5), "the weaver never picked up the batch"
+
+    started = time.monotonic()
+    loom.stop(timeout=0.2)
+    elapsed = time.monotonic() - started
+
+    assert elapsed < 5, "stop(timeout) blocked far beyond its deadline"
+    assert loom.state is LoomState.STOPPED
+    assert any("stop timeout" in str(e) for e in hooks.errors)
+    assert hooks.stopped  # the sink was closed and on_stop fired despite the stuck weaver
+
+    # Cleanup: unblock everything so the test leaves no lingering threads
+    release.set()
+    runner.join(timeout=5)
+    assert not runner.is_alive()
+
+
+@pytest.mark.parametrize("num_weavers", [0, -1])
+def test_loom_rejects_non_positive_num_weavers(num_weavers):
+    """
+    num_weavers < 1 must fail fast: 0 weavers would silently create an
+    unbounded queue (0 * 4 = 0 = no limit) and complete without
+    processing anything.
+    """
+    config = LoomConfig(output_dir=".", batch_size=1, interval_seconds=0)
+    with pytest.raises(ConfigurationError):
+        Loom(
+            config=config,
+            processor=PassthroughProcessor(),
+            sink=InMemorySink(),
+            source=FiniteSource([1]),
+            num_weavers=num_weavers,
+        )
+
+
+def test_loom_start_cannot_be_reused():
+    """
+    A Loom instance is single-use: a second start() must raise instead of
+    spawning Weavers that never receive a stop sentinel (thread leak with
+    the state stuck in RUNNING).
+    """
+
+    class CountingHooks(LoomHooks):
+        def __init__(self):
+            self.start_calls = 0
+
+        def on_start(self):
+            self.start_calls += 1
+
+    hooks = CountingHooks()
+    loom = _make_loom(FiniteSource([1]), hooks=hooks)
+    loom.start()
+    weavers_after_first = len(loom.weavers)
+
+    with pytest.raises(LoomError):
+        loom.start()
+
+    assert len(loom.weavers) == weavers_after_first  # no leaked threads
+    assert loom.state is LoomState.COMPLETED  # not stuck in RUNNING
+    assert hooks.start_calls == 1  # the rejected start never fired hooks
+
+
+def test_loom_start_after_early_stop_raises():
+    """stop() before start() must also make start() unusable (same leak scenario)."""
+    loom = _make_loom(FiniteSource([1]))
+    loom.stop()
+
+    with pytest.raises(LoomError):
+        loom.start()
+
+    assert loom.weavers == []
+
+
 def test_loom_as_context_manager():
     """The with block yields the Loom itself and guarantees stop() on exit."""
     hooks = RecordingHooks()

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/tests/test_optional.py

@@ -29,6 +29,12 @@ CORE_MODULES = [
 ]
 
 
+def test_package_exposes_version():
+    """__version__ mirrors the installed distribution metadata."""
+    assert isinstance(dataloom_engine.__version__, str)
+    assert dataloom_engine.__version__
+
+
 def test_core_modules_do_not_import_numpy():
     """
     No core module may import numpy at module level. Lazy imports inside

{dataloom_engine-0.4.0 → dataloom_engine-0.5.0}/tests/test_sinks.py

@@ -166,6 +166,40 @@ def test_callback_sink_close_without_handler_is_noop():
     sink.close()  # must not raise
 
 
+def test_threaded_sink_send_racing_close_never_drops_silently():
+    """
+    Regression for the send()/close() race: send() used to check _closed
+    outside the close lock, so an item could be enqueued behind the stop
+    sentinel and silently lost — no delivery, no LoomError. Now the
+    check+put is atomic: every send() either delivers or raises.
+    """
+    for _ in range(50):  # repeat to give the race a chance
+        target = MockSink()
+        buffered_sink = ThreadedBufferedSink(target)
+        outcome = {}
+        barrier = threading.Barrier(2)
+
+        def producer(sink=buffered_sink, outcome=outcome, barrier=barrier):
+            barrier.wait()  # line up with close() for maximum contention
+            try:
+                sink.send({"id": 1})
+                outcome["sent"] = True
+            except LoomError:
+                outcome["sent"] = False
+
+        thread = threading.Thread(target=producer)
+        thread.start()
+        barrier.wait()
+        buffered_sink.close()
+        thread.join(timeout=2)
+        assert not thread.is_alive()
+
+        if outcome["sent"]:
+            assert target.results == [{"id": 1}], "accepted item was dropped"
+        else:
+            assert target.results == []
+
+
 def test_threaded_sink_no_data_loss_on_racy_close():
     """
     Regression for the old worker race (stop_event + queue.empty()):