dataloom-engine 0.4.0__tar.gz → 0.4.1__tar.gz

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.4.0
-Summary: DataLoom: Um orquestrador de threads leve e eficiente para dados.
+Version: 0.4.1
+Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT
 Project-URL: Homepage, https://github.com/dionipadilha/dataloom
@@ -175,6 +175,10 @@ if __name__ == "__main__":
         print("\n🛑 Loom stopped.")
 ```
 
+More runnable, real-world examples — API enrichment with parallel speedup,
+a resilient file-processing pipeline, a buffered sensor stream — live in the
+[`examples/`](examples/) directory.
+
 ## 🏗️ Architecture
 
 DataLoom is built around a weaving metaphor:

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/README.md

@@ -141,6 +141,10 @@ if __name__ == "__main__":
         print("\n🛑 Loom stopped.")
 ```
 
+More runnable, real-world examples — API enrichment with parallel speedup,
+a resilient file-processing pipeline, a buffered sensor stream — live in the
+[`examples/`](examples/) directory.
+
 ## 🏗️ Architecture
 
 DataLoom is built around a weaving metaphor:

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/dataloom_engine/__init__.py

@@ -8,6 +8,8 @@ such as 'Weaver' are deliberately not exposed, keeping the usage surface
 clean and safe for consumers.
 """
 
+from importlib.metadata import PackageNotFoundError, version
+
 from dataloom_engine.config import LoomConfig
 from dataloom_engine.exceptions import ConfigurationError, LoomError, WeaverError
 from dataloom_engine.hooks import LoomHooks
@@ -24,7 +26,13 @@ from dataloom_engine.sinks import (
 from dataloom_engine.sources import Source
 from dataloom_engine.types import LoomState
 
+try:
+    __version__ = version("dataloom-engine")
+except PackageNotFoundError:  # pragma: no cover -- running from an uninstalled source tree
+    __version__ = "0+unknown"
+
 __all__ = [
+    "__version__",
     "Loom",
     "LoomConfig",
     "LoomState",

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/dataloom_engine/loom.py

@@ -11,6 +11,7 @@ import threading
 from typing import TYPE_CHECKING, Optional
 
 from dataloom_engine.config import LoomConfig
+from dataloom_engine.exceptions import ConfigurationError, LoomError
 from dataloom_engine.hooks import LoomHooks
 from dataloom_engine.processors import Processor
 from dataloom_engine.sinks import Sink
@@ -45,6 +46,9 @@ class Loom:
         hooks: Optional[LoomHooks] = None,
         num_weavers: int = 2,
     ):
+        if num_weavers < 1:
+            raise ConfigurationError(f"num_weavers must be at least 1 (got: {num_weavers}).")
+
         self.config = config
         self.processor = processor
         self.sink = sink
@@ -90,7 +94,18 @@ class Loom:
         """
         Starts the Weavers and begins the task production loop.
         This method blocks until an error occurs or the loom is stopped.
+
+        A Loom instance is single-use: calling start() again after it has
+        run (or after stop()) raises LoomError.
         """
+        # Restarting a stopped instance would spawn Weavers that never
+        # receive a stop sentinel (stop() is idempotent), leaking threads.
+        if self.state is not LoomState.PENDING or self._stopped:
+            raise LoomError(
+                "This Loom has already been started or stopped; "
+                "create a new instance to run another pipeline."
+            )
+
         self.state = LoomState.RUNNING
         self.hooks.on_start()
 

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/dataloom_engine/sinks.py

@@ -44,17 +44,16 @@ class JsonFileSink(Sink):
     Uses a threading.Lock to keep concurrent writes consistent.
     """
 
-    def __init__(self, output_dir: Path):
-        self.output_dir = output_dir
+    def __init__(self, output_dir: Path, filename: str = "results.json"):
+        self.output_dir = Path(output_dir)
         self.output_dir.mkdir(parents=True, exist_ok=True)
+        self._path = self.output_dir / filename
         # The lock ensures only one Weaver writes to the file at a time
         self._lock = threading.Lock()
 
     def send(self, result: Dict[str, Any]) -> None:
-        filename = self.output_dir / "results.json"
-
         with self._lock:
-            with open(filename, "a") as f:
+            with open(self._path, "a") as f:
                 json.dump(result, f)
                 f.write("\n")
 
@@ -138,9 +137,14 @@ class ThreadedBufferedSink(Sink):
         self.worker_thread.start()
 
     def send(self, result: Dict[str, Any]) -> None:
-        if self._closed:
-            raise LoomError("ThreadedBufferedSink is already closed; send() is not allowed.")
-        self.queue.put(result)
+        # The closed check and the put must be atomic with respect to
+        # close(): otherwise a send() racing with close() could pass the
+        # check, lose the CPU while close() drains the buffer, and then
+        # enqueue the item behind the stop sentinel — silently lost.
+        with self._close_lock:
+            if self._closed:
+                raise LoomError("ThreadedBufferedSink is already closed; send() is not allowed.")
+            self.queue.put(result)
 
     def _worker(self) -> None:
         while True:
@@ -157,15 +161,16 @@ class ThreadedBufferedSink(Sink):
                 self.queue.task_done()
 
     def close(self) -> None:
-        # Idempotent: only the first call performs the shutdown
+        # Idempotent: only the first call performs the shutdown. The
+        # sentinel is enqueued under the same lock as send(), so it is
+        # guaranteed to land behind every accepted item — the worker
+        # drains all of them before exiting.
         with self._close_lock:
             if self._closed:
                 return
             self._closed = True
+            self.queue.put(self._STOP)
 
-        # The sentinel goes in behind any pending items: the worker
-        # drains everything before exiting
-        self.queue.put(self._STOP)
         self.worker_thread.join()
 
         # Propagate the close

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/dataloom_engine.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: dataloom-engine
-Version: 0.4.0
-Summary: DataLoom: Um orquestrador de threads leve e eficiente para dados.
+Version: 0.4.1
+Summary: DataLoom: a lightweight and efficient thread orchestration engine for data pipelines.
 Author-email: Dioni Padilha <dionipdl@gmail.com>
 License: MIT
 Project-URL: Homepage, https://github.com/dionipadilha/dataloom
@@ -175,6 +175,10 @@ if __name__ == "__main__":
         print("\n🛑 Loom stopped.")
 ```
 
+More runnable, real-world examples — API enrichment with parallel speedup,
+a resilient file-processing pipeline, a buffered sensor stream — live in the
+[`examples/`](examples/) directory.
+
 ## 🏗️ Architecture
 
 DataLoom is built around a weaving metaphor:

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataloom-engine"
-version = "0.4.0"
-description = "DataLoom: Um orquestrador de threads leve e eficiente para dados."
+version = "0.4.1"
+description = "DataLoom: a lightweight and efficient thread orchestration engine for data pipelines."
 readme = "README.md"
 requires-python = ">=3.9"
 license = {text = "MIT"}

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/tests/test_core.py

@@ -11,11 +11,12 @@ import threading
 
 import numpy as np
 
-from dataloom_engine import JsonFileSink, Processor, Sink
+from dataloom_engine import JsonFileSink, LoomConfig, Processor, Sink
 
 # Importing internal classes explicitly for testing
 from dataloom_engine.exceptions import WeaverError
 from dataloom_engine.processors import StatisticsProcessor
+from dataloom_engine.sources import RandomNumPySource
 from dataloom_engine.weaver import STOP_SENTINEL, Weaver
 
 # --- Mocks and helpers ---
@@ -76,6 +77,30 @@ def test_json_sink_writes_file(tmp_path):
     assert loaded_json == data
 
 
+def test_json_sink_accepts_custom_filename(tmp_path):
+    """The output filename is configurable (parity with CsvFileSink)."""
+    sink = JsonFileSink(output_dir=tmp_path, filename="custom.jsonl")
+    sink.send({"id": 7})
+
+    assert (tmp_path / "custom.jsonl").exists()
+    assert not (tmp_path / "results.json").exists()
+    assert json.loads((tmp_path / "custom.jsonl").read_text()) == {"id": 7}
+
+
+def test_random_numpy_source_respects_limit_and_batch_size():
+    """The demo source yields exactly `limit` batches of `batch_size` values in [0, 1)."""
+    config = LoomConfig(output_dir=".", batch_size=5, interval_seconds=0)
+    source = RandomNumPySource(config, limit=3)
+
+    batches = list(source)
+
+    assert len(batches) == 3
+    for batch in batches:
+        assert len(batch) == 5
+        assert float(batch.min()) >= 0.0
+        assert float(batch.max()) < 1.0
+
+
 # --- Integration tests (Weaver/flow) ---
 
 

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/tests/test_loom.py

@@ -7,7 +7,7 @@ import numpy as np
 import pytest
 
 from dataloom_engine import Loom, LoomConfig, LoomHooks, LoomState, Processor, Sink
-from dataloom_engine.exceptions import WeaverError
+from dataloom_engine.exceptions import ConfigurationError, LoomError, WeaverError
 from dataloom_engine.sources import Source
 
 # --- Mocks ---
@@ -61,11 +61,8 @@ def test_loom_uses_custom_source():
 
     loom = Loom(config=config, processor=processor, sink=sink, source=source, num_weavers=2)
 
-    # Run Loom (start blocks until source is exhausted or error)
-    # But wait, Loom.start() blocks until source is exhausted AND then calls stop().
-    # Since our source is finite, it should finish naturally.
-    if hasattr(loom, "start"):
-        loom.start()
+    # start() blocks until the finite source is exhausted, then stops itself
+    loom.start()
 
     # Verify results
     assert len(sink.results) == 3
@@ -278,6 +275,62 @@ def test_loom_survives_broken_metrics_hook():
     assert loom.state is LoomState.COMPLETED
 
 
+@pytest.mark.parametrize("num_weavers", [0, -1])
+def test_loom_rejects_non_positive_num_weavers(num_weavers):
+    """
+    num_weavers < 1 must fail fast: 0 weavers would silently create an
+    unbounded queue (0 * 4 = 0 = no limit) and complete without
+    processing anything.
+    """
+    config = LoomConfig(output_dir=".", batch_size=1, interval_seconds=0)
+    with pytest.raises(ConfigurationError):
+        Loom(
+            config=config,
+            processor=PassthroughProcessor(),
+            sink=InMemorySink(),
+            source=FiniteSource([1]),
+            num_weavers=num_weavers,
+        )
+
+
+def test_loom_start_cannot_be_reused():
+    """
+    A Loom instance is single-use: a second start() must raise instead of
+    spawning Weavers that never receive a stop sentinel (thread leak with
+    the state stuck in RUNNING).
+    """
+
+    class CountingHooks(LoomHooks):
+        def __init__(self):
+            self.start_calls = 0
+
+        def on_start(self):
+            self.start_calls += 1
+
+    hooks = CountingHooks()
+    loom = _make_loom(FiniteSource([1]), hooks=hooks)
+    loom.start()
+    weavers_after_first = len(loom.weavers)
+
+    with pytest.raises(LoomError):
+        loom.start()
+
+    assert len(loom.weavers) == weavers_after_first  # no leaked threads
+    assert loom.state is LoomState.COMPLETED  # not stuck in RUNNING
+    assert hooks.start_calls == 1  # the rejected start never fired hooks
+
+
+def test_loom_start_after_early_stop_raises():
+    """stop() before start() must also make start() unusable (same leak scenario)."""
+    loom = _make_loom(FiniteSource([1]))
+    loom.stop()
+
+    with pytest.raises(LoomError):
+        loom.start()
+
+    assert loom.weavers == []
+
+
 def test_loom_as_context_manager():
     """The with block yields the Loom itself and guarantees stop() on exit."""
     hooks = RecordingHooks()

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/tests/test_optional.py

@@ -29,6 +29,12 @@ CORE_MODULES = [
 ]
 
 
+def test_package_exposes_version():
+    """__version__ mirrors the installed distribution metadata."""
+    assert isinstance(dataloom_engine.__version__, str)
+    assert dataloom_engine.__version__
+
+
 def test_core_modules_do_not_import_numpy():
     """
     No core module may import numpy at module level. Lazy imports inside

{dataloom_engine-0.4.0 → dataloom_engine-0.4.1}/tests/test_sinks.py

@@ -166,6 +166,40 @@ def test_callback_sink_close_without_handler_is_noop():
     sink.close()  # must not raise
 
 
+def test_threaded_sink_send_racing_close_never_drops_silently():
+    """
+    Regression for the send()/close() race: send() used to check _closed
+    outside the close lock, so an item could be enqueued behind the stop
+    sentinel and silently lost — no delivery, no LoomError. Now the
+    check+put is atomic: every send() either delivers or raises.
+    """
+    for _ in range(50):  # repeat to give the race a chance
+        target = MockSink()
+        buffered_sink = ThreadedBufferedSink(target)
+        outcome = {}
+        barrier = threading.Barrier(2)
+
+        def producer(sink=buffered_sink, outcome=outcome, barrier=barrier):
+            barrier.wait()  # line up with close() for maximum contention
+            try:
+                sink.send({"id": 1})
+                outcome["sent"] = True
+            except LoomError:
+                outcome["sent"] = False
+
+        thread = threading.Thread(target=producer)
+        thread.start()
+        barrier.wait()
+        buffered_sink.close()
+        thread.join(timeout=2)
+        assert not thread.is_alive()
+
+        if outcome["sent"]:
+            assert target.results == [{"id": 1}], "accepted item was dropped"
+        else:
+            assert target.results == []
+
+
 def test_threaded_sink_no_data_loss_on_racy_close():
     """
     Regression for the old worker race (stop_event + queue.empty()):