durable-worker 0.3.0__tar.gz → 0.4.0__tar.gz

{durable_worker-0.3.0 → durable_worker-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-worker
-Version: 0.3.0
+Version: 0.4.0
 Summary: Python worker SDK for nestjs-durable — run durable workflow steps in Python.
 Project-URL: Homepage, https://github.com/DavideCarvalho/nestjs-durable
 Project-URL: Repository, https://github.com/DavideCarvalho/nestjs-durable

{durable_worker-0.3.0 → durable_worker-0.4.0}/durable_worker/__init__.py

@@ -6,8 +6,17 @@ same step name implemented here is callable from a TypeScript workflow via ``ctx
 """
 
 from .cancellation import Cancelled, CancellationRegistry
+from .redis_runner import redis_url_from_env, run_redis_worker
 from .routing import reply_target
-from .worker import FatalError, StepContext, Worker
+from .worker import (
+    FatalError,
+    StepContext,
+    Worker,
+    current_step,
+    log,
+    set_process,
+    sub,
+)
 
 __all__ = [
     "Worker",
@@ -16,5 +25,13 @@ __all__ = [
     "Cancelled",
     "CancellationRegistry",
     "reply_target",
+    # Context-local step access — record events from anywhere inside a handler without threading ctx.
+    "current_step",
+    "log",
+    "sub",
+    "set_process",
+    # Transport bootstrap helpers.
+    "run_redis_worker",
+    "redis_url_from_env",
 ]
-__version__ = "0.3.0"
+__version__ = "0.4.0"

{durable_worker-0.3.0 → durable_worker-0.4.0}/durable_worker/redis_runner.py

@@ -26,6 +26,20 @@ from .worker import Worker
 _INSTANCE_ID = f"py-{socket.gethostname()}-{os.getpid()}"
 
 
+def redis_url_from_env(prefix: str = "REDIS") -> str:
+    """Build a ``redis://`` URL from ``{prefix}_HOST/_PORT/_USERNAME/_PASSWORD`` env vars. The
+    credentials are URL-encoded — a generated password often contains ``@ : /`` which would corrupt
+    the netloc if left raw. Defaults to ``localhost:6379`` with no auth. Handy for ``Worker.run``."""
+    from urllib.parse import quote
+
+    host = os.getenv(f"{prefix}_HOST", "localhost")
+    port = os.getenv(f"{prefix}_PORT", "6379")
+    user = os.getenv(f"{prefix}_USERNAME") or ""
+    password = os.getenv(f"{prefix}_PASSWORD") or ""
+    auth = f"{quote(user, safe='')}:{quote(password, safe='')}@" if (user or password) else ""
+    return f"redis://{auth}{host}:{port}"
+
+
 def _names(prefix: str, group: str) -> tuple[str, str]:
     # Must match the TS BullMQTransport: '<prefix>-tasks-<group>' and '<prefix>-results'.
     return f"{prefix}-tasks-{group}", f"{prefix}-results"

{durable_worker-0.3.0 → durable_worker-0.4.0}/durable_worker/worker.py

@@ -8,12 +8,48 @@ and ships results out.
 from __future__ import annotations
 
 import asyncio
+import contextvars
 import inspect
 import time
 from typing import Any, Awaitable, Callable, Dict, List, Optional, Union
 
 Handler = Callable[..., Union[Any, Awaitable[Any]]]
 
+# The step a handler is currently running, bound for the duration of the call (in the loop for an
+# async handler, in the executor thread for a blocking one). Lets code DEEP inside a handler — that
+# never received `ctx` — record events via the module-level helpers below, without threading the
+# context through every call. Outside a step it's None and the helpers are no-ops.
+_current_step: "contextvars.ContextVar[Optional[StepContext]]" = contextvars.ContextVar(
+    "durable_current_step", default=None
+)
+
+
+def current_step() -> "Optional[StepContext]":
+    """The :class:`StepContext` of the step running on this task/thread, or None outside a step."""
+    return _current_step.get()
+
+
+def log(level: str, message: str, data: Any = None) -> None:
+    """Record a log line on the current step (level: debug/info/warn/error). No-op outside a step."""
+    ctx = _current_step.get()
+    if ctx is not None:
+        getattr(ctx, level, ctx.info)(message, data)
+
+
+def sub(name: str, status: str, message: Optional[str] = None, data: Any = None) -> None:
+    """Record a sub-process outcome (ok/failed/skipped) on the current step. No-op outside a step."""
+    ctx = _current_step.get()
+    if ctx is not None:
+        ctx.sub(name, status, message, data)
+
+
+def set_process(name: Optional[str]) -> None:
+    """Tag the current step's subsequent log lines with sub-process ``name`` (None clears). No-op
+    outside a step — so the same business code runs on a non-durable path untouched."""
+    ctx = _current_step.get()
+    if ctx is not None:
+        ctx.process(name)
+
 
 class FatalError(Exception):
     """Raise inside a handler to signal a non-retryable failure (mirrors the TS ``FatalError``).
@@ -148,12 +184,20 @@ class Worker:
     def __init__(self, group: str = "default") -> None:
         self.group = group
         self._handlers: Dict[str, Handler] = {}
+        self._blocking: Dict[str, bool] = {}
 
-    def step(self, name: str) -> Callable[[Handler], Handler]:
-        """Decorator registering ``fn`` as the handler for step ``name``."""
+    def step(self, name: str, *, blocking: bool = False) -> Callable[[Handler], Handler]:
+        """Decorator registering ``fn`` as the handler for step ``name``.
+
+        Set ``blocking=True`` for a synchronous handler that does CPU/DB work: the worker runs it in
+        a thread so the event loop stays free to renew the broker's job lock (otherwise a long step
+        looks stalled and gets redelivered). The step context is bound inside that thread too, so the
+        module-level :func:`log`/:func:`sub`/:func:`set_process` work from anywhere in the call.
+        """
 
         def register(fn: Handler) -> Handler:
             self._handlers[name] = fn
+            self._blocking[name] = blocking
             return fn
 
         return register
@@ -161,6 +205,47 @@ class Worker:
     def handles(self, name: str) -> bool:
         return name in self._handlers
 
+    def run(self, *, redis: str = "redis://localhost:6379", prefix: str = "durable") -> None:
+        """Run this worker against the Redis/BullMQ transport until the process is signalled, then
+        close gracefully. Owns the whole long-running bootstrap a worker process needs — the event
+        loop, SIGTERM/SIGINT handling, and the broker connection — so the entrypoint is one call::
+
+            worker = Worker(group="processing")
+
+            @worker.step("processing", blocking=True)
+            def handle(data, ctx): ...
+
+            worker.run(redis=redis_url_from_env())
+
+        Blocks until a shutdown signal closes the worker (which finishes the active job and releases
+        its lock). For finer control (your own loop/signals), use :func:`run_redis_worker` directly.
+        """
+        import signal as _signal
+
+        from .redis_runner import run_redis_worker
+
+        async def _main() -> None:
+            bull_worker = await run_redis_worker(
+                self, group=self.group, connection=redis, prefix=prefix
+            )
+            stop = asyncio.Event()
+            loop = asyncio.get_running_loop()
+
+            async def _graceful() -> None:
+                try:
+                    await bull_worker.close()
+                finally:
+                    stop.set()
+
+            for sig in (_signal.SIGTERM, _signal.SIGINT):
+                try:
+                    loop.add_signal_handler(sig, lambda: asyncio.ensure_future(_graceful()))
+                except (NotImplementedError, RuntimeError):
+                    pass  # no signal support here (e.g. not the main thread) — rely on redelivery
+            await stop.wait()
+
+        asyncio.run(_main())
+
     def process_task(
         self,
         task: Dict[str, Any],
@@ -181,7 +266,7 @@ class Worker:
             return self._no_handler(base, task["name"])
         ctx = self._ctx(task, is_cancelled, on_event)
         try:
-            output = self._invoke(handler, task.get("input"), ctx)
+            output = _run_bound(handler, task.get("input"), ctx)
             if inspect.isawaitable(output):
                 output = asyncio.run(output)
             return self._completed(base, output, ctx)
@@ -195,7 +280,8 @@ class Worker:
         on_event: Optional[Callable[[Dict[str, Any]], None]] = None,
     ) -> Dict[str, Any]:
         """Async variant — awaits async handlers in the current loop. Use from a transport that
-        already runs inside an event loop (e.g. the BullMQ runner)."""
+        already runs inside an event loop (e.g. the BullMQ runner). A handler registered
+        ``blocking=True`` runs in a thread so the loop stays free to renew the job lock."""
 
         base = self._base(task)
         handler = self._handlers.get(task["name"])
@@ -203,9 +289,21 @@ class Worker:
             return self._no_handler(base, task["name"])
         ctx = self._ctx(task, is_cancelled, on_event)
         try:
-            output = self._invoke(handler, task.get("input"), ctx)
-            if inspect.isawaitable(output):
-                output = await output
+            if self._blocking.get(task["name"]):
+                loop = asyncio.get_running_loop()
+                output = await loop.run_in_executor(
+                    None, _run_bound, handler, task.get("input"), ctx
+                )
+            else:
+                output = _run_bound(handler, task.get("input"), ctx)
+                if inspect.isawaitable(output):
+                    # The context var was reset by _run_bound after the sync part returned the
+                    # coroutine; re-bind it for the duration of the await so deep `log()` calls work.
+                    token = _current_step.set(ctx)
+                    try:
+                        output = await output
+                    finally:
+                        _current_step.reset(token)
             return self._completed(base, output, ctx)
         except Exception as err:  # noqa: BLE001
             return self._failure(base, err, ctx)
@@ -223,14 +321,6 @@ class Worker:
             on_event=on_event,
         )
 
-    @staticmethod
-    def _invoke(handler: Handler, input_: Any, ctx: StepContext) -> Any:
-        """Call ``handler`` with the step input, passing ``ctx`` only if it declares a second
-        parameter — so plain ``def handler(data)`` handlers keep working unchanged."""
-        if _wants_ctx(handler):
-            return handler(input_, ctx)
-        return handler(input_)
-
     @staticmethod
     def _base(task: Dict[str, Any]) -> Dict[str, Any]:
         # Stamp the worker's pickup time so the engine can report queue-wait (startedAt − enqueuedAt),
@@ -265,6 +355,18 @@ class Worker:
         return _with_events({**base, "status": "failed", "error": error}, ctx)
 
 
+def _run_bound(handler: Handler, input_: Any, ctx: "StepContext") -> Any:
+    """Call ``handler`` with ``ctx`` bound as the current step for the duration of the call, so the
+    module-level :func:`log`/:func:`sub`/:func:`set_process` reach it from deep in the call tree.
+    Passes ``ctx`` as a second arg only if the handler declares one (plain ``def h(data)`` still
+    works). Also the target run in the executor thread for a ``blocking=True`` handler."""
+    token = _current_step.set(ctx)
+    try:
+        return handler(input_, ctx) if _wants_ctx(handler) else handler(input_)
+    finally:
+        _current_step.reset(token)
+
+
 def _wants_ctx(handler: Handler) -> bool:
     """True if ``handler`` can accept the step context as a second positional argument."""
     try:

{durable_worker-0.3.0 → durable_worker-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "durable-worker"
-version = "0.3.0"
+version = "0.4.0"
 description = "Python worker SDK for nestjs-durable — run durable workflow steps in Python."
 readme = "README.md"
 requires-python = ">=3.9"

durable_worker-0.4.0/tests/test_redis_url.py

@@ -0,0 +1,51 @@
+import unittest
+
+from durable_worker import redis_url_from_env
+
+
+class RedisUrlFromEnvTest(unittest.TestCase):
+    def setUp(self):
+        import os
+
+        self._saved = {
+            k: os.environ.pop(k, None)
+            for k in ("REDIS_HOST", "REDIS_PORT", "REDIS_USERNAME", "REDIS_PASSWORD")
+        }
+
+    def tearDown(self):
+        import os
+
+        for k, v in self._saved.items():
+            if v is None:
+                os.environ.pop(k, None)
+            else:
+                os.environ[k] = v
+
+    def test_defaults_to_localhost_without_auth(self):
+        self.assertEqual(redis_url_from_env(), "redis://localhost:6379")
+
+    def test_host_port_and_auth(self):
+        import os
+
+        os.environ["REDIS_HOST"] = "redis.internal"
+        os.environ["REDIS_PORT"] = "6380"
+        self.assertEqual(redis_url_from_env(), "redis://redis.internal:6380")
+        os.environ["REDIS_USERNAME"] = "default"
+        os.environ["REDIS_PASSWORD"] = "secret"
+        self.assertEqual(redis_url_from_env(), "redis://default:secret@redis.internal:6380")
+
+    def test_url_encodes_credentials(self):
+        import os
+
+        os.environ["REDIS_HOST"] = "redis.internal"
+        os.environ["REDIS_PORT"] = "6380"
+        os.environ["REDIS_USERNAME"] = "default"
+        # A password with @ : / would corrupt the netloc if left raw.
+        os.environ["REDIS_PASSWORD"] = "p@ss:w/rd"
+        self.assertEqual(
+            redis_url_from_env(), "redis://default:p%40ss%3Aw%2Frd@redis.internal:6380"
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

{durable_worker-0.3.0 → durable_worker-0.4.0}/tests/test_worker.py

@@ -1,6 +1,15 @@
+import asyncio
 import unittest
 
-from durable_worker import FatalError, StepContext, Worker
+from durable_worker import (
+    FatalError,
+    StepContext,
+    Worker,
+    current_step,
+    log,
+    set_process,
+    sub,
+)
 
 
 def task(name="payments.charge-card", **over):
@@ -179,5 +188,54 @@ class WorkerTest(unittest.TestCase):
         self.assertEqual(len(result["events"]), 2)
 
 
+    def test_module_level_helpers_reach_the_current_step(self):
+        worker = Worker()
+
+        def deep_business_logic():
+            # No `ctx` in scope here — the context-local helpers find it.
+            set_process("proc-a")
+            log("debug", "querying")
+            sub("proc-a", "ok")
+
+        @worker.step("payments.charge-card")
+        def charge(_data):  # note: no ctx param — handler doesn't even take it
+            deep_business_logic()
+            return {"ok": True}
+
+        events = worker.process_task(task())["events"]
+        self.assertEqual(events[0]["message"], "querying")
+        self.assertEqual(events[0]["process"], "proc-a")
+        self.assertEqual(events[1]["name"], "proc-a")
+        self.assertEqual(events[1]["status"], "ok")
+
+    def test_module_level_helpers_are_noops_outside_a_step(self):
+        # Same business code on a non-durable path must not blow up.
+        self.assertIsNone(current_step())
+        log("info", "no step here")
+        sub("x", "ok")
+        set_process("x")
+
+    def test_blocking_handler_runs_in_a_thread_with_the_step_bound(self):
+        import threading
+
+        worker = Worker()
+        ran_on = {}
+
+        @worker.step("payments.charge-card", blocking=True)
+        def charge(_data):
+            ran_on["thread"] = threading.current_thread().name
+            log("info", "from the worker thread")
+            return {"ok": True}
+
+        async def drive():
+            return await worker.aprocess_task(task())
+
+        loop_thread = threading.current_thread().name
+        result = asyncio.run(drive())
+        self.assertEqual(result["events"][0]["message"], "from the worker thread")
+        # it really ran off the event-loop thread (so the lock keeps renewing)
+        self.assertNotEqual(ran_on["thread"], loop_thread)
+
+
 if __name__ == "__main__":
     unittest.main()