pyworkflow-engine 0.3.0__tar.gz → 0.3.1__tar.gz

{pyworkflow_engine-0.3.0 → pyworkflow_engine-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyworkflow-engine
-Version: 0.3.0
+Version: 0.3.1
 Summary: A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow
 Author: PyWorkflow Contributors
 License: MIT

{pyworkflow_engine-0.3.0 → pyworkflow_engine-0.3.1}/pyproject.toml

@@ -7,7 +7,7 @@ packages = [{include = "pyworkflow"}]
 
 [project]
 name = "pyworkflow-engine"
-version = "0.3.0"
+version = "0.3.1"
 description = "A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow"
 readme = "README.md"
 requires-python = ">=3.11"

{pyworkflow_engine-0.3.0 → pyworkflow_engine-0.3.1}/pyworkflow/__init__.py

@@ -29,7 +29,7 @@ Quick Start:
     >>> run_id = await start(my_workflow, "Alice")
 """
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 
 # Configuration
 from pyworkflow.config import (

{pyworkflow_engine-0.3.0 → pyworkflow_engine-0.3.1}/pyworkflow/celery/singleton.py

@@ -11,8 +11,9 @@ Based on:
 
 import inspect
 import json
+from collections.abc import Callable
 from hashlib import md5
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeVar
 from uuid import uuid4
 
 from celery import Task
@@ -20,8 +21,11 @@ from celery.exceptions import WorkerLostError
 from loguru import logger
 
 if TYPE_CHECKING:
+    from redis import Redis
     from redis.sentinel import Sentinel
 
+T = TypeVar("T")
+
 
 def generate_lock_key(
     task_name: str,
@@ -74,10 +78,17 @@ class SingletonConfig:
 
 
 class RedisLockBackend:
-    """Redis backend for distributed locking with Sentinel support."""
+    """Redis backend for distributed locking with Sentinel support.
+
+    During Sentinel failover (master rotation), cached connections can raise
+    MasterNotFoundError or socket TimeoutError. On those, we refresh the client
+    via ``sentinel.master_for()`` and retry the operation once. See
+    ``_execute_with_refresh``.
+    """
 
     _sentinel: "Sentinel | None"
     _master_name: str | None
+    redis: "Redis"
 
     def __init__(
         self,
@@ -91,21 +102,62 @@ class RedisLockBackend:
             from redis.sentinel import Sentinel
 
             sentinels = self._parse_sentinel_url(url)
+            # socket_timeout=5 matches FlowHunt's production-tested value.
+            # The previous 0.5s was too tight during Sentinel failover windows.
             self._sentinel = Sentinel(
                 sentinels,
-                socket_timeout=0.5,
+                socket_timeout=5,
                 decode_responses=True,
             )
             self._master_name = sentinel_master or "mymaster"
-            self.redis = self._sentinel.master_for(
-                self._master_name,
-                decode_responses=True,
-            )
+            self.redis = self._build_master_client()
         else:
             self._sentinel = None
             self._master_name = None
             self.redis = redis.from_url(url, decode_responses=True)
 
+    def _build_master_client(self) -> "Redis":
+        """Get a fresh master client from Sentinel. Sentinel mode only."""
+        assert self._sentinel is not None and self._master_name is not None
+        return self._sentinel.master_for(
+            self._master_name,
+            decode_responses=True,
+        )
+
+    def _refresh_client(self) -> None:
+        """Re-resolve master via Sentinel after a failover-related error."""
+        if self._sentinel is None:
+            return
+        self.redis = self._build_master_client()
+        logger.info(
+            "Refreshed Redis master connection for sentinel master '{}'",
+            self._master_name,
+        )
+
+    def _execute_with_refresh(self, fn: Callable[..., T], *args: Any, **kwargs: Any) -> T:
+        """Run ``fn``; on Sentinel failover errors, refresh the client and retry once.
+
+        In non-sentinel mode this is a no-op passthrough — refresh has nothing
+        to do, so any exception propagates as before.
+        """
+        from redis.exceptions import ConnectionError as RedisConnectionError
+        from redis.exceptions import TimeoutError as RedisTimeoutError
+        from redis.sentinel import MasterNotFoundError
+
+        try:
+            return fn(*args, **kwargs)
+        except (MasterNotFoundError, RedisConnectionError, RedisTimeoutError) as exc:
+            if self._sentinel is None:
+                raise
+            logger.warning(
+                "Redis lock op hit sentinel failover error ({}: {}). "
+                "Refreshing master and retrying once.",
+                type(exc).__name__,
+                exc,
+            )
+            self._refresh_client()
+            return fn(*args, **kwargs)
+
     @staticmethod
     def _parse_sentinel_url(url: str) -> list[tuple[str, int]]:
         """
@@ -142,15 +194,19 @@ class RedisLockBackend:
 
     def lock(self, lock_key: str, task_id: str, expiry: int | None = None) -> bool:
         """Acquire lock atomically. Returns True if acquired."""
-        return bool(self.redis.set(lock_key, task_id, nx=True, ex=expiry))
+        return bool(
+            self._execute_with_refresh(
+                lambda: self.redis.set(lock_key, task_id, nx=True, ex=expiry)
+            )
+        )
 
     def unlock(self, lock_key: str) -> None:
         """Release the lock."""
-        self.redis.delete(lock_key)
+        self._execute_with_refresh(lambda: self.redis.delete(lock_key))
 
     def get(self, lock_key: str) -> str | None:
         """Get the task ID holding the lock."""
-        return self.redis.get(lock_key)
+        return self._execute_with_refresh(lambda: self.redis.get(lock_key))
 
 
 class DuplicateTaskError(Exception):
@@ -364,6 +420,16 @@ class SingletonWorkflowTask(Task):
 
         # Lock not acquired - check for existing task
         existing_task_id = self.get_existing_task_id(lock_key)
+
+        # Self-retry: when celery's Task.retry() calls apply_async with the
+        # same task_id we already hold, it's the running task re-queueing
+        # itself. Allow the queue write through — on_retry will release the
+        # lock before the retried message is picked up. Without this branch,
+        # apply_async would silently return AsyncResult without enqueueing,
+        # and the retry would be lost.
+        if existing_task_id == task_id:
+            return super().apply_async(args, kwargs, task_id=task_id, **options)
+
         if existing_task_id:
             logger.debug(
                 "Singleton: duplicate task blocked",
@@ -427,13 +493,17 @@ class SingletonWorkflowTask(Task):
                     error=str(exc),
                 )
 
-        # Log appropriately
+        # Log appropriately. Keep str(exc) out of the message string: when
+        # kwargs are passed, Loguru calls .format() on the message, which
+        # parses any `{` from the exception (e.g. JSON in the message) as a
+        # format spec and raises KeyError.
         if isinstance(exc, WorkerLostError):
             logger.warning("Task interrupted due to worker loss", task_id=task_id)
         else:
             logger.error(
-                f"Task {self.name} failed: {exc}",
+                f"Task {self.name} failed",
                 task_id=task_id,
+                error=str(exc),
                 traceback=einfo.traceback if einfo else None,
             )
 

{pyworkflow_engine-0.3.0 → pyworkflow_engine-0.3.1}/pyworkflow/celery/tasks.py

@@ -442,8 +442,12 @@ def execute_step_task(
         if self.request.retries < max_retries:
             # Use exponential backoff for unexpected errors
             countdown = _calculate_exponential_backoff(self.request.retries)
+            # Don't inline str(e) into the message: Loguru calls .format() on
+            # the message when kwargs are present, and any `{` from the
+            # exception (e.g. a JSON body) is parsed as a format spec and
+            # raises KeyError, which would skip the retry below.
             logger.warning(
-                f"Step failed (unexpected): {step_name}, retrying in {countdown:.1f}s...: {str(e)}",
+                f"Step failed (unexpected): {step_name}, retrying in {countdown:.1f}s...",
                 run_id=run_id,
                 step_id=step_id,
                 error=str(e),

{pyworkflow_engine-0.3.0 → pyworkflow_engine-0.3.1}/tests/unit/test_singleton.py

@@ -785,6 +785,132 @@ class TestRedisLockBackendSentinel:
             assert backend._sentinel is None
 
 
+class TestRedisLockBackendAutoRefresh:
+    """Sentinel failover recovery in RedisLockBackend.
+
+    When the Sentinel master rotates, the cached Redis client can raise
+    ``MasterNotFoundError`` or ``TimeoutError``. The backend should transparently
+    refresh the client via ``sentinel.master_for()`` and retry the operation once.
+    """
+
+    def _make_sentinel_backend(self, mock_sentinel_class, stale_client, fresh_client):
+        """Build a sentinel-backed backend where master_for returns stale then fresh."""
+        mock_sentinel = mock_sentinel_class.return_value
+        mock_sentinel.master_for.side_effect = [stale_client, fresh_client]
+
+        return RedisLockBackend(
+            "sentinel://host1:26379/0",
+            is_sentinel=True,
+            sentinel_master="mymaster",
+        )
+
+    def test_lock_recovers_from_master_not_found(self):
+        """MasterNotFoundError on lock() triggers refresh and succeeds on retry."""
+        from redis.sentinel import MasterNotFoundError
+
+        with patch("redis.sentinel.Sentinel") as mock_sentinel_class:
+            stale = MagicMock()
+            stale.set.side_effect = MasterNotFoundError("No master found for 'mymaster'")
+            fresh = MagicMock()
+            fresh.set.return_value = True
+
+            backend = self._make_sentinel_backend(mock_sentinel_class, stale, fresh)
+            result = backend.lock("test_key", "task_123", expiry=3600)
+
+            assert result is True
+            stale.set.assert_called_once()
+            fresh.set.assert_called_once_with("test_key", "task_123", nx=True, ex=3600)
+            # master_for called twice: initial + refresh
+            assert mock_sentinel_class.return_value.master_for.call_count == 2
+
+    def test_lock_recovers_from_timeout_error(self):
+        """Redis TimeoutError on lock() triggers refresh and succeeds on retry."""
+        from redis.exceptions import TimeoutError as RedisTimeoutError
+
+        with patch("redis.sentinel.Sentinel") as mock_sentinel_class:
+            stale = MagicMock()
+            stale.set.side_effect = RedisTimeoutError("Timeout reading from socket")
+            fresh = MagicMock()
+            fresh.set.return_value = True
+
+            backend = self._make_sentinel_backend(mock_sentinel_class, stale, fresh)
+            result = backend.lock("test_key", "task_123", expiry=3600)
+
+            assert result is True
+            assert mock_sentinel_class.return_value.master_for.call_count == 2
+
+    def test_lock_recovers_from_connection_error(self):
+        """Redis ConnectionError on lock() triggers refresh and succeeds on retry."""
+        from redis.exceptions import ConnectionError as RedisConnectionError
+
+        with patch("redis.sentinel.Sentinel") as mock_sentinel_class:
+            stale = MagicMock()
+            stale.set.side_effect = RedisConnectionError("Connection reset by peer")
+            fresh = MagicMock()
+            fresh.set.return_value = True
+
+            backend = self._make_sentinel_backend(mock_sentinel_class, stale, fresh)
+            assert backend.lock("test_key", "task_123", expiry=3600) is True
+            assert mock_sentinel_class.return_value.master_for.call_count == 2
+
+    def test_unlock_recovers_from_master_rotation(self):
+        """unlock() refreshes and retries once on Sentinel failover errors."""
+        from redis.sentinel import MasterNotFoundError
+
+        with patch("redis.sentinel.Sentinel") as mock_sentinel_class:
+            stale = MagicMock()
+            stale.delete.side_effect = MasterNotFoundError("No master found")
+            fresh = MagicMock()
+
+            backend = self._make_sentinel_backend(mock_sentinel_class, stale, fresh)
+            backend.unlock("test_key")
+
+            fresh.delete.assert_called_once_with("test_key")
+
+    def test_get_recovers_from_master_rotation(self):
+        """get() refreshes and retries once on Sentinel failover errors."""
+        from redis.sentinel import MasterNotFoundError
+
+        with patch("redis.sentinel.Sentinel") as mock_sentinel_class:
+            stale = MagicMock()
+            stale.get.side_effect = MasterNotFoundError("No master found")
+            fresh = MagicMock()
+            fresh.get.return_value = "task_123"
+
+            backend = self._make_sentinel_backend(mock_sentinel_class, stale, fresh)
+            assert backend.get("test_key") == "task_123"
+
+    def test_second_attempt_failure_propagates(self):
+        """If the retry also fails, the exception propagates."""
+        from redis.sentinel import MasterNotFoundError
+
+        with patch("redis.sentinel.Sentinel") as mock_sentinel_class:
+            stale = MagicMock()
+            stale.set.side_effect = MasterNotFoundError("No master found")
+            fresh = MagicMock()
+            fresh.set.side_effect = MasterNotFoundError("Still no master")
+
+            backend = self._make_sentinel_backend(mock_sentinel_class, stale, fresh)
+            with pytest.raises(MasterNotFoundError):
+                backend.lock("test_key", "task_123", expiry=3600)
+
+    def test_non_sentinel_mode_does_not_refresh(self):
+        """Non-sentinel backend has no refresh path — exceptions propagate."""
+        from redis.sentinel import MasterNotFoundError
+
+        with patch("redis.from_url") as mock_from_url:
+            mock_redis = MagicMock()
+            mock_redis.set.side_effect = MasterNotFoundError("No master found")
+            mock_from_url.return_value = mock_redis
+
+            backend = RedisLockBackend("redis://localhost:6379/0")
+            with pytest.raises(MasterNotFoundError):
+                backend.lock("test_key", "task_123", expiry=3600)
+
+            # Only the initial resolution — no refresh attempts.
+            assert mock_redis.set.call_count == 1
+
+
 class TestSingletonConfigSentinel:
     """Test SingletonConfig Sentinel properties."""