scalable-pypeline 2.1.22__tar.gz → 2.1.24__tar.gz

{scalable-pypeline-2.1.22/scalable_pypeline.egg-info → scalable-pypeline-2.1.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.22
+Version: 2.1.24
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

scalable-pypeline-2.1.24/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.1.24"

scalable-pypeline-2.1.24/pypeline/barrier.py

@@ -0,0 +1,63 @@
+import time
+import redis
+from redis.sentinel import Sentinel
+from urllib.parse import urlparse
+from constants import (
+    REDIS_SENTINEL_MASTER_NAME,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+)
+
+
+class LockingParallelBarrier:
+    def __init__(self, redis_url, task_key="task_counter", lock_key="task_lock"):
+        # Connect to Redis using the provided URL
+        if REDIS_SENTINEL_MASTER_NAME is not None:
+            parsed_redis_url = urlparse(redis_url)
+            redis_sentinel = Sentinel(
+                sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+            )
+            self.redis = redis_sentinel.master_for(
+                REDIS_SENTINEL_MASTER_NAME,
+                db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+                password=parsed_redis_url.password,
+                socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+                socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+                socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+                health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+                decode_responses=True,
+            )
+        else:
+            self.redis = redis.StrictRedis.from_url(redis_url, decode_responses=True)
+        self.task_key = task_key
+        self.lock_key = lock_key
+
+    def acquire_lock(self, timeout=5):
+        """Acquire a lock using Redis."""
+        while True:
+            if self.redis.set(self.lock_key, "locked", nx=True, ex=timeout):
+                return True
+            time.sleep(0.1)
+
+    def release_lock(self):
+        """Release the lock in Redis."""
+        self.redis.delete(self.lock_key)
+
+    def set_task_count(self, count):
+        """Initialize the task counter in Redis."""
+        self.redis.set(self.task_key, count)
+
+    def decrement_task_count(self):
+        """Decrement the task counter in Redis."""
+        return self.redis.decr(self.task_key)
+
+    def task_exists(self):
+        return self.redis.exists(self.task_key)
+
+    def get_task_count(self):
+        """Get the current value of the task counter."""
+        return int(self.redis.get(self.task_key) or 0)

{scalable-pypeline-2.1.22 → scalable-pypeline-2.1.24}/pypeline/constants.py

@@ -1,5 +1,4 @@
-""" Pypeline Constants
-"""
+"""Pypeline Constants"""
 
 import os
 
@@ -15,6 +14,7 @@ DEFAULT_BROKER_CALLABLE = os.environ.get(
 # Pypeline broker connections
 RABBIT_URL = os.environ.get("RABBIT_URL", "amqp://admin:password@127.0.0.1:5672")
 REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
+REDIS_SENTINEL_MASTER_NAME = os.environ.get("REDIS_SENTINEL_MASTER_NAME", None)
 
 # Pypeline task defaults
 PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL = int(
@@ -39,6 +39,16 @@ DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT = int(
 DEFAULT_BROKER_HEARTBEAT_TIMEOUT = int(
     os.getenv("DEFAULT_BROKER_HEARTBEAT_TIMEOUT", 300000)
 )
+DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT = int(
+    os.getenv("DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT", 1)
+)
+DEFAULT_REDIS_SOCKET_TIMEOUT = int(os.getenv("DEFAULT_REDIS_SOCKET_TIMEOUT", 2))
+DEFAULT_REDIS_RETRY_ON_TIMEOUT = bool(os.getenv("DEFAULT_REDIS_RETRY_ON_TIMEOUT", True))
+DEFAULT_REDIS_SOCKET_KEEPALIVE = bool(os.getenv("DEFAULT_REDIS_SOCKET_KEEPALIVE", True))
+DEFAULT_REDIS_HEALTH_CHECK_INTERVAL = int(
+    os.getenv("DEFAULT_REDIS_HEALTH_CHECK_INTERVAL", 30)
+)
+
 MESSAGE_BROKER = os.getenv("MESSAGE_BROKER", "RABBITMQ")
 
 MS_IN_SECONDS = 1000

{scalable-pypeline-2.1.22 → scalable-pypeline-2.1.24}/pypeline/dramatiq.py

@@ -8,7 +8,7 @@ import click
 from urllib.parse import urlparse
 
 from dramatiq.brokers.redis import RedisBroker
-
+from redis.sentinel import Sentinel
 from pypeline.extensions import pypeline_config
 from warnings import warn
 from apscheduler.schedulers.blocking import BlockingScheduler
@@ -30,6 +30,7 @@ from flask.cli import with_appcontext
 
 from pypeline.constants import (
     REDIS_URL,
+    REDIS_SENTINEL_MASTER_NAME,
     RABBIT_URL,
     DEFAULT_BROKER_CALLABLE,
     DEFAULT_BROKER_CONNECTION_HEARTBEAT,
@@ -37,6 +38,11 @@ from pypeline.constants import (
     DEFAULT_BROKER_CONNECTION_ATTEMPTS,
     MESSAGE_BROKER,
     DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
 )
 from pypeline.pipelines.middleware.get_active_worker_id_middleware import (
     GetActiveWorkerIdMiddleware,
@@ -45,7 +51,8 @@ from pypeline.pipelines.middleware.parallel_pipeline_middleware import ParallelP
 from pypeline.pipelines.middleware.pypeline_middleware import PypelineMiddleware
 from pypeline.utils.config_utils import (
     retrieve_latest_schedule_config,
-    get_service_config_for_worker, retrieve_executable_job_config,
+    get_service_config_for_worker,
+    retrieve_executable_job_config,
 )
 from pypeline.utils.dramatiq_utils import (
     guess_code_directory,
@@ -62,7 +69,23 @@ logger = logging.getLogger(__name__)
 
 
 def configure_default_broker(broker: Broker = None):
-    redis_backend = RedisBackend(url=REDIS_URL)
+    redis_client = None
+    if REDIS_SENTINEL_MASTER_NAME is not None:
+        parsed_redis_url = urlparse(REDIS_URL)
+        redis_sentinel = Sentinel(
+            sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+        )
+        redis_client = redis_sentinel.master_for(
+            REDIS_SENTINEL_MASTER_NAME,
+            db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+            password=parsed_redis_url.password,
+            socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+            socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+            retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+            socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+            health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+        )
+    redis_backend = RedisBackend(client=redis_client, url=REDIS_URL)
 
     if MESSAGE_BROKER == "RABBITMQ":
         parsed_url = urlparse(RABBIT_URL)
@@ -85,7 +108,9 @@ def configure_default_broker(broker: Broker = None):
             broker
             if broker is not None
             else RedisBroker(
-                url=REDIS_URL, heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT
+                client=redis_client,
+                url=REDIS_URL,
+                heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
             )
         )
 
@@ -169,7 +194,6 @@ def register_actors_for_workers(broker: Broker):
                 logger.exception(f"Unable to add a task to dramatiq: {e}")
 
 
-
 class Dramatiq:
     """Flask extension bridging Dramatiq broker and Flask app.
 

{scalable-pypeline-2.1.22 → scalable-pypeline-2.1.24}/pypeline/utils/dramatiq_utils.py

@@ -113,11 +113,15 @@ class LazyActor(object):
         return getattr(self.actor, name)
 
     def register(self, broker):
-        self.actor = register_actor(
-            actor_name=f"{self.fn.__module__}.{self.fn.__name__}-{self.kw['queue_name']}",
-            broker=broker,
-            **self.kw,
-        )(ensure_return_value(default_value=True)(self.fn))
+        actor_name = f"{self.fn.__module__}.{self.fn.__name__}-{self.kw['queue_name']}"
+        if actor_name in broker.actors:
+            self.actor = broker.actors[actor_name]
+        else:
+            self.actor = register_actor(
+                actor_name=actor_name,
+                broker=broker,
+                **self.kw,
+            )(ensure_return_value(default_value=True)(self.fn))
 
     # Next is regular actor API.
     def send(self, *a, **kw):

{scalable-pypeline-2.1.22 → scalable-pypeline-2.1.24/scalable_pypeline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.22
+Version: 2.1.24
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable-pypeline-2.1.22 → scalable-pypeline-2.1.24}/scalable_pypeline.egg-info/requires.txt

@@ -20,7 +20,7 @@ pytest-cov<3,>=2.6.1
 tox<4,>=3.14.1
 mock<2,>=1
 responses<0.11,>=0.10.16
-fakeredis<3,>=2.10.3
+fakeredis<2.31,>=2.10.3
 
 [web]
 gunicorn

{scalable-pypeline-2.1.22 → scalable-pypeline-2.1.24}/setup.py

@@ -190,7 +190,7 @@ setup(
             "tox>=3.14.1,<4",
             "mock>=1,<2",
             "responses>=0.10.16,<0.11",
-            "fakeredis>=2.10.3,<3",
+            "fakeredis>=2.10.3,<2.31",  # fakeredis version compatible with redis 4.x
         ],
     },
     entry_points={

scalable-pypeline-2.1.22/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.1.22"

scalable-pypeline-2.1.22/pypeline/barrier.py

@@ -1,37 +0,0 @@
-import time
-
-import redis
-
-
-class LockingParallelBarrier:
-    def __init__(self, redis_url, task_key="task_counter", lock_key="task_lock"):
-        # Connect to Redis using the provided URL
-        self.redis = redis.StrictRedis.from_url(redis_url, decode_responses=True)
-        self.task_key = task_key
-        self.lock_key = lock_key
-
-    def acquire_lock(self, timeout=5):
-        """Acquire a lock using Redis."""
-        while True:
-            if self.redis.set(self.lock_key, "locked", nx=True, ex=timeout):
-                return True
-            time.sleep(0.1)
-
-    def release_lock(self):
-        """Release the lock in Redis."""
-        self.redis.delete(self.lock_key)
-
-    def set_task_count(self, count):
-        """Initialize the task counter in Redis."""
-        self.redis.set(self.task_key, count)
-
-    def decrement_task_count(self):
-        """Decrement the task counter in Redis."""
-        return self.redis.decr(self.task_key)
-
-    def task_exists(self):
-        return self.redis.exists(self.task_key)
-
-    def get_task_count(self):
-        """Get the current value of the task counter."""
-        return int(self.redis.get(self.task_key) or 0)