scalable-pypeline 2.1.22__py2.py3-none-any.whl → 2.1.23__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.22"
+__version__ = "2.1.23"

pypeline/constants.py

@@ -1,5 +1,4 @@
-""" Pypeline Constants
-"""
+"""Pypeline Constants"""
 
 import os
 
@@ -15,6 +14,7 @@ DEFAULT_BROKER_CALLABLE = os.environ.get(
 # Pypeline broker connections
 RABBIT_URL = os.environ.get("RABBIT_URL", "amqp://admin:password@127.0.0.1:5672")
 REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
+REDIS_SENTINAL_MASTER_NAME = os.environ.get("REDIS_SENTINAL_MASTER_NAME", None)
 
 # Pypeline task defaults
 PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL = int(
@@ -39,6 +39,16 @@ DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT = int(
 DEFAULT_BROKER_HEARTBEAT_TIMEOUT = int(
     os.getenv("DEFAULT_BROKER_HEARTBEAT_TIMEOUT", 300000)
 )
+DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT = int(
+    os.getenv("DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT", 1)
+)
+DEFAULT_REDIS_SOCKET_TIMEOUT = int(os.getenv("DEFAULT_REDIS_SOCKET_TIMEOUT", 2))
+DEFAULT_REDIS_RETRY_ON_TIMEOUT = bool(os.getenv("DEFAULT_REDIS_RETRY_ON_TIMEOUT", True))
+DEFAULT_REDIS_SOCKET_KEEPALIVE = bool(os.getenv("DEFAULT_REDIS_SOCKET_KEEPALIVE", True))
+DEFAULT_REDIS_HEALTH_CHECK_INTERVAL = int(
+    os.getenv("DEFAULT_REDIS_HEALTH_CHECK_INTERVAL", 30)
+)
+
 MESSAGE_BROKER = os.getenv("MESSAGE_BROKER", "RABBITMQ")
 
 MS_IN_SECONDS = 1000

pypeline/dramatiq.py

@@ -8,7 +8,7 @@ import click
 from urllib.parse import urlparse
 
 from dramatiq.brokers.redis import RedisBroker
-
+from redis.sentinel import Sentinel
 from pypeline.extensions import pypeline_config
 from warnings import warn
 from apscheduler.schedulers.blocking import BlockingScheduler
@@ -30,6 +30,7 @@ from flask.cli import with_appcontext
 
 from pypeline.constants import (
     REDIS_URL,
+    REDIS_SENTINAL_MASTER_NAME,
     RABBIT_URL,
     DEFAULT_BROKER_CALLABLE,
     DEFAULT_BROKER_CONNECTION_HEARTBEAT,
@@ -37,6 +38,11 @@ from pypeline.constants import (
     DEFAULT_BROKER_CONNECTION_ATTEMPTS,
     MESSAGE_BROKER,
     DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
 )
 from pypeline.pipelines.middleware.get_active_worker_id_middleware import (
     GetActiveWorkerIdMiddleware,
@@ -45,7 +51,8 @@ from pypeline.pipelines.middleware.parallel_pipeline_middleware import ParallelP
 from pypeline.pipelines.middleware.pypeline_middleware import PypelineMiddleware
 from pypeline.utils.config_utils import (
     retrieve_latest_schedule_config,
-    get_service_config_for_worker, retrieve_executable_job_config,
+    get_service_config_for_worker,
+    retrieve_executable_job_config,
 )
 from pypeline.utils.dramatiq_utils import (
     guess_code_directory,
@@ -62,7 +69,23 @@ logger = logging.getLogger(__name__)
 
 
 def configure_default_broker(broker: Broker = None):
-    redis_backend = RedisBackend(url=REDIS_URL)
+    redis_client = None
+    if REDIS_SENTINAL_MASTER_NAME is not None:
+        parsed_redis_url = urlparse(REDIS_URL)
+        redis_sentinel = Sentinel(
+            sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+        )
+        redis_client = redis_sentinel.master_for(
+            REDIS_SENTINAL_MASTER_NAME,
+            db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+            password=parsed_redis_url.password,
+            socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+            socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+            retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+            socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+            health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+        )
+    redis_backend = RedisBackend(client=redis_client, url=REDIS_URL)
 
     if MESSAGE_BROKER == "RABBITMQ":
         parsed_url = urlparse(RABBIT_URL)
@@ -85,7 +108,9 @@ def configure_default_broker(broker: Broker = None):
             broker
             if broker is not None
             else RedisBroker(
-                url=REDIS_URL, heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT
+                client=redis_client,
+                url=REDIS_URL,
+                heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
             )
         )
 
@@ -169,7 +194,6 @@ def register_actors_for_workers(broker: Broker):
                 logger.exception(f"Unable to add a task to dramatiq: {e}")
 
 
-
 class Dramatiq:
     """Flask extension bridging Dramatiq broker and Flask app.
 

pypeline/utils/dramatiq_utils.py

@@ -113,11 +113,15 @@ class LazyActor(object):
         return getattr(self.actor, name)
 
     def register(self, broker):
-        self.actor = register_actor(
-            actor_name=f"{self.fn.__module__}.{self.fn.__name__}-{self.kw['queue_name']}",
-            broker=broker,
-            **self.kw,
-        )(ensure_return_value(default_value=True)(self.fn))
+        actor_name = f"{self.fn.__module__}.{self.fn.__name__}-{self.kw['queue_name']}"
+        if actor_name in broker.actors:
+            self.actor = broker.actors[actor_name]
+        else:
+            self.actor = register_actor(
+                actor_name=actor_name,
+                broker=broker,
+                **self.kw,
+            )(ensure_return_value(default_value=True)(self.fn))
 
     # Next is regular actor API.
     def send(self, *a, **kw):

{scalable_pypeline-2.1.22.dist-info → scalable_pypeline-2.1.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.22
+Version: 2.1.23
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation
@@ -25,7 +25,7 @@ Requires-Dist: pytest-cov (<3,>=2.6.1) ; extra == 'test'
 Requires-Dist: tox (<4,>=3.14.1) ; extra == 'test'
 Requires-Dist: mock (<2,>=1) ; extra == 'test'
 Requires-Dist: responses (<0.11,>=0.10.16) ; extra == 'test'
-Requires-Dist: fakeredis (<3,>=2.10.3) ; extra == 'test'
+Requires-Dist: fakeredis (<2.31,>=2.10.3) ; extra == 'test'
 Provides-Extra: web
 Requires-Dist: gunicorn ; extra == 'web'
 Requires-Dist: gevent (<22,>=21.12.0) ; extra == 'web'

{scalable_pypeline-2.1.22.dist-info → scalable_pypeline-2.1.23.dist-info}/RECORD

@@ -1,7 +1,7 @@
-pypeline/__init__.py,sha256=8X_w-VRCyU1lQMWejIqkf5tmhnrgO7csLi27xV88LJs,23
+pypeline/__init__.py,sha256=v7ybXce2sltwQL38gMDsgF0NlBfnohuHthBrSTULxMw,23
 pypeline/barrier.py,sha256=oO964l9qOCOibweOHyNivmAvufdXOke9nz2tdgclouo,1172
-pypeline/constants.py,sha256=EGSuLq4KhZ4bxrbtnUgKclELRyya5ipvv0WeybCzNAs,3049
-pypeline/dramatiq.py,sha256=S_rZMpGgE-VAmx9YjDBkHpSjFdhwp7mz4uuuKBUx4iQ,14405
+pypeline/constants.py,sha256=SLYWQ56zMQI75Imijw1z33bugcmJ5zCYfhc6uTb-w1I,3592
+pypeline/dramatiq.py,sha256=ElqzpLDuNOCNrW2aw3Ml1p6_0AnwOmQ5jcl5GQ6JuFM,15527
 pypeline/executable_job_config_schema.py,sha256=P2Z8SO057Jgyt4I5oZxcbEi1iaZkLoAh7qp8PtuqcqU,1010
 pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
 pypeline/pipeline_config_schema.py,sha256=kRZcCMlk2FIITDzVrAfcSmHnxi1mIWmDzasTW0TnaAU,11169
@@ -26,16 +26,16 @@ pypeline/pipelines/middleware/parallel_pipeline_middleware.py,sha256=kTp6niYoe2n
 pypeline/pipelines/middleware/pypeline_middleware.py,sha256=AGQCRmqXcYNyCxinwtGZplHwltrmZ_sH73CVRLu_0EE,8961
 pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/utils/config_utils.py,sha256=bblh8clRDDNQpQIDkBrtskZBo-csakoO1IJiaqVGyr8,2508
-pypeline/utils/dramatiq_utils.py,sha256=DUdgVywm1182A4i69XzH9EIh1EJ9zAHmJLtOaVSW7pw,3844
+pypeline/utils/dramatiq_utils.py,sha256=tbG3o5FD0zUOKtJJJECE2cM2ovDv3OLQ5CplJ9cXfM4,4001
 pypeline/utils/executable_job_util.py,sha256=Id4FYvCdzqOCzQYSoU2yeOncubKwSwITUf_BgIPZQUw,1206
 pypeline/utils/graceful_shutdown_util.py,sha256=gdyf4r_vlunZZP-prcol_9aJQEG9Hk8evivVSe_H0N0,1293
 pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdVU,3682
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.22.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.22.dist-info/METADATA,sha256=j5IwcFIS2j3cTjCEU8uE9h3UYyC1rzkZT7v_SYc7dis,5982
-scalable_pypeline-2.1.22.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.22.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.22.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.22.dist-info/RECORD,,
+scalable_pypeline-2.1.23.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.23.dist-info/METADATA,sha256=OdNeYdHW9jO0ILRNPcCWM-yG7CQn-GEawq02AfxUP1w,5985
+scalable_pypeline-2.1.23.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.23.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.1.23.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.23.dist-info/RECORD,,