scalable-pypeline 2.1.23__py2.py3-none-any.whl → 2.1.25__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.23"
+__version__ = "2.1.25"

pypeline/barrier.py

@@ -1,12 +1,38 @@
 import time
-
 import redis
+from redis.sentinel import Sentinel
+from urllib.parse import urlparse
+from pypeline.constants import (
+    REDIS_SENTINEL_MASTER_NAME,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+)
 
 
 class LockingParallelBarrier:
     def __init__(self, redis_url, task_key="task_counter", lock_key="task_lock"):
         # Connect to Redis using the provided URL
-        self.redis = redis.StrictRedis.from_url(redis_url, decode_responses=True)
+        if REDIS_SENTINEL_MASTER_NAME is not None:
+            parsed_redis_url = urlparse(redis_url)
+            redis_sentinel = Sentinel(
+                sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+            )
+            self.redis = redis_sentinel.master_for(
+                REDIS_SENTINEL_MASTER_NAME,
+                db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+                password=parsed_redis_url.password,
+                socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+                socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+                socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+                health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+                decode_responses=True,
+            )
+        else:
+            self.redis = redis.StrictRedis.from_url(redis_url, decode_responses=True)
         self.task_key = task_key
         self.lock_key = lock_key
 

pypeline/constants.py

@@ -14,7 +14,7 @@ DEFAULT_BROKER_CALLABLE = os.environ.get(
 # Pypeline broker connections
 RABBIT_URL = os.environ.get("RABBIT_URL", "amqp://admin:password@127.0.0.1:5672")
 REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
-REDIS_SENTINAL_MASTER_NAME = os.environ.get("REDIS_SENTINAL_MASTER_NAME", None)
+REDIS_SENTINEL_MASTER_NAME = os.environ.get("REDIS_SENTINEL_MASTER_NAME", None)
 
 # Pypeline task defaults
 PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL = int(

pypeline/dramatiq.py

@@ -30,7 +30,7 @@ from flask.cli import with_appcontext
 
 from pypeline.constants import (
     REDIS_URL,
-    REDIS_SENTINAL_MASTER_NAME,
+    REDIS_SENTINEL_MASTER_NAME,
     RABBIT_URL,
     DEFAULT_BROKER_CALLABLE,
     DEFAULT_BROKER_CONNECTION_HEARTBEAT,
@@ -70,13 +70,13 @@ logger = logging.getLogger(__name__)
 
 def configure_default_broker(broker: Broker = None):
     redis_client = None
-    if REDIS_SENTINAL_MASTER_NAME is not None:
+    if REDIS_SENTINEL_MASTER_NAME is not None:
         parsed_redis_url = urlparse(REDIS_URL)
         redis_sentinel = Sentinel(
             sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
         )
         redis_client = redis_sentinel.master_for(
-            REDIS_SENTINAL_MASTER_NAME,
+            REDIS_SENTINEL_MASTER_NAME,
             db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
             password=parsed_redis_url.password,
             socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,

{scalable_pypeline-2.1.23.dist-info → scalable_pypeline-2.1.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.23
+Version: 2.1.25
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable_pypeline-2.1.23.dist-info → scalable_pypeline-2.1.25.dist-info}/RECORD

@@ -1,7 +1,7 @@
-pypeline/__init__.py,sha256=v7ybXce2sltwQL38gMDsgF0NlBfnohuHthBrSTULxMw,23
-pypeline/barrier.py,sha256=oO964l9qOCOibweOHyNivmAvufdXOke9nz2tdgclouo,1172
-pypeline/constants.py,sha256=SLYWQ56zMQI75Imijw1z33bugcmJ5zCYfhc6uTb-w1I,3592
-pypeline/dramatiq.py,sha256=ElqzpLDuNOCNrW2aw3Ml1p6_0AnwOmQ5jcl5GQ6JuFM,15527
+pypeline/__init__.py,sha256=eNwueZdE6xOPV8_Xyr7CpRZhy-MW6j5G9dkoGmwRE54,23
+pypeline/barrier.py,sha256=ojSgbuZnGKpKiSBYXTV4CxG9j1Z01YdzBSORli4MnzI,2376
+pypeline/constants.py,sha256=7COt9jfmLDvCNAFeN6ddRpwdvv2LpbYOCIQs6dPXpOQ,3592
+pypeline/dramatiq.py,sha256=XPpgPgiOaEFK8zORx9eveJ45wzcUXMjVGryFKY5Xiwg,15527
 pypeline/executable_job_config_schema.py,sha256=P2Z8SO057Jgyt4I5oZxcbEi1iaZkLoAh7qp8PtuqcqU,1010
 pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
 pypeline/pipeline_config_schema.py,sha256=kRZcCMlk2FIITDzVrAfcSmHnxi1mIWmDzasTW0TnaAU,11169
@@ -33,9 +33,9 @@ pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdV
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.23.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.23.dist-info/METADATA,sha256=OdNeYdHW9jO0ILRNPcCWM-yG7CQn-GEawq02AfxUP1w,5985
-scalable_pypeline-2.1.23.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.23.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.23.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.23.dist-info/RECORD,,
+scalable_pypeline-2.1.25.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.25.dist-info/METADATA,sha256=QMI9GTwNr1B_dg7P0Fk64PpWDZzJj_BVXPrCrJ8wOBA,5985
+scalable_pypeline-2.1.25.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.25.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.1.25.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.25.dist-info/RECORD,,