scalable-pypeline 2.1.24__py2.py3-none-any.whl → 2.1.26__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.24"
+__version__ = "2.1.26"

pypeline/barrier.py

@@ -2,7 +2,7 @@ import time
 import redis
 from redis.sentinel import Sentinel
 from urllib.parse import urlparse
-from constants import (
+from pypeline.constants import (
     REDIS_SENTINEL_MASTER_NAME,
     DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
     DEFAULT_REDIS_SOCKET_TIMEOUT,

pypeline/pipelines/composition/parallel_pipeline_composition.py

@@ -5,12 +5,22 @@ import json
 import time
 import typing
 from uuid import uuid4
+from urllib.parse import urlparse
 
 from dramatiq.broker import get_broker
 from dramatiq.results import ResultMissing
 from db_medley.redis_conf import RedisConnector
 from redis.exceptions import RedisError
-
+from redis.sentinel import Sentinel
+from pypeline.constants import (
+    REDIS_URL,
+    REDIS_SENTINEL_MASTER_NAME,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+)
 from pypeline.barrier import LockingParallelBarrier
 from pypeline.constants import DEFAULT_RESULT_TTL
 from pypeline.dramatiq import REDIS_URL
@@ -240,9 +250,26 @@ class PipelineResult:
         self.pipeline: parallel_pipeline = None
         self.execution_id = execution_id
         self.redis_key = f"{execution_id}-results-key"
-        self.redis_conn = RedisConnector().get_connection()
         self.result_ttl = result_ttl
 
+        if REDIS_SENTINEL_MASTER_NAME is not None:
+            parsed_redis_url = urlparse(REDIS_URL)
+            redis_sentinel = Sentinel(
+                sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+            )
+            self.redis_conn = redis_sentinel.master_for(
+                REDIS_SENTINEL_MASTER_NAME,
+                db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+                password=parsed_redis_url.password,
+                socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+                socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+                socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+                health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+            )
+        else:
+            self.redis_conn = RedisConnector().get_connection()
+
     def create_result_entry(self, pipeline_json_str: str):
         """
         Store the serialized pipeline data in Redis with a specified TTL.

{scalable_pypeline-2.1.24.dist-info → scalable_pypeline-2.1.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.24
+Version: 2.1.26
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable_pypeline-2.1.24.dist-info → scalable_pypeline-2.1.26.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pypeline/__init__.py,sha256=qc6AJ16ihdkGGzdf0nDZHdeSsAAbLwShB1HyYj8ASPE,23
-pypeline/barrier.py,sha256=lIWKikEtE34PQf4ZfSl6RNBu0n5vP8x-oy2Pe2S0tqM,2367
+pypeline/__init__.py,sha256=kDsQujSOEts9DhAOHBOlEjAEDf0lqYMUYrA9xzjlvh0,23
+pypeline/barrier.py,sha256=ojSgbuZnGKpKiSBYXTV4CxG9j1Z01YdzBSORli4MnzI,2376
 pypeline/constants.py,sha256=7COt9jfmLDvCNAFeN6ddRpwdvv2LpbYOCIQs6dPXpOQ,3592
 pypeline/dramatiq.py,sha256=XPpgPgiOaEFK8zORx9eveJ45wzcUXMjVGryFKY5Xiwg,15527
 pypeline/executable_job_config_schema.py,sha256=P2Z8SO057Jgyt4I5oZxcbEi1iaZkLoAh7qp8PtuqcqU,1010
@@ -17,7 +17,7 @@ pypeline/flask/api/schedules.py,sha256=8PKCMdPucaer8opchNlI5aDssK2UqT79hHpeg5BMt
 pypeline/pipelines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/pipelines/factory.py,sha256=ftm6AwXIdkC8LEzy3c4CLPzXtoQonBKUo3yojbYTsUE,3309
 pypeline/pipelines/composition/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/pipelines/composition/parallel_pipeline_composition.py,sha256=pTw9Xb9h4JnV4siFc3JStm5lB-i9djUADo3Kh5K3s7g,12976
+pypeline/pipelines/composition/parallel_pipeline_composition.py,sha256=JrEsIxaB36wLWg5ztCMkANye9VizGuRIhyxyCP4DCl8,14162
 pypeline/pipelines/composition/pypeline_composition.py,sha256=lY1RO7luLqDBLfoX99YbBatx1Qov1cxGoS5lRAjB_DQ,8534
 pypeline/pipelines/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/pipelines/middleware/get_active_worker_id_middleware.py,sha256=X4ZfRk3L8MD00DTsGHth7oOdy-W7LQV96T8vu5UC42A,755
@@ -33,9 +33,9 @@ pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdV
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.24.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.24.dist-info/METADATA,sha256=Ai98ywPiFPQ01sTYY-H1HV_XcoHQH6R6UYwMVm1_iLw,5985
-scalable_pypeline-2.1.24.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.24.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.24.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.24.dist-info/RECORD,,
+scalable_pypeline-2.1.26.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.26.dist-info/METADATA,sha256=T-yqcbiyL6nJ6SvJkmrH9n0lPvXjdJ9m4QLny-VPHyU,5985
+scalable_pypeline-2.1.26.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.26.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.1.26.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.26.dist-info/RECORD,,