scalable-pypeline 2.1.25__tar.gz → 2.1.27__tar.gz

{scalable-pypeline-2.1.25/scalable_pypeline.egg-info → scalable-pypeline-2.1.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.25
+Version: 2.1.27
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

scalable-pypeline-2.1.27/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.1.27"

{scalable-pypeline-2.1.25 → scalable-pypeline-2.1.27}/pypeline/pipeline_settings_schema.py

@@ -465,6 +465,7 @@ class PipelineScenarioSchema(Schema):
             "produce alternative calculations and or results."
         },
     )
+    execution_id = fields.String(required=False, metadata={"description":"Execution id for a known scenario"})
 
 
 class PipelineScenariosSchema(Schema):

{scalable-pypeline-2.1.25 → scalable-pypeline-2.1.27}/pypeline/pipelines/composition/parallel_pipeline_composition.py

@@ -5,12 +5,22 @@ import json
 import time
 import typing
 from uuid import uuid4
+from urllib.parse import urlparse
 
 from dramatiq.broker import get_broker
 from dramatiq.results import ResultMissing
 from db_medley.redis_conf import RedisConnector
 from redis.exceptions import RedisError
-
+from redis.sentinel import Sentinel
+from pypeline.constants import (
+    REDIS_URL,
+    REDIS_SENTINEL_MASTER_NAME,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+)
 from pypeline.barrier import LockingParallelBarrier
 from pypeline.constants import DEFAULT_RESULT_TTL
 from pypeline.dramatiq import REDIS_URL
@@ -240,9 +250,26 @@ class PipelineResult:
         self.pipeline: parallel_pipeline = None
         self.execution_id = execution_id
         self.redis_key = f"{execution_id}-results-key"
-        self.redis_conn = RedisConnector().get_connection()
         self.result_ttl = result_ttl
 
+        if REDIS_SENTINEL_MASTER_NAME is not None:
+            parsed_redis_url = urlparse(REDIS_URL)
+            redis_sentinel = Sentinel(
+                sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+            )
+            self.redis_conn = redis_sentinel.master_for(
+                REDIS_SENTINEL_MASTER_NAME,
+                db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+                password=parsed_redis_url.password,
+                socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+                socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+                socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+                health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+            )
+        else:
+            self.redis_conn = RedisConnector().get_connection()
+
     def create_result_entry(self, pipeline_json_str: str):
         """
         Store the serialized pipeline data in Redis with a specified TTL.

{scalable-pypeline-2.1.25 → scalable-pypeline-2.1.27}/pypeline/pipelines/middleware/pypeline_middleware.py

@@ -76,7 +76,13 @@ class PypelineMiddleware(Middleware):
                 # If the lock didn't exist for the current tasks execution id then it would indicate
                 # that this is the start of a new scenario.  Therefore we need to find the ancestor
                 # that is executed in the base case execution id and make sure it has completed
-                if remaining_tasks is None:
+                tasks_to_run_in_scenario = None
+
+                for scenario in message.options["scenarios"]:
+                    if scenario["execution_id"] == execution_id:
+                        tasks_to_run_in_scenario = scenario["tasksToRunInScenario"]
+
+                if ancestor not in tasks_to_run_in_scenario and remaining_tasks is None:
                     ancestor_task_key = (
                         f"{message.options['base_case_execution_id']}-{ancestor}"
                     )
@@ -95,15 +101,17 @@ class PypelineMiddleware(Middleware):
                             remaining_tasks = locking_parallel_barrier.get_task_count()
                     finally:
                         locking_parallel_barrier.release_lock()
+
+
                 if remaining_tasks is None or remaining_tasks >= 1:
                     ancestor_tasks_complete = False
                     break
 
             # If the child's ancestor tasks aren't complete move onto the next child to check
             if not ancestor_tasks_complete:
-                break
+                continue
 
-            # Handle situation where base case kicks off new scenario
+            # Handle situation where base case kicks off new scenario.
             if (
                 message.options["base_case_execution_id"]
                 == message.options["execution_id"]
@@ -160,7 +168,7 @@ class PypelineMiddleware(Middleware):
                             )
                         messages.append(scenario_message)
 
-            # Kick off child task for current scenario
+            # If we've made it here all ancestors of this child are complete and it's time to run.
             task_key = f"{execution_id}-{child}"
             locking_parallel_barrier = LockingParallelBarrier(
                 self.redis_url,

{scalable-pypeline-2.1.25 → scalable-pypeline-2.1.27/scalable_pypeline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.25
+Version: 2.1.27
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

scalable-pypeline-2.1.25/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.1.25"