scalable-pypeline 2.1.26__tar.gz → 2.1.28__tar.gz

{scalable-pypeline-2.1.26/scalable_pypeline.egg-info → scalable-pypeline-2.1.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.26
+Version: 2.1.28
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

scalable-pypeline-2.1.28/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.1.28"

{scalable-pypeline-2.1.26 → scalable-pypeline-2.1.28}/pypeline/pipeline_settings_schema.py

@@ -465,6 +465,7 @@ class PipelineScenarioSchema(Schema):
             "produce alternative calculations and or results."
         },
     )
+    execution_id = fields.String(required=False, metadata={"description":"Execution id for a known scenario"})
 
 
 class PipelineScenariosSchema(Schema):

{scalable-pypeline-2.1.26 → scalable-pypeline-2.1.28}/pypeline/pipelines/middleware/pypeline_middleware.py

@@ -46,74 +46,74 @@ class PypelineMiddleware(Middleware):
 
         graph = get_execution_graph(pipeline_config)
         children_tasks = pipeline_config["dagAdjacency"].get(task_name, [])
-
         messages = []
         for child in children_tasks:
             child_ancestors = sorted(graph.predecessors(child))
+            child_ancestors_complete = {a: False for a in child_ancestors}
 
-            ancestor_tasks_complete = True
+            for scenario in message.options["scenarios"]:
+                if scenario["execution_id"] == execution_id:
+                    tasks_to_run_in_scenario = scenario["tasksToRunInScenario"]
 
             for ancestor in child_ancestors:
-                ancestor_task_key = f"{execution_id}-{ancestor}"
-
-                locking_parallel_barrier = LockingParallelBarrier(
-                    self.redis_url,
-                    task_key=ancestor_task_key,
-                    lock_key=f"{message.options['base_case_execution_id']}-lock",
-                )
-                try:
-                    locking_parallel_barrier.acquire_lock(
-                        timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+                if ancestor in tasks_to_run_in_scenario:
+                    current_scenario_ancestor_task_key = f"{execution_id}-{ancestor}"
+                    locking_parallel_barrier = LockingParallelBarrier(
+                        self.redis_url,
+                        task_key=current_scenario_ancestor_task_key,
+                        lock_key=f"{message.options['base_case_execution_id']}-lock",
                     )
-
-                    if locking_parallel_barrier.task_exists():
-                        remaining_tasks = locking_parallel_barrier.get_task_count()
-                    else:
-                        remaining_tasks = None
-                finally:
-                    locking_parallel_barrier.release_lock()
-
-                # If the lock didn't exist for the current tasks execution id then it would indicate
-                # that this is the start of a new scenario.  Therefore we need to find the ancestor
-                # that is executed in the base case execution id and make sure it has completed
-                if remaining_tasks is None:
-                    ancestor_task_key = (
+                    try:
+                        locking_parallel_barrier.acquire_lock(
+                            timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+                        )
+                        if not locking_parallel_barrier.task_exists():
+                            child_ancestors_complete[ancestor] = False
+                        elif locking_parallel_barrier.get_task_count() <= 0:
+                            child_ancestors_complete[ancestor] = True
+                    finally:
+                        locking_parallel_barrier.release_lock()
+                else:
+                    base_scenario_ancestor_task_key = (
                         f"{message.options['base_case_execution_id']}-{ancestor}"
                     )
-
                     locking_parallel_barrier = LockingParallelBarrier(
                         self.redis_url,
-                        task_key=ancestor_task_key,
+                        task_key=base_scenario_ancestor_task_key,
                         lock_key=f"{message.options['base_case_execution_id']}-lock",
                     )
                     try:
                         locking_parallel_barrier.acquire_lock(
                             timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
                         )
-
-                        if locking_parallel_barrier.task_exists():
-                            remaining_tasks = locking_parallel_barrier.get_task_count()
+                        if not locking_parallel_barrier.task_exists():
+                            child_ancestors_complete[ancestor] = False
+                        elif locking_parallel_barrier.get_task_count() <= 0:
+                            child_ancestors_complete[ancestor] = True
                     finally:
                         locking_parallel_barrier.release_lock()
-                if remaining_tasks is None or remaining_tasks >= 1:
-                    ancestor_tasks_complete = False
-                    break
 
-            # If the child's ancestor tasks aren't complete move onto the next child to check
-            if not ancestor_tasks_complete:
-                break
+            if any(complete is False for complete in child_ancestors_complete.values()):
+                continue
 
-            # Handle situation where base case kicks off new scenario
             if (
                 message.options["base_case_execution_id"]
                 == message.options["execution_id"]
             ):
                 for scenario in message.options["scenarios"]:
-                    child_predecessors = list(graph.predecessors(child))
+                    child_ancestors = list(graph.predecessors(child))
+                    child_has_other_ancestors_in_scenario = False
+
+                    for ancestor in child_ancestors:
+                        if ancestor in scenario["tasksToRunInScenario"]:
+                            child_has_other_ancestors_in_scenario = True
+                            break
+
                     if (
                         child in scenario["tasksToRunInScenario"]
-                        and task_name in child_predecessors
+                        and task_name in child_ancestors
                         and task_name not in scenario["tasksToRunInScenario"]
+                        and not child_has_other_ancestors_in_scenario
                     ):
                         task_key = f"{scenario['execution_id']}-{child}"
                         locking_parallel_barrier = LockingParallelBarrier(
@@ -160,7 +160,7 @@ class PypelineMiddleware(Middleware):
                             )
                         messages.append(scenario_message)
 
-            # Kick off child task for current scenario
+            # If we've made it here all ancestors of this child are complete, and it's time to run.
             task_key = f"{execution_id}-{child}"
             locking_parallel_barrier = LockingParallelBarrier(
                 self.redis_url,

{scalable-pypeline-2.1.26 → scalable-pypeline-2.1.28/scalable_pypeline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.26
+Version: 2.1.28
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

scalable-pypeline-2.1.26/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.1.26"