scalable-pypeline 2.1.27__py2.py3-none-any.whl → 2.1.28__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.27"
+__version__ = "2.1.28"

pypeline/pipelines/middleware/pypeline_middleware.py

@@ -46,82 +46,74 @@ class PypelineMiddleware(Middleware):
 
         graph = get_execution_graph(pipeline_config)
         children_tasks = pipeline_config["dagAdjacency"].get(task_name, [])
-
         messages = []
         for child in children_tasks:
             child_ancestors = sorted(graph.predecessors(child))
+            child_ancestors_complete = {a: False for a in child_ancestors}
 
-            ancestor_tasks_complete = True
+            for scenario in message.options["scenarios"]:
+                if scenario["execution_id"] == execution_id:
+                    tasks_to_run_in_scenario = scenario["tasksToRunInScenario"]
 
             for ancestor in child_ancestors:
-                ancestor_task_key = f"{execution_id}-{ancestor}"
-
-                locking_parallel_barrier = LockingParallelBarrier(
-                    self.redis_url,
-                    task_key=ancestor_task_key,
-                    lock_key=f"{message.options['base_case_execution_id']}-lock",
-                )
-                try:
-                    locking_parallel_barrier.acquire_lock(
-                        timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+                if ancestor in tasks_to_run_in_scenario:
+                    current_scenario_ancestor_task_key = f"{execution_id}-{ancestor}"
+                    locking_parallel_barrier = LockingParallelBarrier(
+                        self.redis_url,
+                        task_key=current_scenario_ancestor_task_key,
+                        lock_key=f"{message.options['base_case_execution_id']}-lock",
                     )
-
-                    if locking_parallel_barrier.task_exists():
-                        remaining_tasks = locking_parallel_barrier.get_task_count()
-                    else:
-                        remaining_tasks = None
-                finally:
-                    locking_parallel_barrier.release_lock()
-
-                # If the lock didn't exist for the current tasks execution id then it would indicate
-                # that this is the start of a new scenario.  Therefore we need to find the ancestor
-                # that is executed in the base case execution id and make sure it has completed
-                tasks_to_run_in_scenario = None
-
-                for scenario in message.options["scenarios"]:
-                    if scenario["execution_id"] == execution_id:
-                        tasks_to_run_in_scenario = scenario["tasksToRunInScenario"]
-
-                if ancestor not in tasks_to_run_in_scenario and remaining_tasks is None:
-                    ancestor_task_key = (
+                    try:
+                        locking_parallel_barrier.acquire_lock(
+                            timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+                        )
+                        if not locking_parallel_barrier.task_exists():
+                            child_ancestors_complete[ancestor] = False
+                        elif locking_parallel_barrier.get_task_count() <= 0:
+                            child_ancestors_complete[ancestor] = True
+                    finally:
+                        locking_parallel_barrier.release_lock()
+                else:
+                    base_scenario_ancestor_task_key = (
                         f"{message.options['base_case_execution_id']}-{ancestor}"
                     )
-
                     locking_parallel_barrier = LockingParallelBarrier(
                         self.redis_url,
-                        task_key=ancestor_task_key,
+                        task_key=base_scenario_ancestor_task_key,
                         lock_key=f"{message.options['base_case_execution_id']}-lock",
                     )
                     try:
                         locking_parallel_barrier.acquire_lock(
                             timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
                         )
-
-                        if locking_parallel_barrier.task_exists():
-                            remaining_tasks = locking_parallel_barrier.get_task_count()
+                        if not locking_parallel_barrier.task_exists():
+                            child_ancestors_complete[ancestor] = False
+                        elif locking_parallel_barrier.get_task_count() <= 0:
+                            child_ancestors_complete[ancestor] = True
                     finally:
                         locking_parallel_barrier.release_lock()
 
-
-                if remaining_tasks is None or remaining_tasks >= 1:
-                    ancestor_tasks_complete = False
-                    break
-
-            # If the child's ancestor tasks aren't complete move onto the next child to check
-            if not ancestor_tasks_complete:
+            if any(complete is False for complete in child_ancestors_complete.values()):
                 continue
 
-            # Handle situation where base case kicks off new scenario.
             if (
                 message.options["base_case_execution_id"]
                 == message.options["execution_id"]
             ):
                 for scenario in message.options["scenarios"]:
-                    child_predecessors = list(graph.predecessors(child))
+                    child_ancestors = list(graph.predecessors(child))
+                    child_has_other_ancestors_in_scenario = False
+
+                    for ancestor in child_ancestors:
+                        if ancestor in scenario["tasksToRunInScenario"]:
+                            child_has_other_ancestors_in_scenario = True
+                            break
+
                     if (
                         child in scenario["tasksToRunInScenario"]
-                        and task_name in child_predecessors
+                        and task_name in child_ancestors
                         and task_name not in scenario["tasksToRunInScenario"]
+                        and not child_has_other_ancestors_in_scenario
                     ):
                         task_key = f"{scenario['execution_id']}-{child}"
                         locking_parallel_barrier = LockingParallelBarrier(
@@ -168,7 +160,7 @@ class PypelineMiddleware(Middleware):
                             )
                         messages.append(scenario_message)
 
-            # If we've made it here all ancestors of this child are complete and it's time to run.
+            # If we've made it here all ancestors of this child are complete, and it's time to run.
             task_key = f"{execution_id}-{child}"
             locking_parallel_barrier = LockingParallelBarrier(
                 self.redis_url,

{scalable_pypeline-2.1.27.dist-info → scalable_pypeline-2.1.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.27
+Version: 2.1.28
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable_pypeline-2.1.27.dist-info → scalable_pypeline-2.1.28.dist-info}/RECORD

@@ -1,4 +1,4 @@
-pypeline/__init__.py,sha256=nRu7ZwJqfHD5II0bBxcWlJ8czOdWw1bjjPVhymDs-NE,23
+pypeline/__init__.py,sha256=uJ5T75rsS9yyIgD9aFR6H760hkKMyujptvhmniMPJSY,23
 pypeline/barrier.py,sha256=ojSgbuZnGKpKiSBYXTV4CxG9j1Z01YdzBSORli4MnzI,2376
 pypeline/constants.py,sha256=7COt9jfmLDvCNAFeN6ddRpwdvv2LpbYOCIQs6dPXpOQ,3592
 pypeline/dramatiq.py,sha256=XPpgPgiOaEFK8zORx9eveJ45wzcUXMjVGryFKY5Xiwg,15527
@@ -23,7 +23,7 @@ pypeline/pipelines/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
 pypeline/pipelines/middleware/get_active_worker_id_middleware.py,sha256=X4ZfRk3L8MD00DTsGHth7oOdy-W7LQV96T8vu5UC42A,755
 pypeline/pipelines/middleware/graceful_shutdown_middleware.py,sha256=k37zmFk9dOye05BoQP7KcB9MEQgvodI16kOJyYhRyAc,1764
 pypeline/pipelines/middleware/parallel_pipeline_middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
-pypeline/pipelines/middleware/pypeline_middleware.py,sha256=tnQcewRCCaQaNMTx9Kz0gx47YZxBJCDW9UH_8cBLlwY,9317
+pypeline/pipelines/middleware/pypeline_middleware.py,sha256=FjREuPDdTfeYOVGVUjJgx8Szh6yu7g8OnHRc5N__448,9385
 pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/utils/config_utils.py,sha256=bblh8clRDDNQpQIDkBrtskZBo-csakoO1IJiaqVGyr8,2508
 pypeline/utils/dramatiq_utils.py,sha256=tbG3o5FD0zUOKtJJJECE2cM2ovDv3OLQ5CplJ9cXfM4,4001
@@ -33,9 +33,9 @@ pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdV
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.27.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.27.dist-info/METADATA,sha256=D-3zqFydd6j3FTAvnz1VSjQBobpx7XcR_Oz4J_-2BU8,5985
-scalable_pypeline-2.1.27.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.27.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.27.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.27.dist-info/RECORD,,
+scalable_pypeline-2.1.28.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.28.dist-info/METADATA,sha256=f0Ib-ADg8STVoxyrTo1PCy0KbOsGW9PB-jBeo5keGF8,5985
+scalable_pypeline-2.1.28.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.28.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.1.28.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.28.dist-info/RECORD,,