scalable-pypeline 2.1.10__py2.py3-none-any.whl → 2.1.12__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.10"
+__version__ = "2.1.12"

pypeline/pipeline_config_schema.py

@@ -120,8 +120,8 @@ class TaskDefinitionsSchemaV1(ExcludeUnknownSchema):
     serverType = fields.String(
         required=False,
         validate=OneOf(
-            ["xs", "s", "m", "l", "xl", "xxl", "xxxl"],
-            error="Invalid serverType. Available options: 'xs', 's', 'm', 'l', 'xl', 'xxl', 'xxxl'.",
+            ["xs", "s", "m", "l", "xl", "xxl", "xxxl", "cpu-xl"],
+            error="Invalid serverType. Available options: 'xs', 's', 'm', 'l', 'xl', 'xxl', 'xxxl', 'cpu-xl'.",
         ),
     )
 
@@ -152,8 +152,8 @@ class TaskDefinitionsSchemaV2(ExcludeUnknownSchema):
     serverType = fields.String(
         required=False,
         validate=OneOf(
-            ["xs", "s", "m", "l", "xl", "xxl", "xxxl"],
-            error="Invalid serverType. Available options: 'xs', 's', 'm', 'l', 'xl', 'xxl', 'xxxl'.",
+            ["xs", "s", "m", "l", "xl", "xxl", "xxxl", "cpu-xl"],
+            error="Invalid serverType. Available options: 'xs', 's', 'm', 'l', 'xl', 'xxl', 'xxxl', 'cpu-xl'.",
         ),
     )
 

pypeline/pipeline_settings_schema.py

@@ -1,4 +1,4 @@
-from marshmallow import Schema, fields, validate, ValidationError, validates_schema
+from marshmallow import Schema, fields, validate, ValidationError, validates_schema, INCLUDE
 
 
 class MissingSettingsException(Exception):
@@ -178,6 +178,9 @@ def create_pipeline_settings_schema(pipeline_settings_schema_data):
         else:
             schema_fields[key] = field_type(**field_args)
 
+    schema_fields["Meta"] = type(
+        "Meta", (), {"unknown": INCLUDE}
+    )
     # Dynamically create a schema class with the generated fields
     DynamicPipelineSettingsSchema = type(
         "DynamicPipelineSettingsSchema", (Schema,), schema_fields

pypeline/pipelines/middleware/pypeline_middleware.py

@@ -48,16 +48,16 @@ class PypelineMiddleware(Middleware):
 
         messages = []
         for child in children_tasks:
-            child_ancestors = sorted(nx.ancestors(graph, child))
+            child_ancestors = sorted(graph.predecessors(child))
 
             ancestor_tasks_complete = True
 
             for ancestor in child_ancestors:
-                task_key = f"{execution_id}-{ancestor}"
+                ancestor_task_key = f"{execution_id}-{ancestor}"
 
                 locking_parallel_barrier = LockingParallelBarrier(
                     self.redis_url,
-                    task_key=task_key,
+                    task_key=ancestor_task_key,
                     lock_key=f"{message.options['base_case_execution_id']}-lock",
                 )
                 try:
@@ -75,12 +75,14 @@ class PypelineMiddleware(Middleware):
                 # If the lock didn't exist for the current tasks execution id then it would indicate
                 # that this is the start of a new scenario.  Therefore we need to find the ancestor
                 # that is executed in the base case execution id and make sure it has completed
-                if not remaining_tasks:
-                    task_key = f"{message.options['base_case_execution_id']}-{ancestor}"
+                if remaining_tasks is None:
+                    ancestor_task_key = (
+                        f"{message.options['base_case_execution_id']}-{ancestor}"
+                    )
 
                     locking_parallel_barrier = LockingParallelBarrier(
                         self.redis_url,
-                        task_key=task_key,
+                        task_key=ancestor_task_key,
                         lock_key=f"{message.options['base_case_execution_id']}-lock",
                     )
                     try:

{scalable_pypeline-2.1.10.dist-info → scalable_pypeline-2.1.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.10
+Version: 2.1.12
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable_pypeline-2.1.10.dist-info → scalable_pypeline-2.1.12.dist-info}/RECORD

@@ -1,10 +1,10 @@
-pypeline/__init__.py,sha256=zBzBCTHx_lpM20-2wHeccRHskIUls_oSQ1tzAOiGGrU,23
+pypeline/__init__.py,sha256=2ZypnXz9pLMxwCwfHNJV9M24HPgZwt56lxlCb2BYeEA,23
 pypeline/barrier.py,sha256=oO964l9qOCOibweOHyNivmAvufdXOke9nz2tdgclouo,1172
 pypeline/constants.py,sha256=EGSuLq4KhZ4bxrbtnUgKclELRyya5ipvv0WeybCzNAs,3049
 pypeline/dramatiq.py,sha256=nnNxm2akL6hjqfjwQglK76LspZrdrL_2E_rDN5ThwZ8,13432
 pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
-pypeline/pipeline_config_schema.py,sha256=ApgDqgINFDcmErVP7jJUJgkaVfwNRanj4q12ehWVJUU,11143
-pypeline/pipeline_settings_schema.py,sha256=IBBtWqxrRA0MrUwNZEnGhYJbmag-sus1Ln6VCrUvuyY,20313
+pypeline/pipeline_config_schema.py,sha256=kprvmfPfmNVP5MOqtWzDm4ON5NJWIKQC8GWbIy5EIuk,11183
+pypeline/pipeline_settings_schema.py,sha256=VgbMvTxQSqnvLX593RNoD9gTsZnJwzCc4wGpczV5CEI,20403
 pypeline/pypeline_yaml.py,sha256=Og08sUKwOjq7JYPnkg-NIcGbHravYCkC5Arz22rZEtA,16981
 pypeline/schedule_config_schema.py,sha256=vtZV-5wpGcAiYcXxdBPRkrjsbR6x_9E-1PC2elrKKbE,3611
 pypeline/flask/__init__.py,sha256=AdljRh0lMiS8ExgDmgzObwVs8jW7hqQuf83Ml8kn8GQ,491
@@ -21,7 +21,7 @@ pypeline/pipelines/composition/pypeline_composition.py,sha256=G6hptAN3LaVGm6DE7y
 pypeline/pipelines/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/pipelines/middleware/get_active_worker_id_middleware.py,sha256=X4ZfRk3L8MD00DTsGHth7oOdy-W7LQV96T8vu5UC42A,755
 pypeline/pipelines/middleware/parallel_pipeline_middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
-pypeline/pipelines/middleware/pypeline_middleware.py,sha256=X3nQDiQQGs-rRyD8_-6SFMpM6-GGR2Qn0A6qlxmX3Qs,8756
+pypeline/pipelines/middleware/pypeline_middleware.py,sha256=wdqcnya9SrcBs2VOfBOIXY03VC0uadMcS5SD4hGOk8U,8843
 pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/utils/config_utils.py,sha256=rAIATyoW7kGETZ_Z2DqiXtGd7bJp5uPfcLtfNPOYsNs,2167
 pypeline/utils/dramatiq_utils.py,sha256=DUdgVywm1182A4i69XzH9EIh1EJ9zAHmJLtOaVSW7pw,3844
@@ -29,9 +29,9 @@ pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdV
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.10.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.10.dist-info/METADATA,sha256=FbXEulwsHZccTmDU7WkEgoO4kPS2z6xAt-29n3gsl6g,5927
-scalable_pypeline-2.1.10.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.10.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.10.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.10.dist-info/RECORD,,
+scalable_pypeline-2.1.12.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.12.dist-info/METADATA,sha256=_4e8M2zofZzICdZZp0arkXFIfyLV7hocyqoTEoxbEts,5927
+scalable_pypeline-2.1.12.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.12.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.1.12.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.12.dist-info/RECORD,,