scalable-pypeline 2.0.3__py2.py3-none-any.whl → 3.0.0__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.0.3"
+__version__ = "3.0.0"

pypeline/composition.py

@@ -51,9 +51,8 @@ class parallel_pipeline:
 
         self.execution_graph = execution_graph
 
-        for message_group in self.messages:
-            for m in message_group:
-                m.options["execution_graph"] = execution_graph
+        for m in self.messages[0]:
+            m.options["execution_graph"] = execution_graph
 
     def __len__(self):
         """Returns the length of the parallel_pipeline."""

pypeline/dramatiq.py

@@ -19,7 +19,7 @@ from dramatiq.cli import (
     make_argument_parser as dramatiq_argument_parser,
     import_object,
 )
-from dramatiq.middleware import default_middleware
+from dramatiq.middleware import default_middleware, CurrentMessage
 from dramatiq.results import Results
 from dramatiq.results.backends.redis import RedisBackend
 from flask import current_app, Flask
@@ -50,6 +50,7 @@ else:
     P = TypeVar("P")
 R = TypeVar("R")
 
+logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
@@ -58,6 +59,7 @@ def configure_default_broker(broker: Broker = None):
     rabbit_broker = broker if broker is not None else RabbitmqBroker(url=RABBIT_URL)
     rabbit_broker.add_middleware(Results(backend=redis_backend))
     rabbit_broker.add_middleware(ParallelPipeline(redis_url=REDIS_URL))
+    rabbit_broker.add_middleware(CurrentMessage())
     register_actors_for_workers(rabbit_broker)
     set_broker(rabbit_broker)
 
@@ -129,7 +131,6 @@ def register_actors_for_workers(broker: Broker):
     if not service:
         return
     for task in service.get("registeredTasks", []):
-        print(task)
         pipeline_meta = None
         for pipeline_key, pipeline in pypeline_config["pipelines"].items():
             pipeline_config = pipeline["config"]
@@ -317,9 +318,11 @@ class LazyActor(object):
         return getattr(self.actor, name)
 
     def register(self, broker):
-        self.actor = register_actor(broker=broker, **self.kw)(
-            ensure_return_value(default_value=True)(self.fn)
-        )
+        self.actor = register_actor(
+            actor_name=f"{self.fn.__module__}.{self.fn.__name__}",
+            broker=broker,
+            **self.kw,
+        )(ensure_return_value(default_value=True)(self.fn))
 
     # Next is regular actor API.
     def send(self, *a, **kw):
@@ -463,8 +466,8 @@ def pypeline_worker(
         command += ["--queues"] + queues
     command += verbose * ["-v"]
     args = parser.parse_args(command)
-    current_app.logger.info("Able to execute the following actors:")
+    logger.info("Able to execute the following actors:")
     for actor in list_managed_actors(broker, queues):
-        current_app.logger.info("    %s.", format_actor(actor))
+        logger.info("    %s.", format_actor(actor))
 
     dramatiq_worker(args)

pypeline/flask/api/pipelines.py

@@ -135,7 +135,7 @@ class PipelineInvoke(MethodView):
         tags=["Pipelines"],
     )
     @bp.arguments(InvokePipelineSchema)
-    @bp.response(InvokePipelineResponseSchema)
+    @bp.response(200, InvokePipelineResponseSchema)
     def post(self, payload: dict, pipeline_id: str):
         """Invoke a pipeline by it's ID; optionally provide pipeline arguments."""
         pipeline_config = retrieve_latest_pipeline_config(pipeline_id=pipeline_id)
@@ -191,7 +191,7 @@ class PipelineResults(MethodView):
         ],
         tags=["Pipelines"],
     )
-    @bp.response(GetPipelineResultResponseSchema)
+    @bp.response(200, GetPipelineResultResponseSchema)
     def get(self, execution_id: str):
         """Retrieve results of a pipeline's execution based on execution_id
 

pypeline/middleware.py

@@ -1,6 +1,4 @@
 import copy
-import os
-
 from dramatiq.middleware import Middleware
 
 from pypeline.barrier import LockingParallelBarrier
@@ -29,7 +27,6 @@ class ParallelPipeline(Middleware):
                     remaining_tasks = locking_parallel_barrier.decrement_task_count()
                 finally:
                     locking_parallel_barrier.release_lock()
-
                 if remaining_tasks <= 0:
                     execution_graph = message.options.get("execution_graph")
 
@@ -54,8 +51,10 @@ class ParallelPipeline(Middleware):
                                     lock_key=f"{completion_uuid}-lock",
                                 )
                                 locking_parallel_barrier.set_task_count(len(next_group))
+                                execution_graph_copy = copy.deepcopy(execution_graph)
+
                                 for next_message in next_group:
                                     next_message["options"][
                                         "execution_graph"
-                                    ] = copy.deepcopy(execution_graph)
+                                    ] = execution_graph_copy
                                     broker.enqueue(Message(**next_message))

pypeline/utils/pipeline_utils.py

@@ -79,8 +79,6 @@ def process_non_none_value(value: T) -> None:
     """
     if value is None:
         raise ValueError("None value is not allowed")
-    # Process the value
-    print(f"Processing value: {value}")
 
 
 def topological_sort_with_parallelism(

{scalable_pypeline-2.0.3.dist-info → scalable_pypeline-3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.0.3
+Version: 3.0.0
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation
@@ -8,8 +8,6 @@ License: Apache License 2.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: PyYAML (<7,>=6.0.1)
-Requires-Dist: click (==8.0.4)
-Requires-Dist: marshmallow (<4,>=3.2.1)
 Requires-Dist: redis (<5,>=4.5.4)
 Requires-Dist: db-medley[redis] (<2,>=1.0.2)
 Requires-Dist: croniter (<2,>=1.0.15)
@@ -19,11 +17,7 @@ Requires-Dist: twine ; extra == 'build'
 Provides-Extra: dev
 Requires-Dist: blackd ; extra == 'dev'
 Provides-Extra: flask
-Requires-Dist: Werkzeug (==2.0.3) ; extra == 'flask'
-Requires-Dist: itsdangerous (==2.0.1) ; extra == 'flask'
-Requires-Dist: Flask (<2,>=1.1.2) ; extra == 'flask'
-Requires-Dist: flask-smorest (<0.29,>=0.23.0) ; extra == 'flask'
-Requires-Dist: Jinja2 (==3.0.3) ; extra == 'flask'
+Requires-Dist: flask-smorest (<0.45,>=0.44.0) ; extra == 'flask'
 Provides-Extra: test
 Requires-Dist: pytest-cov (<3,>=2.6.1) ; extra == 'test'
 Requires-Dist: tox (<4,>=3.14.1) ; extra == 'test'

{scalable_pypeline-2.0.3.dist-info → scalable_pypeline-3.0.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
-pypeline/__init__.py,sha256=_GEKEa6BYjBV34SZkSlAR87aCM5Y9G0aSI0LXL52iJg,22
+pypeline/__init__.py,sha256=EPmgXOdWKks5S__ZMH7Nu6xpAeVrZpfxaFy4pykuyeI,22
 pypeline/barrier.py,sha256=dLDaprH5NB-C7MQjZqPpBBhMjmO0VV_kTonlgweznHc,1096
-pypeline/composition.py,sha256=tnSe9CHejayBGI__VMFpQ6NYuebTqoPTWuKiBMnANKc,13025
+pypeline/composition.py,sha256=pTw9Xb9h4JnV4siFc3JStm5lB-i9djUADo3Kh5K3s7g,12976
 pypeline/constants.py,sha256=vi4UZz1xd0ZeIuelp4QgCQsMlIHW65-lVB8l_iA8kBE,2578
-pypeline/dramatiq.py,sha256=jr9WORqtusC1gnbvF59CNXny8ORk2Lmlbmf1qsbiLXo,14799
+pypeline/dramatiq.py,sha256=vO79DHqLldh5mqQf6-okn6gjFWxYU_bGdkMvTyV9Qmc,14942
 pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
-pypeline/middleware.py,sha256=6vWNCoRVqnASJ40CAOEpe0JcYRpB6BTkLz8E51q4z2Y,2756
+pypeline/middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
 pypeline/pipeline_config_schema.py,sha256=DQ_RMucnA0AyrndlW6lkb0orGromcO6C9GgLHyG6lJ0,8013
 pypeline/pypeline_yaml.py,sha256=Og08sUKwOjq7JYPnkg-NIcGbHravYCkC5Arz22rZEtA,16981
 pypeline/schedule_config_schema.py,sha256=vtZV-5wpGcAiYcXxdBPRkrjsbR6x_9E-1PC2elrKKbE,3611
@@ -12,16 +12,16 @@ pypeline/flask/__init__.py,sha256=AdljRh0lMiS8ExgDmgzObwVs8jW7hqQuf83Ml8kn8GQ,49
 pypeline/flask/decorators.py,sha256=ki6jkjZwbDbCWuj7ET7N-ncZwrASp4Fy7257WIYiAAQ,1102
 pypeline/flask/flask_pypeline.py,sha256=Uqyu3PnSP3DoVZUJPqV9chjT4xdRgvcL3OMXxkbdTEg,5490
 pypeline/flask/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/flask/api/pipelines.py,sha256=sPvEoNwmnJPSA96lZHYS2fwKqZlVyE2OSjUmOPFi91o,7267
+pypeline/flask/api/pipelines.py,sha256=Hne41W0VSmHuiDBE9YwpTt9dUZsTb8ew9U9dKoB85bs,7277
 pypeline/flask/api/schedules.py,sha256=31lwoFlGv-S-2ahGUCnD5YbmKws8yddj6_PEzzdBi9s,1321
 pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/utils/config_utils.py,sha256=rAIATyoW7kGETZ_Z2DqiXtGd7bJp5uPfcLtfNPOYsNs,2167
 pypeline/utils/module_utils.py,sha256=boEP9IYr4p_ick7HlVUfIxOYHQlEmo7dgvDBCQc-C28,2914
-pypeline/utils/pipeline_utils.py,sha256=dewzkMajs7uyPHyHjJfISA9pc2-1J5A99Hm4XqNw5qM,6031
+pypeline/utils/pipeline_utils.py,sha256=tt71hLEFgPieokJZlC1rP2dmCTctrOPt7K1rGlbnT4o,5967
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.0.3.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.0.3.dist-info/METADATA,sha256=5P802DsrRS3QyRxK07sls35o6P7_gHJBEkuhb8GGPwQ,6239
-scalable_pypeline-2.0.3.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.0.3.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.0.3.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.0.3.dist-info/RECORD,,
+scalable_pypeline-3.0.0.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-3.0.0.dist-info/METADATA,sha256=rRgk8isNwrBWH4qXZ3pmtN6SlJm_-MhdQF78LPWRf60,5954
+scalable_pypeline-3.0.0.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-3.0.0.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-3.0.0.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-3.0.0.dist-info/RECORD,,