scalable-pypeline 2.0.10__py2.py3-none-any.whl → 2.1.1__py2.py3-none-any.whl

pypeline/utils/pipeline_utils.py

@@ -1,10 +1,7 @@
 import logging
 import typing
 import networkx as nx
-from dramatiq import get_broker, Message
-from pypeline.composition import parallel_pipeline
-from pypeline.dramatiq import LazyActor, get_callable, register_lazy_actor
-from pypeline.utils.config_utils import retrieve_latest_pipeline_config
+
 
 T = typing.TypeVar("T")  # T can be any type
 
@@ -124,36 +121,24 @@ def topological_sort_with_parallelism(
         return topological_sort_with_parallelism(graph, executable_nodes)
 
 
-def dag_generator(pipeline_id: str, *args, **kwargs):
-    pipeline_config = retrieve_latest_pipeline_config(pipeline_id=pipeline_id)["config"]
-    graph = get_execution_graph(pipeline_config)
-    optimal_execution_graph = topological_sort_with_parallelism(graph.copy())
-    broker = get_broker()
-
-    registered_actors: typing.Dict[str, LazyActor] = {}
-    broker.actors.clear()
-
-    messages: typing.List[typing.List[Message]] = []
-
-    task_definitions = pipeline_config["taskDefinitions"]
-    for task_group in optimal_execution_graph:
-        message_group = []
-        for task in task_group:
-            module_path = task_definitions[task]["handler"]
-            tmp_handler = get_callable(module_path)
-            lazy_actor = register_lazy_actor(
-                broker, tmp_handler, pipeline_config["metadata"]
-            )
-            registered_actors[task] = lazy_actor
-            if args and not kwargs:
-                message_group.append(registered_actors[task].message(*args))
-            elif kwargs and not args:
-                message_group.append(registered_actors[task].message(**kwargs))
-            elif args and kwargs:
-                message_group.append(registered_actors[task].message(*args, **kwargs))
-            else:
-                message_group.append(registered_actors[task].message())
-        messages.append(message_group)
-    p = parallel_pipeline(messages)
-
-    return p
+def plt_execution_tree(G):
+    import networkx as nx
+    import matplotlib.pyplot as plt
+
+    # Draw the graph
+    plt.figure(figsize=(8, 6))
+    pos = nx.spring_layout(G)  # Compute positions for nodes
+    nx.draw(
+        G,
+        pos,
+        with_labels=True,
+        node_color="lightblue",
+        node_size=3000,
+        edge_color="gray",
+        arrowsize=20,
+        font_size=12,
+    )
+
+    # Show the plot
+    plt.title("Directed Graph Visualization")
+    plt.show()

pypeline/utils/schema_utils.py

@@ -0,0 +1,24 @@
+def get_clean_validation_messages(validation_error):
+    """
+    Extract and format clean validation error messages.
+
+    Args:
+        validation_error (ValidationError): The Marshmallow ValidationError instance.
+
+    Returns:
+        str: A formatted string with all validation error messages.
+    """
+
+    def format_errors(errors, parent_key=""):
+        messages = []
+        for key, value in errors.items():
+            full_key = f"{parent_key}.{key}" if parent_key else key
+            if isinstance(value, dict):
+                # Recursively format nested errors
+                messages.extend(format_errors(value, full_key))
+            else:
+                # Append error messages
+                messages.append(f"{full_key}: {', '.join(value)}")
+        return messages
+
+    return "\n".join(format_errors(validation_error.messages))

{scalable_pypeline-2.0.10.dist-info → scalable_pypeline-2.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.0.10
+Version: 2.1.1
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

scalable_pypeline-2.1.1.dist-info/RECORD

@@ -0,0 +1,36 @@
+pypeline/__init__.py,sha256=zPJIgPGcoSNiD0qme18OnYJYE3A9VVytlhO-V5DaAW0,22
+pypeline/barrier.py,sha256=oO964l9qOCOibweOHyNivmAvufdXOke9nz2tdgclouo,1172
+pypeline/constants.py,sha256=coiF8dMP25qIwoNYSnS7oy7hCd4-5yqPFmdPsN93Q1A,2892
+pypeline/dramatiq.py,sha256=LWsl0o0t5FdxewIl87ARZKrNK0ENoYJEJAEVDSNFa40,12272
+pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
+pypeline/pipeline_config_schema.py,sha256=hK2_egtg-YFx_XJDs_NyrOTGKkel7W83X-G0sic52sM,10592
+pypeline/pipeline_settings_schema.py,sha256=84AuNFYsOUpoADsjEo_n9T6Ica-c21oK_V9s15I4lCg,20212
+pypeline/pypeline_yaml.py,sha256=Og08sUKwOjq7JYPnkg-NIcGbHravYCkC5Arz22rZEtA,16981
+pypeline/schedule_config_schema.py,sha256=vtZV-5wpGcAiYcXxdBPRkrjsbR6x_9E-1PC2elrKKbE,3611
+pypeline/flask/__init__.py,sha256=AdljRh0lMiS8ExgDmgzObwVs8jW7hqQuf83Ml8kn8GQ,491
+pypeline/flask/decorators.py,sha256=ki6jkjZwbDbCWuj7ET7N-ncZwrASp4Fy7257WIYiAAQ,1102
+pypeline/flask/flask_pypeline.py,sha256=Uqyu3PnSP3DoVZUJPqV9chjT4xdRgvcL3OMXxkbdTEg,5490
+pypeline/flask/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/flask/api/pipelines.py,sha256=8Y5dkIVb32dMc0jBI7lB2sQgsAIe7WYmPn-G9tlUY5o,10161
+pypeline/flask/api/schedules.py,sha256=8PKCMdPucaer8opchNlI5aDssK2UqT79hHpeg5BMtTA,1210
+pypeline/pipelines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/pipelines/factory.py,sha256=4HNGUJzYtgBOWP7fStXF0M61CYNAid9l9PGru9HyhXA,4115
+pypeline/pipelines/composition/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/pipelines/composition/parallel_pipeline_composition.py,sha256=pTw9Xb9h4JnV4siFc3JStm5lB-i9djUADo3Kh5K3s7g,12976
+pypeline/pipelines/composition/pypeline_composition.py,sha256=ieTuQZ8zxTtvmPEkrWFbItjGtvO3JUotXcR-Jim2mss,7204
+pypeline/pipelines/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/pipelines/middleware/parallel_pipeline_middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
+pypeline/pipelines/middleware/pypeline_middleware.py,sha256=kvt5A9OxDwpIo0PsH11Im62tH6VquUc6OFoZDw2Gxsk,8036
+pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/utils/config_utils.py,sha256=rAIATyoW7kGETZ_Z2DqiXtGd7bJp5uPfcLtfNPOYsNs,2167
+pypeline/utils/dramatiq_utils.py,sha256=5GDcOvKY-8S8r---wb6Q8QAywhbKVJ-qILjcYNHei8Y,3658
+pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdVU,3682
+pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
+pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
+tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scalable_pypeline-2.1.1.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.1.dist-info/METADATA,sha256=wgJ50QCkHF7mvEqD1XN4X6x3XVbb7Ne2dso-NWe7Qsc,5926
+scalable_pypeline-2.1.1.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.1.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.1.1.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.1.dist-info/RECORD,,

scalable_pypeline-2.0.10.dist-info/RECORD

@@ -1,27 +0,0 @@
-pypeline/__init__.py,sha256=QvVarwQu86KS14HXwAYbCqU1tjzA5eNjQxH2V34_iIU,23
-pypeline/barrier.py,sha256=dLDaprH5NB-C7MQjZqPpBBhMjmO0VV_kTonlgweznHc,1096
-pypeline/composition.py,sha256=pTw9Xb9h4JnV4siFc3JStm5lB-i9djUADo3Kh5K3s7g,12976
-pypeline/constants.py,sha256=coiF8dMP25qIwoNYSnS7oy7hCd4-5yqPFmdPsN93Q1A,2892
-pypeline/dramatiq.py,sha256=Y909HoNhH5Berd61N6nHrpE1dTU-zmvimH91SldP-SI,15912
-pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
-pypeline/middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
-pypeline/pipeline_config_schema.py,sha256=DQ_RMucnA0AyrndlW6lkb0orGromcO6C9GgLHyG6lJ0,8013
-pypeline/pypeline_yaml.py,sha256=Og08sUKwOjq7JYPnkg-NIcGbHravYCkC5Arz22rZEtA,16981
-pypeline/schedule_config_schema.py,sha256=vtZV-5wpGcAiYcXxdBPRkrjsbR6x_9E-1PC2elrKKbE,3611
-pypeline/flask/__init__.py,sha256=AdljRh0lMiS8ExgDmgzObwVs8jW7hqQuf83Ml8kn8GQ,491
-pypeline/flask/decorators.py,sha256=ki6jkjZwbDbCWuj7ET7N-ncZwrASp4Fy7257WIYiAAQ,1102
-pypeline/flask/flask_pypeline.py,sha256=Uqyu3PnSP3DoVZUJPqV9chjT4xdRgvcL3OMXxkbdTEg,5490
-pypeline/flask/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/flask/api/pipelines.py,sha256=RzRmSL5Zaia7ftXmXyDAC0ZAkPoFsvyefzHfIyWhRqk,8118
-pypeline/flask/api/schedules.py,sha256=31lwoFlGv-S-2ahGUCnD5YbmKws8yddj6_PEzzdBi9s,1321
-pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/utils/config_utils.py,sha256=rAIATyoW7kGETZ_Z2DqiXtGd7bJp5uPfcLtfNPOYsNs,2167
-pypeline/utils/module_utils.py,sha256=boEP9IYr4p_ick7HlVUfIxOYHQlEmo7dgvDBCQc-C28,2914
-pypeline/utils/pipeline_utils.py,sha256=tt71hLEFgPieokJZlC1rP2dmCTctrOPt7K1rGlbnT4o,5967
-tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.0.10.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.0.10.dist-info/METADATA,sha256=mFn_XkdRsU4d5oT48HAQNeP9wEUifs2-aWR43-3SG-4,5927
-scalable_pypeline-2.0.10.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.0.10.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.0.10.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.0.10.dist-info/RECORD,,