runnable 0.16.0__tar.gz → 0.17.1__tar.gz

{runnable-0.16.0 → runnable-0.17.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.16.0
+Version: 0.17.1
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.16.0 → runnable-0.17.1}/examples/02-sequential/on_failure_fail.py

@@ -28,7 +28,8 @@ def main():
     step_3 = Stub(name="step 3", terminate_with_success=True)
     step_4 = Stub(name="step 4", terminate_with_failure=True)  # (1)
 
-    step_1.on_failure = step_4.name
+    on_failure_pipeline = Pipeline(steps=[step_4])
+    step_1.on_failure = on_failure_pipeline  # (2)
 
     pipeline = Pipeline(
         steps=[step_1, step_2, step_3],

{runnable-0.16.0 → runnable-0.17.1}/examples/02-sequential/on_failure_succeed.py

@@ -28,10 +28,12 @@ def main():
     step_3 = Stub(name="step 3", terminate_with_success=True)
     step_4 = Stub(name="step 4", terminate_with_success=True)  # (1)
 
-    step_1.on_failure = step_4.name
+    on_failure_pipeline = Pipeline(steps=[step_4])
+
+    step_1.on_failure = on_failure_pipeline  # (2)
 
     pipeline = Pipeline(
-        steps=[step_1, step_2, step_3, [step_4]],
+        steps=[step_1, step_2, step_3],
     )
     pipeline.execute()
 

runnable-0.17.1/examples/configs/in-memory.yaml

@@ -0,0 +1,14 @@
+pipeline-executor:
+  type: local # (1)
+  config:
+    object_serialisation: false
+
+
+run-log-store:
+  type: buffered # (2)
+
+catalog:
+  type: do-nothing # (3)
+
+secrets:
+  type: do-nothing # (4)

{runnable-0.16.0 → runnable-0.17.1}/extensions/pipeline_executor/local.py

@@ -1,5 +1,7 @@
 import logging
 
+from pydantic import Field, PrivateAttr
+
 from extensions.pipeline_executor import GenericPipelineExecutor
 from runnable import defaults
 from runnable.defaults import TypeMapVariable
@@ -22,7 +24,18 @@ class LocalExecutor(GenericPipelineExecutor):
     """
 
     service_name: str = "local"
-    _is_local: bool = True
+
+    object_serialisation: bool = Field(default=True)
+
+    _is_local: bool = PrivateAttr(default=True)
+
+    def execute_from_graph(
+        self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs
+    ):
+        if not self.object_serialisation:
+            self._context.object_serialisation = False
+
+        super().execute_from_graph(node=node, map_variable=map_variable, **kwargs)
 
     def trigger_node_execution(
         self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs
@@ -47,30 +60,3 @@ class LocalExecutor(GenericPipelineExecutor):
             map_variable (dict[str, str], optional): _description_. Defaults to None.
         """
         self._execute_node(node=node, map_variable=map_variable, **kwargs)
-
-    # def execute_job(self, node: TaskNode):
-    #     """
-    #     Set up the step log and call the execute node
-
-    #     Args:
-    #         node (BaseNode): _description_
-    #     """
-
-    #     step_log = self._context.run_log_store.create_step_log(
-    #         node.name, node._get_step_log_name(map_variable=None)
-    #     )
-
-    #     self.add_code_identities(node=node, step_log=step_log)
-
-    #     step_log.step_type = node.node_type
-    #     step_log.status = defaults.PROCESSING
-    #     self._context.run_log_store.add_step_log(step_log, self._context.run_id)
-    #     self.execute_node(node=node)
-
-    #     # Update the run log status
-    #     step_log = self._context.run_log_store.get_step_log(
-    #         node._get_step_log_name(), self._context.run_id
-    #     )
-    #     self._context.run_log_store.update_run_log_status(
-    #         run_id=self._context.run_id, status=step_log.status
-    #     )

{runnable-0.16.0 → runnable-0.17.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runnable"
-version = "0.16.0"
+version = "0.17.1"
 description = "Add your description here"
 readme = "README.md"
 authors = [

{runnable-0.16.0 → runnable-0.17.1}/runnable/context.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny
 from rich.progress import Progress
@@ -29,6 +29,8 @@ class Context(BaseModel):
     from_sdk: bool = False
 
     run_id: str = ""
+    object_serialisation: bool = True
+    return_objects: Dict[str, Any] = {}
 
     tag: str = ""
     variables: Dict[str, str] = {}

{runnable-0.16.0 → runnable-0.17.1}/runnable/datastore.py

@@ -98,22 +98,33 @@ class ObjectParameter(BaseModel):
     @computed_field  # type: ignore
     @property
     def description(self) -> str:
-        return f"Pickled object stored in catalog as: {self.value}"
+        if context.run_context.object_serialisation:
+            return f"Pickled object stored in catalog as: {self.value}"
+
+        return f"Object stored in memory as: {self.value}"
 
     @property
     def file_name(self) -> str:
         return f"{self.value}{context.run_context.pickler.extension}"
 
     def get_value(self) -> Any:
-        # Get the pickled object
-        catalog_handler = context.run_context.catalog_handler
+        # If there was no serialisation, return the object from the return objects
+        if not context.run_context.object_serialisation:
+            return context.run_context.return_objects[self.value]
 
+        # If the object was serialised, get it from the catalog
+        catalog_handler = context.run_context.catalog_handler
         catalog_handler.get(name=self.file_name, run_id=context.run_context.run_id)
         obj = context.run_context.pickler.load(path=self.file_name)
         os.remove(self.file_name)  # Remove after loading
         return obj
 
     def put_object(self, data: Any) -> None:
+        if not context.run_context.object_serialisation:
+            context.run_context.return_objects[self.value] = data
+            return
+
+        # If the object was serialised, put it in the catalog
         context.run_context.pickler.dump(data=data, path=self.file_name)
 
         catalog_handler = context.run_context.catalog_handler

{runnable-0.16.0 → runnable-0.17.1}/runnable/executor.py

@@ -34,9 +34,7 @@ class BaseExecutor(ABC, BaseModel):
     service_name: str = ""
     service_type: str = "executor"
 
-    _is_local: bool = (
-        False  # This is a flag to indicate whether the executor is local or not.
-    )
+    _is_local: bool = PrivateAttr(default=False)
 
     model_config = ConfigDict(extra="forbid")
 

{runnable-0.16.0 → runnable-0.17.1}/runnable/sdk.py

@@ -84,7 +84,7 @@ class BaseTraversal(ABC, BaseModel):
     next_node: str = Field(default="", serialization_alias="next_node")
     terminate_with_success: bool = Field(default=False, exclude=True)
     terminate_with_failure: bool = Field(default=False, exclude=True)
-    on_failure: str = Field(default="", alias="on_failure")
+    on_failure: Optional[Pipeline] = Field(default=None)
 
     model_config = ConfigDict(extra="forbid")
 
@@ -117,18 +117,6 @@ class BaseTraversal(ABC, BaseModel):
 
         return other
 
-    def depends_on(self, node: StepType) -> Self:
-        assert not isinstance(node, Success)
-        assert not isinstance(node, Fail)
-
-        if node.next_node:
-            raise Exception(
-                f"The {node} node already has a next node: {node.next_node}"
-            )
-
-        node.next_node = self.name
-        return self
-
     @model_validator(mode="after")
     def validate_terminations(self) -> Self:
         if self.terminate_with_failure and self.terminate_with_success:
@@ -175,7 +163,6 @@ class BaseTask(BaseTraversal):
             if isinstance(x, str):
                 task_returns.append(TaskReturns(name=x, kind="json"))
                 continue
-
             # Its already task returns
             task_returns.append(x)
 
@@ -188,6 +175,9 @@ class BaseTask(BaseTraversal):
                     "A node not being terminated must have a user defined next node"
                 )
 
+        if self.on_failure:
+            self.on_failure = self.on_failure.steps[0].name  # type: ignore
+
         return TaskNode.parse_from_config(
             self.model_dump(exclude_none=True, by_alias=True)
         )
@@ -605,8 +595,6 @@ class Pipeline(BaseModel):
             The order of steps is important as it determines the order of execution.
             Any on failure behavior should the first step in ```on_failure``` pipelines.
 
-
-
         on_failure (List[List[Pipeline], optional): A list of Pipelines to execute in case of failure.
 
             For example, for the below pipeline:
@@ -624,7 +612,7 @@ class Pipeline(BaseModel):
 
     """
 
-    steps: List[Union[StepType, List["Pipeline"]]]
+    steps: List[StepType]
     name: str = ""
     description: str = ""
 
@@ -637,114 +625,67 @@ class Pipeline(BaseModel):
     _dag: graph.Graph = PrivateAttr()
     model_config = ConfigDict(extra="forbid")
 
-    def _validate_path(self, path: List[StepType], failure_path: bool = False) -> None:
-        # TODO: Drastically simplify this
-        # Check if one and only one step terminates with success
-        # Check no more than one step terminates with failure
-
-        reached_success = False
-        reached_failure = False
-
-        for step in path:
-            if step.terminate_with_success:
-                if reached_success:
-                    raise Exception(
-                        "A pipeline cannot have more than one step that terminates with success"
-                    )
-                reached_success = True
-                continue
-            if step.terminate_with_failure:
-                if reached_failure:
-                    raise Exception(
-                        "A pipeline cannot have more than one step that terminates with failure"
-                    )
-                reached_failure = True
-
-        if not reached_success and not reached_failure:
-            raise Exception(
-                "A pipeline must have at least one step that terminates with success"
-            )
-
-    def _construct_path(self, path: List[StepType]) -> None:
-        prev_step = path[0]
-
-        for step in path:
-            if step == prev_step:
-                continue
-
-            if prev_step.terminate_with_success or prev_step.terminate_with_failure:
-                raise Exception(
-                    f"A step that terminates with success/failure cannot have a next step: {prev_step}"
-                )
-
-            if prev_step.next_node and prev_step.next_node not in ["success", "fail"]:
-                raise Exception(f"Step already has a next node: {prev_step} ")
-
-            prev_step.next_node = step.name
-            prev_step = step
-
     def model_post_init(self, __context: Any) -> None:
         """
         The sequence of steps can either be:
-            [step1, step2,..., stepN, [step11, step12,..., step1N], [step21, step22,...,]]
+            [step1, step2,..., stepN]
             indicates:
                 - step1 > step2 > ... > stepN
                 - We expect terminate with success or fail to be explicitly stated on a step.
                     - If it is stated, the step cannot have a next step defined apart from "success" and "fail".
-
-                The inner list of steps is only to accommodate on-failure behaviors.
-                    - For sake of simplicity, lets assume that it has the same behavior as the happy pipeline.
-                    - A task which was already seen should not be part of this.
-                    - There should be at least one step which terminates with success
-
                 Any definition of pipeline should have one node that terminates with success.
         """
-        # TODO: Bug with repeat names
-        # TODO: https://github.com/AstraZeneca/runnable/issues/156
+        # The last step of the pipeline is defaulted to be a success step
+        # unless it is explicitly stated to terminate with failure.
+        terminal_step: StepType = self.steps[-1]
+        if not terminal_step.terminate_with_failure:
+            terminal_step.terminate_with_success = True
 
-        success_path: List[StepType] = []
-        on_failure_paths: List[List[StepType]] = []
+        # assert that there is only one termination node with success or failure
+        # Assert that there are no duplicate step names
+        observed: Dict[str, str] = {}
+        count_termination: int = 0
 
         for step in self.steps:
             if isinstance(
                 step, (Stub, PythonTask, NotebookTask, ShellTask, Parallel, Map)
             ):
-                success_path.append(step)
-                continue
-            # on_failure_paths.append(step)
-
-        if not success_path:
-            raise Exception("There should be some success path")
-
-        # Check all paths are valid and construct the path
-        paths = [success_path] + on_failure_paths
-        failure_path = False
-        for path in paths:
-            self._validate_path(path, failure_path)
-            self._construct_path(path)
-
-            failure_path = True
+                if step.terminate_with_success or step.terminate_with_failure:
+                    count_termination += 1
+            if step.name in observed:
+                raise Exception(
+                    f"Step names should be unique. Found duplicate: {step.name}"
+                )
+            observed[step.name] = step.name
 
-        all_steps: List[StepType] = []
+        if count_termination > 1:
+            raise AssertionError(
+                "A pipeline can only have one termination node with success or failure"
+            )
 
-        for path in paths:
-            for step in path:
-                all_steps.append(step)
+        # link the steps by assigning the next_node name to be that name of the node
+        # immediately after it.
+        for i in range(len(self.steps) - 1):
+            self.steps[i] >> self.steps[i + 1]
 
-        seen = set()
-        unique = [x for x in all_steps if not (x in seen or seen.add(x))]  # type: ignore
+        # Add any on_failure pipelines to the steps
+        gathered_on_failure: List[StepType] = []
+        for step in self.steps:
+            if step.on_failure:
+                gathered_on_failure.extend(step.on_failure.steps)
 
         self._dag = graph.Graph(
-            start_at=all_steps[0].name,
+            start_at=self.steps[0].name,
             description=self.description,
             internal_branch_name=self.internal_branch_name,
         )
 
-        for step in unique:
+        self.steps.extend(gathered_on_failure)
+
+        for step in self.steps:
             self._dag.add_node(step.create_node())
 
-        if self.add_terminal_nodes:
-            self._dag.add_terminal_nodes()
+        self._dag.add_terminal_nodes()
 
         self._dag.check_graph()