runnable 0.17.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

runnable/sdk.py

@@ -84,7 +84,7 @@ class BaseTraversal(ABC, BaseModel):
     next_node: str = Field(default="", serialization_alias="next_node")
     terminate_with_success: bool = Field(default=False, exclude=True)
     terminate_with_failure: bool = Field(default=False, exclude=True)
-    on_failure: str = Field(default="", alias="on_failure")
+    on_failure: Optional[Pipeline] = Field(default=None)
 
     model_config = ConfigDict(extra="forbid")
 
@@ -117,18 +117,6 @@ class BaseTraversal(ABC, BaseModel):
 
         return other
 
-    def depends_on(self, node: StepType) -> Self:
-        assert not isinstance(node, Success)
-        assert not isinstance(node, Fail)
-
-        if node.next_node:
-            raise Exception(
-                f"The {node} node already has a next node: {node.next_node}"
-            )
-
-        node.next_node = self.name
-        return self
-
     @model_validator(mode="after")
     def validate_terminations(self) -> Self:
         if self.terminate_with_failure and self.terminate_with_success:
@@ -175,7 +163,6 @@ class BaseTask(BaseTraversal):
             if isinstance(x, str):
                 task_returns.append(TaskReturns(name=x, kind="json"))
                 continue
-
             # Its already task returns
             task_returns.append(x)
 
@@ -188,6 +175,9 @@ class BaseTask(BaseTraversal):
                     "A node not being terminated must have a user defined next node"
                 )
 
+        if self.on_failure:
+            self.on_failure = self.on_failure.steps[0].name  # type: ignore
+
         return TaskNode.parse_from_config(
             self.model_dump(exclude_none=True, by_alias=True)
         )
@@ -605,8 +595,6 @@ class Pipeline(BaseModel):
             The order of steps is important as it determines the order of execution.
             Any on failure behavior should the first step in ```on_failure``` pipelines.
 
-
-
         on_failure (List[List[Pipeline], optional): A list of Pipelines to execute in case of failure.
 
             For example, for the below pipeline:
@@ -624,7 +612,7 @@ class Pipeline(BaseModel):
 
     """
 
-    steps: List[Union[StepType, List["Pipeline"]]]
+    steps: List[StepType]
     name: str = ""
     description: str = ""
 
@@ -637,114 +625,67 @@ class Pipeline(BaseModel):
     _dag: graph.Graph = PrivateAttr()
     model_config = ConfigDict(extra="forbid")
 
-    def _validate_path(self, path: List[StepType], failure_path: bool = False) -> None:
-        # TODO: Drastically simplify this
-        # Check if one and only one step terminates with success
-        # Check no more than one step terminates with failure
-
-        reached_success = False
-        reached_failure = False
-
-        for step in path:
-            if step.terminate_with_success:
-                if reached_success:
-                    raise Exception(
-                        "A pipeline cannot have more than one step that terminates with success"
-                    )
-                reached_success = True
-                continue
-            if step.terminate_with_failure:
-                if reached_failure:
-                    raise Exception(
-                        "A pipeline cannot have more than one step that terminates with failure"
-                    )
-                reached_failure = True
-
-        if not reached_success and not reached_failure:
-            raise Exception(
-                "A pipeline must have at least one step that terminates with success"
-            )
-
-    def _construct_path(self, path: List[StepType]) -> None:
-        prev_step = path[0]
-
-        for step in path:
-            if step == prev_step:
-                continue
-
-            if prev_step.terminate_with_success or prev_step.terminate_with_failure:
-                raise Exception(
-                    f"A step that terminates with success/failure cannot have a next step: {prev_step}"
-                )
-
-            if prev_step.next_node and prev_step.next_node not in ["success", "fail"]:
-                raise Exception(f"Step already has a next node: {prev_step} ")
-
-            prev_step.next_node = step.name
-            prev_step = step
-
     def model_post_init(self, __context: Any) -> None:
         """
         The sequence of steps can either be:
-            [step1, step2,..., stepN, [step11, step12,..., step1N], [step21, step22,...,]]
+            [step1, step2,..., stepN]
             indicates:
                 - step1 > step2 > ... > stepN
                 - We expect terminate with success or fail to be explicitly stated on a step.
                     - If it is stated, the step cannot have a next step defined apart from "success" and "fail".
-
-                The inner list of steps is only to accommodate on-failure behaviors.
-                    - For sake of simplicity, lets assume that it has the same behavior as the happy pipeline.
-                    - A task which was already seen should not be part of this.
-                    - There should be at least one step which terminates with success
-
                 Any definition of pipeline should have one node that terminates with success.
         """
-        # TODO: Bug with repeat names
-        # TODO: https://github.com/AstraZeneca/runnable/issues/156
+        # The last step of the pipeline is defaulted to be a success step
+        # unless it is explicitly stated to terminate with failure.
+        terminal_step: StepType = self.steps[-1]
+        if not terminal_step.terminate_with_failure:
+            terminal_step.terminate_with_success = True
 
-        success_path: List[StepType] = []
-        on_failure_paths: List[List[StepType]] = []
+        # assert that there is only one termination node with success or failure
+        # Assert that there are no duplicate step names
+        observed: Dict[str, str] = {}
+        count_termination: int = 0
 
         for step in self.steps:
             if isinstance(
                 step, (Stub, PythonTask, NotebookTask, ShellTask, Parallel, Map)
             ):
-                success_path.append(step)
-                continue
-            # on_failure_paths.append(step)
-
-        if not success_path:
-            raise Exception("There should be some success path")
-
-        # Check all paths are valid and construct the path
-        paths = [success_path] + on_failure_paths
-        failure_path = False
-        for path in paths:
-            self._validate_path(path, failure_path)
-            self._construct_path(path)
-
-            failure_path = True
+                if step.terminate_with_success or step.terminate_with_failure:
+                    count_termination += 1
+            if step.name in observed:
+                raise Exception(
+                    f"Step names should be unique. Found duplicate: {step.name}"
+                )
+            observed[step.name] = step.name
 
-        all_steps: List[StepType] = []
+        if count_termination > 1:
+            raise AssertionError(
+                "A pipeline can only have one termination node with success or failure"
+            )
 
-        for path in paths:
-            for step in path:
-                all_steps.append(step)
+        # link the steps by assigning the next_node name to be that name of the node
+        # immediately after it.
+        for i in range(len(self.steps) - 1):
+            self.steps[i] >> self.steps[i + 1]
 
-        seen = set()
-        unique = [x for x in all_steps if not (x in seen or seen.add(x))]  # type: ignore
+        # Add any on_failure pipelines to the steps
+        gathered_on_failure: List[StepType] = []
+        for step in self.steps:
+            if step.on_failure:
+                gathered_on_failure.extend(step.on_failure.steps)
 
         self._dag = graph.Graph(
-            start_at=all_steps[0].name,
+            start_at=self.steps[0].name,
             description=self.description,
             internal_branch_name=self.internal_branch_name,
         )
 
-        for step in unique:
+        self.steps.extend(gathered_on_failure)
+
+        for step in self.steps:
             self._dag.add_node(step.create_node())
 
-        if self.add_terminal_nodes:
-            self._dag.add_terminal_nodes()
+        self._dag.add_terminal_nodes()
 
         self._dag.check_graph()
 

{runnable-0.17.0.dist-info → runnable-0.18.0.dist-info}/METADATA

@@ -1,23 +1,17 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.17.0
+Version: 0.18.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE
 Requires-Python: >=3.10
-Requires-Dist: catalog
 Requires-Dist: click-plugins>=1.1.1
 Requires-Dist: click<=8.1.3
 Requires-Dist: dill>=0.3.9
-Requires-Dist: job-executor
-Requires-Dist: nodes
-Requires-Dist: pipeline-executor
 Requires-Dist: pydantic>=2.10.3
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: rich>=13.9.4
 Requires-Dist: ruamel-yaml>=0.18.6
-Requires-Dist: run-log-store
-Requires-Dist: secrets
 Requires-Dist: setuptools>=75.6.0
 Requires-Dist: stevedore>=5.4.0
 Requires-Dist: typer>=0.15.1

runnable-0.18.0.dist-info/RECORD

@@ -0,0 +1,58 @@
+extensions/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/catalog/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/catalog/file_system.py,sha256=VZEUx4X-GDSM8rJ_2kiCOyw1eek3roN0CiSB8wdUcOA,9307
+extensions/catalog/pyproject.toml,sha256=lLNxY6v04c8I5QK_zKw_E6sJTArSJRA_V-79ktaA3Hk,279
+extensions/job_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/job_executor/__init__.py,sha256=HINaPjBWz04Ni7GqhuDLi0lS0-gYzq52HcOioYueYJE,5513
+extensions/job_executor/k8s.py,sha256=BHICgJ_TT0JoGKPfX5GH_DpflKtIcSHp8I9i1ycSRZo,11516
+extensions/job_executor/k8s_job_spec.yaml,sha256=7aFpxHdO_p6Hkc3YxusUOuAQTD1Myu0yTPX9DrhxbOg,1158
+extensions/job_executor/local.py,sha256=8ebu4TKo6FnFiUflil6fmE7Pk8eSoe1fHNwX8YwI1BQ,1865
+extensions/job_executor/local_container.py,sha256=7G2ARgoPwXbpNPgd5UDxWZqU2ABPIAk7bkNQkC4cNBQ,6585
+extensions/job_executor/pyproject.toml,sha256=UIEgiCYHTXcRWSByNMFuKJFKgxTBpQqTqyUecIsb_Vc,286
+extensions/nodes/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/nodes/nodes.py,sha256=ib68QE737ihGLIVp3V2wea13u7lmMZdRvK80bgUkRtA,34645
+extensions/nodes/pyproject.toml,sha256=YTu-ETN3JNFSkMzzWeOwn4m-O2nbRH-PmiPBALDCUw4,278
+extensions/pipeline_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/pipeline_executor/__init__.py,sha256=YnKILiy-SxfnG3rYUoinjh1lfkuAF5QXpPePtn6VxBY,25174
+extensions/pipeline_executor/argo.py,sha256=ClfuU_Of_2f5mvqVgY1QQwwJwXHB0LbzwNArG1x2Axc,44666
+extensions/pipeline_executor/argo_specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
+extensions/pipeline_executor/local.py,sha256=H8s6AdML_9_f-vdGG_6k0y9FbLqAqvA1S_7xMNyARzY,1946
+extensions/pipeline_executor/local_container.py,sha256=hqxLkxBOzk8P_PYnnZw5Ve9K0ztoyedv4kUVLkz8vVY,13967
+extensions/pipeline_executor/mocked.py,sha256=SuObJ6Myt7p8duW8sylIp1cYIAnFutsJW1avWaOUY3c,5798
+extensions/pipeline_executor/pyproject.toml,sha256=ykTX7srR10PBYb8LsIwEj8vIPPIEZQ5V_R7VYbZ-ido,291
+extensions/pipeline_executor/retry.py,sha256=KGenhWrLLmOQgzMvqloXHDRJyoNs91t05rRW8aLW6FA,6969
+extensions/run_log_store/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/run_log_store/chunked_fs.py,sha256=ElftNIwBmA2U2QAVGxruhcqepV312M2C9-GWVtiFaMM,3331
+extensions/run_log_store/file_system.py,sha256=SANQ3aFjQeUaq8euvdpwju-8uci9UxdiEDupXtLYppQ,4303
+extensions/run_log_store/generic_chunked.py,sha256=BX0j6S1Fwma3wuitHelUYm69FqXGToh10Zk2kamw6ZY,20253
+extensions/run_log_store/pyproject.toml,sha256=YnmXsFvFG9uv_c0spLYBsNI_1sbktqxtHsOuClyvZ3g,288
+extensions/run_log_store/db/implementation_FF.py,sha256=euTnh0xzNF0e_DyfHQ4W-kG1AwTr8u7OuO3_cZkR5bM,5237
+extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/secrets/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extensions/secrets/dotenv.py,sha256=FbYYd_pVuJuVuIDIvXbzKuSSQ9GPq7xJXTDbJMTQbhM,1583
+extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
+runnable/__init__.py,sha256=KqpLDTD1CfdEj2aDyEkSn2KW-_83qyrRrrWLc5lZVM4,624
+runnable/catalog.py,sha256=MiEmb-18liAKmgeMdDF41VVn0ZEAVLP8hR33oacQ1zs,4930
+runnable/cli.py,sha256=01zmzOdynEmLI4vWDtSHQ6y1od_Jlc8G1RF69fi2L8g,8446
+runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
+runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
+runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
+runnable/entrypoints.py,sha256=67gPBiIIS4Kd9g6LdoGCraRJPda8K1i7Lp7XcD2iY5k,18913
+runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
+runnable/executor.py,sha256=Rafu9EECrNq1LBkJmS6KYCekchP5ufrR04mHWG-JzqQ,15543
+runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
+runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
+runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
+runnable/parameters.py,sha256=g_bJurLjuppFDiDpfFqy6BRF36o_EY0OC5APl7HJFok,5450
+runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
+runnable/sdk.py,sha256=xN5F4XX8r5wCN131kgN2xG7MkNm0bSGJ3Ukw8prHYJ8,31444
+runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
+runnable/tasks.py,sha256=JnIIYQf3YUidHXIN6hiUIfDnegc7_rJMNXuHW4WS9ig,29378
+runnable/utils.py,sha256=wqyN7lMW56cBqyE59iDE6_i2HXPkvEUCQ-66UQnIwTA,19993
+runnable-0.18.0.dist-info/METADATA,sha256=lBeCXTFGeVWpbxnOj2pZY6hTSNyB2Q5OWx5kVAF252s,9945
+runnable-0.18.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.18.0.dist-info/entry_points.txt,sha256=I92DYldRrCb9HCsoum8GjC2UsQrWpuw2kawXTZpkIz4,1559
+runnable-0.18.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.18.0.dist-info/RECORD,,

runnable-0.17.0.dist-info/RECORD

@@ -1,23 +0,0 @@
-runnable/__init__.py,sha256=KqpLDTD1CfdEj2aDyEkSn2KW-_83qyrRrrWLc5lZVM4,624
-runnable/catalog.py,sha256=MiEmb-18liAKmgeMdDF41VVn0ZEAVLP8hR33oacQ1zs,4930
-runnable/cli.py,sha256=01zmzOdynEmLI4vWDtSHQ6y1od_Jlc8G1RF69fi2L8g,8446
-runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
-runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
-runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
-runnable/entrypoints.py,sha256=67gPBiIIS4Kd9g6LdoGCraRJPda8K1i7Lp7XcD2iY5k,18913
-runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
-runnable/executor.py,sha256=Rafu9EECrNq1LBkJmS6KYCekchP5ufrR04mHWG-JzqQ,15543
-runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
-runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
-runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
-runnable/parameters.py,sha256=g_bJurLjuppFDiDpfFqy6BRF36o_EY0OC5APl7HJFok,5450
-runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
-runnable/sdk.py,sha256=hwdk2dLmJsOTs2GnOlayw8WfliyeZFpA6Tcnp3tgblg,33370
-runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
-runnable/tasks.py,sha256=JnIIYQf3YUidHXIN6hiUIfDnegc7_rJMNXuHW4WS9ig,29378
-runnable/utils.py,sha256=wqyN7lMW56cBqyE59iDE6_i2HXPkvEUCQ-66UQnIwTA,19993
-runnable-0.17.0.dist-info/METADATA,sha256=aPu6b9A_JP90oLpCFcnSJStDaizL5-sXbahPvjWU5Wo,10102
-runnable-0.17.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.17.0.dist-info/entry_points.txt,sha256=I92DYldRrCb9HCsoum8GjC2UsQrWpuw2kawXTZpkIz4,1559
-runnable-0.17.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.17.0.dist-info/RECORD,,