runnable 0.17.0__tar.gz → 0.18.0__tar.gz

{runnable-0.17.0 → runnable-0.18.0}/PKG-INFO

@@ -1,23 +1,17 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.17.0
+Version: 0.18.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE
 Requires-Python: >=3.10
-Requires-Dist: catalog
 Requires-Dist: click-plugins>=1.1.1
 Requires-Dist: click<=8.1.3
 Requires-Dist: dill>=0.3.9
-Requires-Dist: job-executor
-Requires-Dist: nodes
-Requires-Dist: pipeline-executor
 Requires-Dist: pydantic>=2.10.3
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: rich>=13.9.4
 Requires-Dist: ruamel-yaml>=0.18.6
-Requires-Dist: run-log-store
-Requires-Dist: secrets
 Requires-Dist: setuptools>=75.6.0
 Requires-Dist: stevedore>=5.4.0
 Requires-Dist: typer>=0.15.1

runnable-0.18.0/extensions/catalog/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "catalog"
+version = "0.0.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

runnable-0.18.0/extensions/job_executor/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "job_executor"
+version = "0.0.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

runnable-0.18.0/extensions/nodes/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "nodes"
+version = "0.0.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

runnable-0.18.0/extensions/pipeline_executor/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "pipeline_executor"
+version = "0.0.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

runnable-0.18.0/extensions/run_log_store/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "run_log_store"
+version = "0.0.0"
+description = "Extensions to run log store"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

runnable-0.18.0/extensions/secrets/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "secrets"
+version = "0.0.0"
+description = "Extension to manage secrets"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

{runnable-0.17.0 → runnable-0.18.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runnable"
-version = "0.17.0"
+version = "0.18.0"
 description = "Add your description here"
 readme = "README.md"
 authors = [
@@ -18,17 +18,8 @@ dependencies = [
     "setuptools>=75.6.0",
     "python-dotenv>=1.0.1",
     "typer>=0.15.1",
-    "nodes",
-    "secrets",
-    "catalog",
-    "run_log_store",
-    "pipeline_executor",
-    "job_executor",
 ]
 
-[project.scripts]
-runnable = 'runnable.cli:app'
-
 [project.optional-dependencies]
 docker = [
     "docker>=7.1.0",
@@ -43,10 +34,6 @@ k8s = [
     "kubernetes>=31.0.0",
 ]
 
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
 [dependency-groups]
 dev = [
     "pytest-cov>=6.0.0",
@@ -68,7 +55,13 @@ release = [
 ]
 
 [tool.uv.workspace]
-members = ["extensions/*"]
+members = ["extensions/catalog",
+        "extensions/job_executor",
+        "extensions/nodes",
+        "extensions/pipeline_executor",
+        "extensions/run_log_store",
+        "extensions/secrets",
+]
 
 [tool.uv.sources]
 nodes = {workspace = true}
@@ -79,6 +72,22 @@ pipeline_executor = {workspace = true}
 job_executor = {workspace = true}
 
 
+[project.scripts]
+runnable = 'runnable.cli:app'
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build]
+ignore-vcs = true
+include = [
+  "/runnable",
+  "/extensions",
+]
+
+
+# Plugin systems
 [project.entry-points.'pipeline_executor']
 "local" = "extensions.pipeline_executor.local:LocalExecutor"
 "local-container" = "extensions.pipeline_executor.local_container:LocalContainerExecutor"
@@ -127,6 +136,7 @@ file-system = "extensions.run_log_store.file_system:FileSystemRunLogstore"
 "notebook" = "runnable.tasks:NotebookTaskType"
 
 
+# Release configuration
 [tool.semantic_release]
 commit_parser = "angular"
 major_on_zero = true

{runnable-0.17.0 → runnable-0.18.0}/runnable/sdk.py

@@ -84,7 +84,7 @@ class BaseTraversal(ABC, BaseModel):
     next_node: str = Field(default="", serialization_alias="next_node")
     terminate_with_success: bool = Field(default=False, exclude=True)
     terminate_with_failure: bool = Field(default=False, exclude=True)
-    on_failure: str = Field(default="", alias="on_failure")
+    on_failure: Optional[Pipeline] = Field(default=None)
 
     model_config = ConfigDict(extra="forbid")
 
@@ -117,18 +117,6 @@ class BaseTraversal(ABC, BaseModel):
 
         return other
 
-    def depends_on(self, node: StepType) -> Self:
-        assert not isinstance(node, Success)
-        assert not isinstance(node, Fail)
-
-        if node.next_node:
-            raise Exception(
-                f"The {node} node already has a next node: {node.next_node}"
-            )
-
-        node.next_node = self.name
-        return self
-
     @model_validator(mode="after")
     def validate_terminations(self) -> Self:
         if self.terminate_with_failure and self.terminate_with_success:
@@ -175,7 +163,6 @@ class BaseTask(BaseTraversal):
             if isinstance(x, str):
                 task_returns.append(TaskReturns(name=x, kind="json"))
                 continue
-
             # Its already task returns
             task_returns.append(x)
 
@@ -188,6 +175,9 @@ class BaseTask(BaseTraversal):
                     "A node not being terminated must have a user defined next node"
                 )
 
+        if self.on_failure:
+            self.on_failure = self.on_failure.steps[0].name  # type: ignore
+
         return TaskNode.parse_from_config(
             self.model_dump(exclude_none=True, by_alias=True)
         )
@@ -605,8 +595,6 @@ class Pipeline(BaseModel):
             The order of steps is important as it determines the order of execution.
             Any on failure behavior should the first step in ```on_failure``` pipelines.
 
-
-
         on_failure (List[List[Pipeline], optional): A list of Pipelines to execute in case of failure.
 
             For example, for the below pipeline:
@@ -624,7 +612,7 @@ class Pipeline(BaseModel):
 
     """
 
-    steps: List[Union[StepType, List["Pipeline"]]]
+    steps: List[StepType]
     name: str = ""
     description: str = ""
 
@@ -637,114 +625,67 @@ class Pipeline(BaseModel):
     _dag: graph.Graph = PrivateAttr()
     model_config = ConfigDict(extra="forbid")
 
-    def _validate_path(self, path: List[StepType], failure_path: bool = False) -> None:
-        # TODO: Drastically simplify this
-        # Check if one and only one step terminates with success
-        # Check no more than one step terminates with failure
-
-        reached_success = False
-        reached_failure = False
-
-        for step in path:
-            if step.terminate_with_success:
-                if reached_success:
-                    raise Exception(
-                        "A pipeline cannot have more than one step that terminates with success"
-                    )
-                reached_success = True
-                continue
-            if step.terminate_with_failure:
-                if reached_failure:
-                    raise Exception(
-                        "A pipeline cannot have more than one step that terminates with failure"
-                    )
-                reached_failure = True
-
-        if not reached_success and not reached_failure:
-            raise Exception(
-                "A pipeline must have at least one step that terminates with success"
-            )
-
-    def _construct_path(self, path: List[StepType]) -> None:
-        prev_step = path[0]
-
-        for step in path:
-            if step == prev_step:
-                continue
-
-            if prev_step.terminate_with_success or prev_step.terminate_with_failure:
-                raise Exception(
-                    f"A step that terminates with success/failure cannot have a next step: {prev_step}"
-                )
-
-            if prev_step.next_node and prev_step.next_node not in ["success", "fail"]:
-                raise Exception(f"Step already has a next node: {prev_step} ")
-
-            prev_step.next_node = step.name
-            prev_step = step
-
     def model_post_init(self, __context: Any) -> None:
         """
         The sequence of steps can either be:
-            [step1, step2,..., stepN, [step11, step12,..., step1N], [step21, step22,...,]]
+            [step1, step2,..., stepN]
             indicates:
                 - step1 > step2 > ... > stepN
                 - We expect terminate with success or fail to be explicitly stated on a step.
                     - If it is stated, the step cannot have a next step defined apart from "success" and "fail".
-
-                The inner list of steps is only to accommodate on-failure behaviors.
-                    - For sake of simplicity, lets assume that it has the same behavior as the happy pipeline.
-                    - A task which was already seen should not be part of this.
-                    - There should be at least one step which terminates with success
-
                 Any definition of pipeline should have one node that terminates with success.
         """
-        # TODO: Bug with repeat names
-        # TODO: https://github.com/AstraZeneca/runnable/issues/156
+        # The last step of the pipeline is defaulted to be a success step
+        # unless it is explicitly stated to terminate with failure.
+        terminal_step: StepType = self.steps[-1]
+        if not terminal_step.terminate_with_failure:
+            terminal_step.terminate_with_success = True
 
-        success_path: List[StepType] = []
-        on_failure_paths: List[List[StepType]] = []
+        # assert that there is only one termination node with success or failure
+        # Assert that there are no duplicate step names
+        observed: Dict[str, str] = {}
+        count_termination: int = 0
 
         for step in self.steps:
             if isinstance(
                 step, (Stub, PythonTask, NotebookTask, ShellTask, Parallel, Map)
             ):
-                success_path.append(step)
-                continue
-            # on_failure_paths.append(step)
-
-        if not success_path:
-            raise Exception("There should be some success path")
-
-        # Check all paths are valid and construct the path
-        paths = [success_path] + on_failure_paths
-        failure_path = False
-        for path in paths:
-            self._validate_path(path, failure_path)
-            self._construct_path(path)
-
-            failure_path = True
+                if step.terminate_with_success or step.terminate_with_failure:
+                    count_termination += 1
+            if step.name in observed:
+                raise Exception(
+                    f"Step names should be unique. Found duplicate: {step.name}"
+                )
+            observed[step.name] = step.name
 
-        all_steps: List[StepType] = []
+        if count_termination > 1:
+            raise AssertionError(
+                "A pipeline can only have one termination node with success or failure"
+            )
 
-        for path in paths:
-            for step in path:
-                all_steps.append(step)
+        # link the steps by assigning the next_node name to be that name of the node
+        # immediately after it.
+        for i in range(len(self.steps) - 1):
+            self.steps[i] >> self.steps[i + 1]
 
-        seen = set()
-        unique = [x for x in all_steps if not (x in seen or seen.add(x))]  # type: ignore
+        # Add any on_failure pipelines to the steps
+        gathered_on_failure: List[StepType] = []
+        for step in self.steps:
+            if step.on_failure:
+                gathered_on_failure.extend(step.on_failure.steps)
 
         self._dag = graph.Graph(
-            start_at=all_steps[0].name,
+            start_at=self.steps[0].name,
             description=self.description,
             internal_branch_name=self.internal_branch_name,
         )
 
-        for step in unique:
+        self.steps.extend(gathered_on_failure)
+
+        for step in self.steps:
             self._dag.add_node(step.create_node())
 
-        if self.add_terminal_nodes:
-            self._dag.add_terminal_nodes()
+        self._dag.add_terminal_nodes()
 
         self._dag.check_graph()
 

runnable-0.17.0/.dockerignore

@@ -1,10 +0,0 @@
-docs/
-.github/
-.mypy_cache/
-.pytest_cache/
-.ruff_cache/
-.tox/
-.scripts/
-.tests/
-.venv/
-minikube/

runnable-0.17.0/.github/workflows/docs.yaml

@@ -1,18 +0,0 @@
-name: docs
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install uv
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
-      - name: "Set up Python"
-        run: uv python install
-      - run: |
-          uv sync --only-group docs
-          uv run mkdocs gh-deploy --force

runnable-0.17.0/.github/workflows/pr.yaml

@@ -1,53 +0,0 @@
-on:
-  pull_request:
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "examples/**"
-      - "mkdocs.yml"
-    branches:
-      - "main"
-  workflow_dispatch:
-  workflow_call:
-
-jobs:
-  PRCheck:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - run: |
-          # Download the binary
-          curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.5.4/argo-linux-amd64.gz
-
-          # Unzip
-          gunzip argo-linux-amd64.gz
-
-          # Make binary executable
-          chmod +x argo-linux-amd64
-
-          # Move binary to path
-          mv ./argo-linux-amd64 /usr/local/bin/argo
-
-          # Test installation
-          argo version
-
-      - name: Install uv
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
-
-      - name: "Set up Python"
-        run: uv python install
-
-      - name: Install the project
-        run: uv sync --all-extras --dev
-
-      - name: Run lint
-        # For example, using `flake8`
-        run: uvx ruff format
-
-      - name: Run tests
-        run: |
-          export PYTHONDONTWRITEBYTECODE=1
-          uv run pytest -m "not container"
-
-      - name: mypy
-        run: uv run mypy runnable extensions

runnable-0.17.0/.github/workflows/release.yaml

@@ -1,101 +0,0 @@
-on:
-  push:
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "examples/**"
-    branches:
-      - "main"
-
-
-jobs:
-  PRCheck:
-    uses: "./.github/workflows/pr.yaml"
-
-  Release:
-    runs-on: ubuntu-latest
-    needs: PRCheck
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Install uv
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
-
-      - name: "Set up Python"
-        run: uv python install
-      - run: |
-          uv sync --only-group release
-      - name: Figure version
-        continue-on-error: true
-        env:
-          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-        id: last_tag
-        run: |
-          CURRENT=$(uv run semantic-release -v --noop version --print-last-released)
-          echo "Current: $CURRENT"
-
-          VERSION=$(uv run semantic-release -v --noop version --print)
-          echo "new: $VERSION"
-
-          if [ "$CURRENT" == "$VERSION" ]; then
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
-            exit 1
-          fi
-
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
-          exit 0
-
-      - name: Apply new tag
-        if: steps.last_tag.outcome == 'success'
-        env:
-          VERSION: ${{ steps.last_tag.outputs.version }}
-        uses: actions/github-script@v6
-        with:
-          script: |
-            const {VERSION} = process.env
-            const tag = `refs/tags/${VERSION}`
-            await github.rest.git.createRef({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              ref: tag,
-              sha: context.sha
-            })
-
-      - name: Publish to PyPI
-        if: steps.last_tag.outcome == 'success'
-        env:
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-          APPLY_TAG: ${{ steps.last_tag.outputs.version }}
-        run: |
-          uvx --from=toml-cli toml set --toml-path=pyproject.toml project.version $APPLY_TAG
-          uv build
-          uv publish --token $PYPI_TOKEN
-
-      - name: test_installation
-        if: steps.last_tag.outcome == 'success'
-        run: uv run --with runnable --no-project -- python -c "import runnable"
-
-      - name: "Create release"
-        if: steps.last_tag.outcome == 'success'
-        env:
-          RELEASE_TAG: ${{ steps.last_tag.outputs.version }}
-        uses: "actions/github-script@v6"
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-          script: |
-            try {
-              const response = await github.rest.repos.createRelease({
-                draft: false,
-                generate_release_notes: true,
-                name: process.env.RELEASE_TAG,
-                owner: context.repo.owner,
-                prerelease: false,
-                repo: context.repo.repo,
-                tag_name: process.env.RELEASE_TAG,
-              });
-              core.exportVariable('RELEASE_ID', response.data.id);
-              core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url);
-            } catch (error) {
-              core.setFailed(error.message);
-            }

runnable-0.17.0/.mypy.ini

@@ -1,48 +0,0 @@
-[mypy]
-exclude = _*FF.py
-ignore_missing_imports = True
-plugins = pydantic.mypy
-show_error_codes = True
-
-follow_imports = silent
-warn_redundant_casts = True
-warn_unused_ignores = True
-#disallow_any_generics = True # This complicates it a lot
-check_untyped_defs = True
-implicit_reexport = True
-
-# for strict mypy: (this is the tricky one :-))
-#disallow_untyped_defs = True
-
-[pydantic-mypy]
-init_forbid_extra = True
-init_typed = True
-warn_required_dynamic_aliases = True
-warn_untyped_fields = True
-
-[mypy-ruamel.*]
-ignore_missing_imports = True
-
-[mypy-docker.*]
-ignore_missing_imports = True
-
-[mypy-pkg_resources.*]
-ignore_missing_imports = True
-
-[mypy-pydantic.*]
-ignore_missing_imports = True
-
-[mypy-yachalk.*]
-ignore_missing_imports = True
-
-[mypy-stevedore.*]
-ignore_missing_imports = True
-
-[mypy-papermill.*]
-ignore_missing_imports = True
-
-[mypy-click.*]
-ignore_missing_imports = True
-
-[mypy-click_plugins.*]
-ignore_missing_imports = True

runnable-0.17.0/.pre-commit-config.yaml

@@ -1,34 +0,0 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
-# using a non-local repo creates a venv for the pre-commit
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: check-yaml
-        exclude: mkdocs.yml
-      - id: check-added-large-files
-        exclude: assets
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    # Ruff version.
-    rev: "v0.8.3"
-    hooks:
-      - id: ruff
-        args: [--fix, --exit-non-zero-on-fix]
-        exclude: ^tests
-      - id: ruff-format
-        exclude: ^tests
-  - repo: https://github.com/jorisroovers/gitlint
-    rev: v0.19.1
-    hooks:
-      - id: gitlint
-        args:
-          [
-            --contrib=CT1,
-            --ignore=title-trailing-punctuation,
-            --ignore=T3,
-            --ignore=B6,
-            --msg-filename,
-          ]

runnable-0.17.0/.python-version

@@ -1 +0,0 @@
-3.10