argo-kedro 0.1.1__tar.gz

argo_kedro-0.1.1/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

argo_kedro-0.1.1/.python-version

@@ -0,0 +1 @@
+3.11

argo_kedro-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 kedro-argo contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

argo_kedro-0.1.1/MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include LICENSE
+recursive-include kedro_argo *.tmpl

argo_kedro-0.1.1/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.4
+Name: argo-kedro
+Version: 0.1.1
+Summary: Kedro plugin for running pipelines on Argo Workflows
+Author-email: Laurens Vijnck <laurens@everycure.org>, Nelson Alfonso <nelson@everycure.org>
+License: MIT
+Project-URL: Homepage, https://github.com/everycure-org/kedro-argo
+Project-URL: Repository, https://github.com/everycure-org/kedro-argo
+Project-URL: Issues, https://github.com/everycure-org/kedro-argo/issues
+Keywords: kedro,argo,workflows,argo workflows,kubernetes,pipeline
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Framework :: Kedro
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: kedro
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: jinja2>=3.0.0
+Dynamic: license-file
+
+# kedro-argo
+
+A Kedro plugin for running pipelines on Argo Workflows in Kubernetes clusters.
+
+## Features
+
+- ✅ Submit Kedro pipelines to Argo Workflows
+- ✅ Fuse pipeline nodes for optimized execution
+- ✅ Memory dataset management during fusing
+- 🚧 Image building guidelines
+- 🚧 Hardware configuration support
+- 🚧 Custom template registration
+
+## Installation
+
+```bash
+pip install kedro-argo
+```
+
+## Prerequisites
+
+- Kubernetes cluster with Argo Workflows installed
+- `kubeconfig` configured correctly
+- Argo Workflows namespace set up
+
+## Quick Start
+
+### 1. Configure your Kedro project
+
+Add Argo configuration to your Kedro project (e.g., in `conf/base/argo.yml`).
+
+### 2. Use the CLI
+
+```bash
+# Submit pipeline to Argo
+kedro argo submit
+
+# Other commands
+kedro argo --help
+```
+
+## Usage
+
+- Ensure kubeconfig is set correctly.
+- Ensure Argo installed correctly on cluster and namespace present.
+
+## Project setup
+
+- Distinguish new cloud environment for running remotely
+
+## Current Assumptions
+
+- For image building, we assume the user enters the path to a valid GAR repository, which the cluster is assumed to have permissions to
+- We work with the `latest` tag only, as Argo ensures to apply `imagePullPolicy: Always` in that case
+
+## Known Issues
+
+- Installing both the k8s and gcfs libs may cause authentication issues
+- Current workaround: `uv pip uninstall gcsfs`
+
+## Development Roadmap
+
+### Horizon 1
+- [x] Package and deploy to pypi
+- [x] Submitting to cluster
+- [ ] Provide image building guidelines
+    - [ ] Kedro MLFlow plugin shows nice way of consuming "config", argo.yml
+
+### Horizon 2
+- [ ] Hardware configuration
+
+### Horizon 3 (making feature complete)
+- [x] Fusing pipelines
+- [x] Set memory datasets during fusing
+- [ ] Allow registering custom templates, e.g., neo4j
+- [ ] Allow environment variables
+
+### Horizon 4 (Open sourcing)
+- [ ] Complete feature set
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+MIT License - see LICENSE file for details

argo_kedro-0.1.1/README.md

@@ -0,0 +1,87 @@
+# kedro-argo
+
+A Kedro plugin for running pipelines on Argo Workflows in Kubernetes clusters.
+
+## Features
+
+- ✅ Submit Kedro pipelines to Argo Workflows
+- ✅ Fuse pipeline nodes for optimized execution
+- ✅ Memory dataset management during fusing
+- 🚧 Image building guidelines
+- 🚧 Hardware configuration support
+- 🚧 Custom template registration
+
+## Installation
+
+```bash
+pip install kedro-argo
+```
+
+## Prerequisites
+
+- Kubernetes cluster with Argo Workflows installed
+- `kubeconfig` configured correctly
+- Argo Workflows namespace set up
+
+## Quick Start
+
+### 1. Configure your Kedro project
+
+Add Argo configuration to your Kedro project (e.g., in `conf/base/argo.yml`).
+
+### 2. Use the CLI
+
+```bash
+# Submit pipeline to Argo
+kedro argo submit
+
+# Other commands
+kedro argo --help
+```
+
+## Usage
+
+- Ensure kubeconfig is set correctly.
+- Ensure Argo installed correctly on cluster and namespace present.
+
+## Project setup
+
+- Distinguish new cloud environment for running remotely
+
+## Current Assumptions
+
+- For image building, we assume the user enters the path to a valid GAR repository, which the cluster is assumed to have permissions to
+- We work with the `latest` tag only, as Argo ensures to apply `imagePullPolicy: Always` in that case
+
+## Known Issues
+
+- Installing both the k8s and gcfs libs may cause authentication issues
+- Current workaround: `uv pip uninstall gcsfs`
+
+## Development Roadmap
+
+### Horizon 1
+- [x] Package and deploy to pypi
+- [x] Submitting to cluster
+- [ ] Provide image building guidelines
+    - [ ] Kedro MLFlow plugin shows nice way of consuming "config", argo.yml
+
+### Horizon 2
+- [ ] Hardware configuration
+
+### Horizon 3 (making feature complete)
+- [x] Fusing pipelines
+- [x] Set memory datasets during fusing
+- [ ] Allow registering custom templates, e.g., neo4j
+- [ ] Allow environment variables
+
+### Horizon 4 (Open sourcing)
+- [ ] Complete feature set
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+MIT License - see LICENSE file for details

argo_kedro-0.1.1/argo_kedro.egg-info/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.4
+Name: argo-kedro
+Version: 0.1.1
+Summary: Kedro plugin for running pipelines on Argo Workflows
+Author-email: Laurens Vijnck <laurens@everycure.org>, Nelson Alfonso <nelson@everycure.org>
+License: MIT
+Project-URL: Homepage, https://github.com/everycure-org/kedro-argo
+Project-URL: Repository, https://github.com/everycure-org/kedro-argo
+Project-URL: Issues, https://github.com/everycure-org/kedro-argo/issues
+Keywords: kedro,argo,workflows,argo workflows,kubernetes,pipeline
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Framework :: Kedro
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: kedro
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: jinja2>=3.0.0
+Dynamic: license-file
+
+# kedro-argo
+
+A Kedro plugin for running pipelines on Argo Workflows in Kubernetes clusters.
+
+## Features
+
+- ✅ Submit Kedro pipelines to Argo Workflows
+- ✅ Fuse pipeline nodes for optimized execution
+- ✅ Memory dataset management during fusing
+- 🚧 Image building guidelines
+- 🚧 Hardware configuration support
+- 🚧 Custom template registration
+
+## Installation
+
+```bash
+pip install kedro-argo
+```
+
+## Prerequisites
+
+- Kubernetes cluster with Argo Workflows installed
+- `kubeconfig` configured correctly
+- Argo Workflows namespace set up
+
+## Quick Start
+
+### 1. Configure your Kedro project
+
+Add Argo configuration to your Kedro project (e.g., in `conf/base/argo.yml`).
+
+### 2. Use the CLI
+
+```bash
+# Submit pipeline to Argo
+kedro argo submit
+
+# Other commands
+kedro argo --help
+```
+
+## Usage
+
+- Ensure kubeconfig is set correctly.
+- Ensure Argo installed correctly on cluster and namespace present.
+
+## Project setup
+
+- Distinguish new cloud environment for running remotely
+
+## Current Assumptions
+
+- For image building, we assume the user enters the path to a valid GAR repository, which the cluster is assumed to have permissions to
+- We work with the `latest` tag only, as Argo ensures to apply `imagePullPolicy: Always` in that case
+
+## Known Issues
+
+- Installing both the k8s and gcfs libs may cause authentication issues
+- Current workaround: `uv pip uninstall gcsfs`
+
+## Development Roadmap
+
+### Horizon 1
+- [x] Package and deploy to pypi
+- [x] Submitting to cluster
+- [ ] Provide image building guidelines
+    - [ ] Kedro MLFlow plugin shows nice way of consuming "config", argo.yml
+
+### Horizon 2
+- [ ] Hardware configuration
+
+### Horizon 3 (making feature complete)
+- [x] Fusing pipelines
+- [x] Set memory datasets during fusing
+- [ ] Allow registering custom templates, e.g., neo4j
+- [ ] Allow environment variables
+
+### Horizon 4 (Open sourcing)
+- [ ] Complete feature set
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+MIT License - see LICENSE file for details

argo_kedro-0.1.1/argo_kedro.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+.gitignore
+.python-version
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+uv.lock
+argo_kedro.egg-info/PKG-INFO
+argo_kedro.egg-info/SOURCES.txt
+argo_kedro.egg-info/dependency_links.txt
+argo_kedro.egg-info/entry_points.txt
+argo_kedro.egg-info/requires.txt
+argo_kedro.egg-info/top_level.txt
+kedro_argo/framework/__init__.py
+kedro_argo/framework/cli/__init__.py
+kedro_argo/framework/cli/cli.py
+kedro_argo/pipeline/__init__.py
+kedro_argo/pipeline/fused_pipeline.py
+kedro_argo/runners/__init__.py
+kedro_argo/runners/fuse_runner.py
+kedro_argo/templates/argo_wf_spec.tmpl

argo_kedro-0.1.1/argo_kedro.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

argo_kedro-0.1.1/argo_kedro.egg-info/entry_points.txt

@@ -0,0 +1,5 @@
+[kedro.global_commands]
+run = argo_kedro.framework.cli.cli:cli
+
+[kedro.project_commands]
+argo = argo_kedro.framework.cli.cli:commands

argo_kedro-0.1.1/argo_kedro.egg-info/requires.txt

@@ -0,0 +1,3 @@
+kedro
+pyyaml>=6.0.2
+jinja2>=3.0.0

argo_kedro-0.1.1/argo_kedro.egg-info/top_level.txt

@@ -0,0 +1 @@
+

argo_kedro-0.1.1/kedro_argo/framework/cli/cli.py

@@ -0,0 +1,242 @@
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Iterable
+from logging import getLogger
+
+import click
+import yaml
+from kubernetes import config
+from kubernetes.dynamic import DynamicClient
+from jinja2 import Environment, FileSystemLoader
+from kedro.framework.cli.utils import CONTEXT_SETTINGS, KedroCliError
+from kedro.framework.session import KedroSession
+from kedro.framework.cli.project import (
+    ASYNC_ARG_HELP,
+    CONF_SOURCE_HELP,
+    FROM_INPUTS_HELP,
+    FROM_NODES_HELP,
+    LOAD_VERSION_HELP,
+    NODE_ARG_HELP,
+    PARAMS_ARG_HELP,
+    PIPELINE_ARG_HELP,
+    RUNNER_ARG_HELP,
+    TAG_ARG_HELP,
+    TO_NODES_HELP,
+    TO_OUTPUTS_HELP,
+    project_group,
+)
+from kedro.framework.project import pipelines as kedro_pipelines
+from kedro.pipeline import Pipeline
+from kedro.pipeline.node import Node
+from kedro.runner.sequential_runner import SequentialRunner
+from kedro_argo.runners.fuse_runner import FusedRunner
+
+LOGGER = getLogger(__name__)
+ARGO_TEMPLATES_DIR_PATH = Path(__file__).parent.parent.parent / "templates"
+
+
+@click.group(context_settings=CONTEXT_SETTINGS)
+def cli():
+    pass
+
+@cli.command(name="run")
+@click.option("--pipeline", "-p", type=str, default="__default__", help="Name of the pipeline to execute")
+@click.option("--env", "-e", type=str, default=None, help="Kedro environment to run the pipeline in")
+@click.option("--config", "-c", type=str, multiple=True, help="Extra config to pass to KedroContext")
+@click.option("--params", type=str, multiple=True, help="Override parameters")
+@click.option("--tags", "-t", type=str, multiple=True, help=TAG_ARG_HELP)
+@click.option("--nodes", "-n", type=str, multiple=True, help="Run only nodes with specified names")
+@click.option("--to-nodes", type=str, multiple=True, help="Run a sub-pipeline up to certain nodes")
+@click.option("--from-nodes", type=str, multiple=True, help="Run a sub-pipeline starting from certain nodes")
+@click.option("--from-inputs", type=str, multiple=True, help="Run a sub-pipeline starting from nodes that produce these inputs")
+@click.option("--to-outputs", type=str, multiple=True, help="Run a sub-pipeline up to nodes that produce these outputs")
+@click.option("--load-version", type=str, multiple=True, help="Specify a particular dataset version")
+@click.option("--namespaces", type=str, multiple=True, help="Namespaces of the pipeline")
+@click.pass_obj
+def _run_command_impl(
+    ctx,
+    pipeline: str,
+    env: str,
+    config: tuple,
+    params: tuple,
+    tags: list[str],
+    nodes: tuple,
+    to_nodes: tuple,
+    from_nodes: tuple,
+    from_inputs: tuple,
+    to_outputs: tuple,
+    load_version: tuple,
+    namespaces: Iterable[str],
+):    
+    """Run the pipeline with the FusedRunner."""
+
+    LOGGER.warning(f"Using plugin entrypoint")
+    
+    load_versions = None
+    if load_version:
+        load_versions = {}
+        for version_spec in load_version:
+            if ":" in version_spec:
+                dataset, version = version_spec.split(":", 1)
+                load_versions[dataset] = version
+
+    conf_source = getattr(ctx, "conf_source", None)
+    env_value = env or getattr(ctx, "env", None)
+
+    with KedroSession.create(
+        env=env_value,
+        conf_source=conf_source,
+    ) as session:
+
+        session.run(
+            pipeline_name=pipeline,
+            tags=tags,
+            runner=FusedRunner(pipeline_name=pipeline),
+            node_names=list(nodes) if nodes else None,
+            from_nodes=list(from_nodes) if from_nodes else None,
+            to_nodes=list(to_nodes) if to_nodes else None,
+            from_inputs=list(from_inputs) if from_inputs else None,
+            to_outputs=list(to_outputs) if to_outputs else None,
+            load_versions=load_versions,
+            namespaces=namespaces,
+        )
+
+@click.group(name="argo")
+def commands():
+    pass
+
+@commands.command(name="submit")
+@click.option("--pipeline", "-p", type=str, default="__default__", help="Specify which pipeline to execute")
+@click.option("--environment", "-e", type=str, default="base", help="Kedro environment to execute in")
+@click.option("--image", type=str, required=True, help="Image to execute")
+@click.option("--namespace", "-n", type=str, required=True, help="Namespace to execute in")
+@click.pass_obj
+def submit(
+    ctx,
+    pipeline: str,
+    image: str,
+    namespace: str,
+    environment: str
+):
+    """Submit the pipeline to Argo."""
+    LOGGER.info("Loading spec template..")
+
+    loader = FileSystemLoader(searchpath=ARGO_TEMPLATES_DIR_PATH)
+    template_env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)
+    template = template_env.get_template("argo_wf_spec.tmpl")
+
+    pipeline_tasks = get_argo_dag(kedro_pipelines[pipeline])
+
+    LOGGER.info("Rendering Argo spec...")
+
+    # Render the template
+    rendered_template = template.render(
+        pipeline_tasks=[task.to_dict() for task in pipeline_tasks.values()],
+        pipeline_name=pipeline,
+        image=image,
+        namespace=namespace,
+        environment=environment
+    )
+
+    # Load as yaml
+    yaml_data = yaml.safe_load(rendered_template)
+    yaml_without_anchors = yaml.dump(yaml_data, sort_keys=False, default_flow_style=False)
+    save_argo_template(
+        yaml_without_anchors,
+    )
+
+    # Use kubeconfig to submit to kubernetes
+    config.load_kube_config()
+    client = DynamicClient(config.new_client_from_config())
+
+    resource = client.resources.get(
+        api_version=yaml_data["apiVersion"],
+        kind=yaml_data["kind"],
+    )
+
+    resource.create(
+        body=yaml_data,
+        namespace=namespace
+    )
+
+
+def save_argo_template(argo_template: str) -> str:
+    file_path = Path("templates") / "argo-workflow-template.yml"
+    with open(file_path, "w") as f:
+        f.write(argo_template)
+    return str(file_path)
+
+
+class ArgoTask:
+    """Class to model an Argo task.
+
+    Argo's operating model slightly differs from Kedro's, i.e., while Kedro uses dataset
+    dependecies to model relationships, Argo uses task dependencies."""
+
+    def __init__(self, node: Node):
+        self._node = node
+        self._parents = []
+
+    @property
+    def node(self):
+        return self._node
+
+    def add_parents(self, nodes: List[Node]):
+        self._parents.extend(nodes)
+
+    def to_dict(self):
+        return {
+            "name": clean_name(self._node.name),
+            "nodes": self._node.name,
+            "deps": [clean_name(parent.name) for parent in sorted(self._parents)],
+        }
+
+
+def get_argo_dag(pipeline: Pipeline) -> List[Dict[str, Any]]:
+    """Function to convert the Kedro pipeline into Argo Tasks. The function
+    iterates the nodes of the pipeline and generates Argo tasks with dependencies.
+    These dependencies are inferred based on the input and output datasets for
+    each node.
+
+    NOTE: This function is now agnostic to the fact that nodes might be fused. The nodes
+    returned as part of the pipeline may optionally contain FusedNodes, which have correct
+    inputs and outputs for the perspective of the Argo Task.
+    """
+    tasks = {}
+
+    # The `grouped_nodes` property returns the nodes list, in a toplogical order,
+    # allowing us to easily translate the Kedro DAG to an Argo WF.
+    for group in pipeline.grouped_nodes:
+        for target_node in group:
+            task = ArgoTask(target_node)
+            task.add_parents(
+                [
+                    parent.node
+                    for parent in tasks.values()
+                    if set(clean_dependencies(target_node.inputs)) & set(clean_dependencies(parent.node.outputs))
+                ]
+            )
+
+            tasks[target_node.name] = task
+
+    return tasks
+
+
+def clean_name(name: str) -> str:
+    """Function to clean the node name.
+
+    Args:
+        name: name of the node
+    Returns:
+        Clean node name, according to Argo's requirements
+    """
+    return re.sub(r"[\W_]+", "-", name).strip("-")
+
+
+def clean_dependencies(elements) -> List[str]:
+    """Function to clean node dependencies.
+
+    Operates by removing `params:` from the list and dismissing
+    the transcoding operator.
+    """
+    return [el.split("@")[0] for el in elements if not el.startswith("params:")]

argo_kedro-0.1.1/kedro_argo/pipeline/__init__.py

@@ -0,0 +1,3 @@
+from .fused_pipeline import FusedPipeline
+
+__all__ = ["FusedPipeline", ]