kblaunch 0.1.0__tar.gz

kblaunch-0.1.0/.github/workflows/release.yaml

@@ -0,0 +1,21 @@
+# publish.yml
+name: "Publish"
+on:
+  release:
+    types: ["published"]
+
+jobs:
+  pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    # Environment and permissions trusted publishing.
+    environment:
+      # Create this environment in the GitHub repository under Settings -> Environments
+      name: release
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - run: uv build
+      - run: uv publish --trusted-publishing always

kblaunch-0.1.0/.github/workflows/test.yaml

@@ -0,0 +1,26 @@
+# publish.yml
+name: "Test"
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  run:
+    name: "Run tests"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Run tests
+        run: uv run --extra test pytest

kblaunch-0.1.0/.gitignore

@@ -0,0 +1,173 @@
+# Byte-compiled / optimized / DLL files
+.python-version
+.vscode/
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc

kblaunch-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Gautier Dagan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

kblaunch-0.1.0/PKG-INFO

@@ -0,0 +1,126 @@
+Metadata-Version: 2.4
+Name: kblaunch
+Version: 0.1.0
+Summary: A CLI tool for launching Kubernetes jobs in EIDF
+Author-email: Gautier Dagan <gautier.dagan@ed.ac.uk>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.9
+Requires-Dist: kubejobs>=0.4.5
+Requires-Dist: kubernetes>=31.0.0
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: typer>=0.15.1
+Provides-Extra: test
+Requires-Dist: pytest-cov>=4.1.0; extra == 'test'
+Requires-Dist: pytest-mock>=3.10.0; extra == 'test'
+Requires-Dist: pytest>=7.0; extra == 'test'
+Description-Content-Type: text/markdown
+
+# kblaunch
+
+[![Test](https://github.com/gautierdag/kblaunch/actions/workflows/test.yaml/badge.svg)](https://github.com/gautierdag/kblaunch/actions/workflows/test.yaml)
+![Python Version](https://img.shields.io/badge/python-3.9+-blue)
+![Ruff](https://img.shields.io/badge/linter-ruff-blue)
+[![PyPI Version](https://img.shields.io/pypi/v/kblaunch)](https://pypi.org/project/kblaunch/)
+
+A CLI tool for launching Kubernetes jobs with environment variable and secret management.
+
+## Installation
+
+```bash
+pip install git+https://github.com/gautierdag/kblaunch.git
+```
+
+## Usage
+
+### Basic Usage
+
+Launch a simple job:
+
+```bash
+kblaunch \
+    --email your.email@ed.ac.uk \
+    --job-name myjob \
+    --command "python script.py"
+```
+
+### With Environment Variables
+
+1. From local environment:
+
+    ```bash
+    kblaunch \
+        --job-name myjob \
+        --command "python script.py" \
+        --local-env-vars PATH \
+        --local-env-vars PYTHONPATH
+    ```
+
+2. From Kubernetes secrets:
+
+    ```bash
+    kblaunch \
+        --job-name myjob \
+        --command "python script.py" \
+        --secrets-env-vars mysecret1 \
+        --secrets-env-vars mysecret2
+    ```
+
+3. From .env file:
+
+    ```bash
+    kblaunch \
+        --job-name myjob \
+        --command "python script.py" \
+        --load-dotenv
+    ```
+
+### GPU Jobs
+
+Specify GPU requirements:
+
+```bash
+kblaunch \
+    --job-name gpu-job \
+    --command "python train.py" \
+    --gpu-limit 2 \
+    --gpu-product "NVIDIA-A100-80GB-PCIe"
+```
+
+### Interactive Mode
+
+Launch an interactive job:
+
+```bash
+kblaunch \
+    --job-name interactive \
+    --interactive
+```
+
+## Options
+
+- `--email`: User email [required]
+- `--job-name`: Name of the Kubernetes job [required]
+- `--docker-image`: Docker image (default: "nvcr.io/nvidia/cuda:12.0.0-devel-ubuntu22.04")
+- `--namespace`: Kubernetes namespace (default: "informatics")
+- `--queue-name`: Kueue queue name
+- `--interactive`: Run in interactive mode (default: False)
+- `--command`: Command to run in the container [required]
+- `--cpu-request`: CPU request (default: "1")
+- `--ram-request`: RAM request (default: "8Gi")
+- `--gpu-limit`: GPU limit (default: 1)
+- `--gpu-product`: GPU product (default: "NVIDIA-A100-80GB-PCIe")
+- `--secrets-env-vars`: List of secret environment variables
+- `--local-env-vars`: List of local environment variables
+- `--load-dotenv`: Load environment variables from .env file (default: True)
+
+## Features
+
+- Kubernetes job management
+- Environment variable handling from multiple sources
+- Kubernetes secrets integration
+- GPU job support
+- Interactive mode
+- Automatic job cleanup
+- Slack notifications (when configured)

kblaunch-0.1.0/README.md

@@ -0,0 +1,107 @@
+# kblaunch
+
+[![Test](https://github.com/gautierdag/kblaunch/actions/workflows/test.yaml/badge.svg)](https://github.com/gautierdag/kblaunch/actions/workflows/test.yaml)
+![Python Version](https://img.shields.io/badge/python-3.9+-blue)
+![Ruff](https://img.shields.io/badge/linter-ruff-blue)
+[![PyPI Version](https://img.shields.io/pypi/v/kblaunch)](https://pypi.org/project/kblaunch/)
+
+A CLI tool for launching Kubernetes jobs with environment variable and secret management.
+
+## Installation
+
+```bash
+pip install git+https://github.com/gautierdag/kblaunch.git
+```
+
+## Usage
+
+### Basic Usage
+
+Launch a simple job:
+
+```bash
+kblaunch \
+    --email your.email@ed.ac.uk \
+    --job-name myjob \
+    --command "python script.py"
+```
+
+### With Environment Variables
+
+1. From local environment:
+
+    ```bash
+    kblaunch \
+        --job-name myjob \
+        --command "python script.py" \
+        --local-env-vars PATH \
+        --local-env-vars PYTHONPATH
+    ```
+
+2. From Kubernetes secrets:
+
+    ```bash
+    kblaunch \
+        --job-name myjob \
+        --command "python script.py" \
+        --secrets-env-vars mysecret1 \
+        --secrets-env-vars mysecret2
+    ```
+
+3. From .env file:
+
+    ```bash
+    kblaunch \
+        --job-name myjob \
+        --command "python script.py" \
+        --load-dotenv
+    ```
+
+### GPU Jobs
+
+Specify GPU requirements:
+
+```bash
+kblaunch \
+    --job-name gpu-job \
+    --command "python train.py" \
+    --gpu-limit 2 \
+    --gpu-product "NVIDIA-A100-80GB-PCIe"
+```
+
+### Interactive Mode
+
+Launch an interactive job:
+
+```bash
+kblaunch \
+    --job-name interactive \
+    --interactive
+```
+
+## Options
+
+- `--email`: User email [required]
+- `--job-name`: Name of the Kubernetes job [required]
+- `--docker-image`: Docker image (default: "nvcr.io/nvidia/cuda:12.0.0-devel-ubuntu22.04")
+- `--namespace`: Kubernetes namespace (default: "informatics")
+- `--queue-name`: Kueue queue name
+- `--interactive`: Run in interactive mode (default: False)
+- `--command`: Command to run in the container [required]
+- `--cpu-request`: CPU request (default: "1")
+- `--ram-request`: RAM request (default: "8Gi")
+- `--gpu-limit`: GPU limit (default: 1)
+- `--gpu-product`: GPU product (default: "NVIDIA-A100-80GB-PCIe")
+- `--secrets-env-vars`: List of secret environment variables
+- `--local-env-vars`: List of local environment variables
+- `--load-dotenv`: Load environment variables from .env file (default: True)
+
+## Features
+
+- Kubernetes job management
+- Environment variable handling from multiple sources
+- Kubernetes secrets integration
+- GPU job support
+- Interactive mode
+- Automatic job cleanup
+- Slack notifications (when configured)

kblaunch-0.1.0/kblaunch/cli.py

@@ -0,0 +1,195 @@
+import base64
+import os
+
+import typer
+from kubejobs.jobs import KubernetesJob, KueueQueue
+from kubernetes import client, config
+from loguru import logger
+
+app = typer.Typer()
+
+
+def check_if_completed(job_name: str, namespace: str = "informatics") -> bool:
+    # Load the kube config
+    config.load_kube_config()
+
+    # Create an instance of the API class
+    api = client.BatchV1Api()
+
+    job_exists = False
+    is_completed = True
+
+    # Check if the job exists in the specified namespace
+    jobs = api.list_namespaced_job(namespace)
+    if job_name in {job.metadata.name for job in jobs.items}:
+        job_exists = True
+
+    if job_exists is True:
+        job = api.read_namespaced_job(job_name, namespace)
+        is_completed = False
+
+        # Check the status conditions
+        if job.status.conditions:
+            for condition in job.status.conditions:
+                if condition.type == "Complete" and condition.status == "True":
+                    is_completed = True
+                elif condition.type == "Failed" and condition.status == "True":
+                    logger.error(f"Job {job_name} has failed.")
+        else:
+            logger.info(f"Job {job_name} still running or status is unknown.")
+
+        if is_completed:
+            api_res = api.delete_namespaced_job(
+                name=job_name,
+                namespace=namespace,
+                body=client.V1DeleteOptions(propagation_policy="Foreground"),
+            )
+            logger.info(f"Job '{job_name}' deleted. Status: {api_res.status}")
+    return is_completed
+
+
+def send_message_command(env_vars: dict) -> str:
+    if "SLACK_WEBHOOK" not in env_vars:
+        logger.debug("SLACK_WEBHOOK not found in env_vars.")
+        return ""
+    webhook = env_vars["SLACK_WEBHOOK"]
+    return (
+        """curl -X POST -H 'Content-type: application/json' --data '{"text":"Job started in '"$POD_NAME"'"}' """
+        + webhook
+        + " ; "
+    )
+
+
+def get_env_vars(
+    local_env_vars: list[str],
+    secrets_env_vars: list[str],
+    load_dotenv: bool = False,
+    namespace: str = "informatics",
+) -> dict:
+    """Get environment variables from local environment and secrets."""
+
+    if load_dotenv:
+        from dotenv import load_dotenv
+
+        load_dotenv()
+
+    env_vars = {}
+    for var_name in local_env_vars:
+        value = os.getenv(var_name)
+        if value is not None:
+            env_vars[var_name] = value
+        else:
+            logger.warning(
+                f"Environment variable {var_name} not found in local environment"
+            )
+
+    for secret_name in secrets_env_vars:
+        v1 = client.CoreV1Api()
+        try:
+            secret = v1.read_namespaced_secret(name=secret_name, namespace=namespace)
+            for key, value in secret.data.items():
+                decoded_value = base64.b64decode(value).decode("utf-8")
+                if key in env_vars:
+                    logger.warning(f"Key {key} already set in env_vars.")
+                env_vars[key] = decoded_value
+        except Exception as e:
+            logger.error(f"Error reading secret {secret_name}: {e}")
+            raise e
+
+    return env_vars
+
+
+def export_env_vars(env_vars: dict) -> str:
+    """Export environment variables."""
+    cmd = ""
+    for key, value in env_vars.items():
+        cmd += f" export {key}='{value}' &&"
+    cmd = cmd.strip(" &&") + " ; "
+    return cmd
+
+
+@app.command()
+def launch(
+    email: str = typer.Option(..., help="User email"),
+    job_name: str = typer.Option(..., help="Name of the Kubernetes job"),
+    docker_image: str = typer.Option(
+        "nvcr.io/nvidia/cuda:12.0.0-devel-ubuntu22.04", help="Docker image"
+    ),
+    namespace: str = typer.Option("informatics", help="Kubernetes namespace"),
+    queue_name: str = typer.Option(KueueQueue.INFORMATICS, help="Kueue queue name"),
+    interactive: bool = typer.Option(False, help="Run in interactive mode"),
+    command: str = typer.Option(..., help="Command to run in the container"),
+    cpu_request: str = typer.Option("1", help="CPU request"),
+    ram_request: str = typer.Option("8Gi", help="RAM request"),
+    gpu_limit: int = typer.Option(1, help="GPU limit"),
+    gpu_product: str = typer.Option("NVIDIA-A100-80GB-PCIe", help="GPU product"),
+    secrets_env_vars: list[str] = typer.Option(
+        [],  # Use empty list as default instead of None
+        help="List of secret environment variables to export to the container",
+    ),
+    local_env_vars: list[str] = typer.Option(
+        [],  # Use empty list as default instead of None
+        help="List of local environment variables to export to the container",
+    ),
+    load_dotenv: bool = typer.Option(
+        True, help="Load environment variables from .env file"
+    ),
+):
+    """Launch a Kubernetes job with the specified configuration."""
+
+    is_completed = check_if_completed(job_name, namespace=namespace)
+
+    if is_completed is True:
+        logger.info(f"Job '{job_name}' is completed. Launching a new job.")
+
+        if interactive:
+            cmd = "while true; do sleep 60; done;"
+        else:
+            cmd = command
+            logger.info(f"Command: {cmd}")
+
+        # Get local environment variables
+        env_vars = get_env_vars(
+            local_env_vars=local_env_vars,
+            secrets_env_vars=secrets_env_vars,
+            load_dotenv=load_dotenv,
+            namespace=namespace,
+        )
+
+        logger.info(f"Creating job for: {cmd}")
+        job = KubernetesJob(
+            name=job_name,
+            cpu_request=cpu_request,
+            ram_request=ram_request,
+            image=docker_image,
+            gpu_type="nvidia.com/gpu",
+            gpu_limit=gpu_limit,
+            gpu_product=gpu_product,
+            backoff_limit=0,
+            command=["/bin/bash", "-c", "--"],
+            args=[export_env_vars(env_vars) + send_message_command(env_vars) + cmd],
+            user_email=email,
+            namespace=namespace,
+            kueue_queue_name=queue_name,
+            secret_env_vars=env_vars,
+            volume_mounts={
+                "nfs": {"mountPath": "/nfs", "server": "10.24.1.255", "path": "/"}
+            },
+        )
+
+        job_yaml = job.generate_yaml()
+        logger.info(job_yaml)
+
+        # Run the Job on the Kubernetes cluster
+        job.run()
+    else:
+        logger.info(f"Job '{job_name}' is still running.")
+
+
+def cli():
+    """Entry point for the application"""
+    app()
+
+
+if __name__ == "__main__":
+    cli()