airoh 0.1.2__py3-none-any.whl

airoh/containers.py

@@ -0,0 +1,160 @@
+# src/airoh/containers.py
+import os
+import shutil
+import tempfile
+import gzip
+from pathlib import Path
+from invoke import task
+
+def _set_image(c, image=None):
+    """
+    Resolve the Docker image name from parameter or invoke.yaml.
+    """
+    image = image or c.config.get("docker_image")
+    if not image:
+        raise ValueError("No Docker image specified. Please set docker_image in invoke.yaml or pass it explicitly.")
+    return image
+
+@task
+def docker_build(c, image=None):
+    """
+    Build the Docker image from the Dockerfile in the project root.
+    """
+    image = _set_image(c, image)
+    print(f"🐳 Building Docker image: {image}")
+    c.run(f"docker build -t {image} .")
+
+@task
+def docker_archive(c, image=None):
+    """
+    Save the Docker image to a compressed archive for Zenodo or transport.
+    """
+    image = _set_image(c, image)
+    output = f"{image}.tar.gz"
+    print(f"📦 Archiving Docker image '{image}' to {output}...")
+    c.run(f"docker save {image} | gzip > {output}")
+    print("🪦 Archive complete.")
+
+@task
+def docker_setup(c, url=None, image=None):
+    """
+    Download and load the prebuilt Docker image from Zenodo.
+    """
+    image = _set_image(c, image)
+    if not url:
+        url = c.config.get("docker_archive")
+        if not url:
+            raise ValueError("No archive URL provided. Set docker_archive in invoke.yaml or pass --url.")
+
+    output = f"{image}.tar.gz"
+    if not os.path.exists(output):
+        print(f"📥 Downloading container from {url}...")
+        c.run(f"wget -O {output} '{url}'")
+    else:
+        print(f"📦 Container archive already exists: {output}")
+
+    print("🐳 Loading Docker image...")
+    c.run(f"gunzip -c {output} | docker load")
+    print("✨ Container setup complete.")
+
+def _ensure_docker_image_loaded(c, image, image_tar):
+    """
+    Ensure the specified Docker image is available. If not, try to load it from a .tar or .tar.gz.
+    """
+    if not shutil.which("docker"):
+        raise RuntimeError("❌ Docker is not installed or not in PATH. Please install Docker.")
+
+    result = c.run(f"docker images -q {image}", hide=True, warn=True)
+    if result.stdout.strip():
+        c.run(f"docker tag {image} {image}:latest", warn=True)
+        return
+
+    print(f"📦 Docker image '{image}' not found. Attempting to load from {image_tar}...")
+
+    image_tar = Path(image_tar)
+    if not image_tar.exists():
+        raise FileNotFoundError(f"❌ Docker image file not found: {image_tar}")
+
+    if image_tar.suffixes[-2:] == ['.tar', '.gz']:
+        print(f"🖜️ Extracting {image_tar}...")
+        with tempfile.NamedTemporaryFile(suffix=".tar", delete=False) as temp_tar:
+            with gzip.open(image_tar, "rb") as f_in:
+                shutil.copyfileobj(f_in, temp_tar)
+            temp_tar_path = temp_tar.name
+        c.run(f"docker load -i {temp_tar_path}")
+        os.remove(temp_tar_path)
+    elif image_tar.suffix == ".tar":
+        c.run(f"docker load -i {image_tar}")
+    else:
+        raise ValueError("❌ Unsupported container format. Use .tar or .tar.gz")
+
+    c.run(f"docker tag {image} {image}:latest", warn=True)
+
+@task
+def docker_run(c, task, args=""):
+    """
+    Run an invoke task inside the Docker container.
+
+    Args:
+        task (str): the invoke task to run
+        args (str): any additional CLI args to pass to the task
+    """
+    image = c.config.get("docker_image")
+    image_tar = f"{image}.tar.gz"
+
+    _ensure_docker_image_loaded(c, image, image_tar)
+
+    hostdir = os.getcwd()
+    workdir = "/home/jovyan/work"
+    cmd = f"invoke {task} {args}"
+    docker_cmd = f'docker run --rm -v {hostdir}:{workdir} -w {workdir} {image} {cmd}'
+
+    print(f"🐳 Running inside container: {cmd}")
+    c.run(docker_cmd)
+
+@task
+def apptainer_archive(c, image=None):
+    """
+    Archive the Apptainer (Singularity) image from a Docker image using Docker daemon.
+    Builds the .sif file if not present.
+    """
+    image = _set_image(c, image)
+    sif_path = Path(f"{image}.sif")
+
+    if not shutil.which("apptainer"):
+        raise RuntimeError("❌ Apptainer is not installed or not in PATH. Please install it.")
+
+    if sif_path.exists():
+        print(f"✅ Apptainer image already exists at {sif_path}. Skipping build.")
+        return
+
+    if not shutil.which("docker"):
+        raise RuntimeError("❌ Docker is required to build from Docker image. Please install it.")
+
+    _ensure_docker_image_loaded(c, image, f"{image}.tar.gz")
+    print(f"🧪 Building Apptainer image {sif_path} from Docker image {image}:latest...")
+    c.run(f"apptainer build {sif_path} docker-daemon:{image}:latest")
+    print("✅ Apptainer image build complete.")
+
+@task
+def apptainer_run(c, task, args=""):
+    """
+    Run an invoke task inside the Apptainer container.
+
+    Args:
+        task (str): the invoke task to run
+        args (str): any additional CLI args to pass to the task
+    """
+    docker_image = c.config.get("docker_image")
+    sif_path = Path(f"{docker_image}.sif")
+
+    if not sif_path.exists():
+        raise FileNotFoundError(f"❌ Apptainer image not found: {sif_path}")
+
+    hostdir = os.getcwd()
+    workdir = "/home/jovyan/work"
+    cmd = f"invoke {task} {args}"
+    apptainer_cmd = f"apptainer exec --cleanenv --bind {hostdir}:{workdir} {sif_path} {cmd}"
+
+    print(f"🧪 Running inside Apptainer: {cmd}")
+    c.run(apptainer_cmd)

airoh/datalad.py

@@ -0,0 +1,82 @@
+# src/airoh/datalad.py
+from invoke import task
+import os
+import shlex
+import shutil
+from pathlib import Path
+
+@task
+def get_data(c, name):
+    """
+    Ensure a Datalad subdataset is installed and retrieved.
+
+    Parameters:
+        name (str): Name of the dataset as defined in invoke.yaml under 'datasets'.
+    """
+    datasets = c.config.get("datasets", {})
+    if name not in datasets:
+        raise ValueError(f"❌ Dataset '{name}' not found in invoke.yaml under 'datasets'.")
+
+    path = datasets[name]
+    print(f"📦 Checking dataset '{name}' at: {path}")
+
+    if not os.path.exists(path):
+        print(f"📥 Installing subdataset '{name}'...")
+        c.run(f"datalad install --recursive {path}")
+
+    print(f"📥 Retrieving data for '{name}'...")
+    c.run(f"datalad get {path}")
+    print("✅ Done.")
+
+@task
+def import_file(c, name):
+    """
+    Download a file using its name from invoke.yaml -> files.<name>.
+
+    Parameters:
+        name (str): Key from the 'files' section in invoke.yaml.
+    """
+    files = c.config.get("files", {})
+    if name not in files:
+        raise ValueError(f"❌ No file config found for '{name}' in invoke.yaml.")
+
+    entry = files[name]
+    url = entry.get("url")
+    output_file = entry.get("output_file")
+
+    if not url or not output_file:
+        raise ValueError(f"❌ Entry for '{name}' must define both 'url' and 'output_file'.")
+
+    c.run(f"datalad download-url -O {shlex.quote(output_file)} {shlex.quote(url)}")
+    print(f"✅ Downloaded {name} to {output_file}")
+
+@task
+def import_archive(c, url, archive_name=None, target_dir=".", drop_archive=False):
+    """
+    Download a remote archive (e.g. from Zenodo or Figshare) and extract its content with Datalad.
+
+    Parameters:
+        url (str): URL to the archive (e.g. .zip, .tar.gz). For Figshare-style links, explicitly provide
+                   `archive_name` if the URL does not end with the actual filename.
+        archive_name (str): Optional filename (default: basename of URL).
+        target_dir (str): Directory to extract into (default: current dir).
+        drop_archive (bool): Whether to drop the original archive from annex after extraction.
+    """
+    archive_name = archive_name or os.path.basename(url)
+    archive_path = os.path.join(target_dir, archive_name)
+
+    import_file(c, url, archive_path)
+
+    archive_exts = ['.zip', '.tar', '.tar.gz', '.tgz', '.tar.bz2', '.7z']
+    if not any(archive_path.endswith(ext) for ext in archive_exts):
+        print("⚠️ Skipping extraction — file does not appear to be a supported archive.")
+        return
+
+    print(f"📦 Extracting archive content into {target_dir}...")
+    c.run(f"datalad add-archive-content --delete --extract {shlex.quote(archive_path)} -d {shlex.quote(target_dir)}")
+
+    if drop_archive:
+        print(f"🧹 Dropping archive from annex: {archive_path}")
+        c.run(f"datalad drop {shlex.quote(archive_path)}")
+
+    print("✅ Archive import complete.")

airoh/utils.py

@@ -0,0 +1,95 @@
+# src/airoh/utils.py
+import os
+import shutil
+from pathlib import Path
+from invoke import task
+
+@task
+def setup_env_python(c, reqs="requirements.txt"):
+    """
+    Set up Python environment by installing from a requirements file.
+    """
+    if not os.path.exists(reqs):
+        raise FileNotFoundError(f"⚠️ Requirements file not found: {reqs}")
+
+    print(f"🐍 Installing Python requirements from {reqs}...")
+    c.run(f"pip install -r {reqs}")
+
+@task
+def ensure_submodule(c, path):
+    """
+    Ensure a git submodule is present and up to date.
+
+        Parameters:
+            path (str): Path to the submodule directory 
+    """
+    if not os.path.exists(path) or not os.path.exists(os.path.join(path, ".git")):
+        print(f"📦 Initializing submodule at {path}...")
+        c.run(f"git submodule update --init --recursive {path}")
+    else:
+        print(f"🔄 Updating submodule at {path}...")
+        c.run(f"git submodule update --remote {path}")
+
+@task
+def install_local(c, path):
+    """
+    Install a local Python package in editable mode using pip.
+
+    Parameters:
+        path (str): Path to the package directory
+    """
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"❌ Package path not found: {path}")
+
+    print(f"🔧 Installing package from {path} in editable mode...")
+    c.run(f"pip install -e {path}")
+    print("✅ Editable install complete.")
+
+@task
+def clean_folder(c, dir_name, label=None):
+    """
+    Remove an entire directory recursively. Use with caution!!!
+    Parameters: 
+        dir_name (str): Path to be removed 
+        label (str, optional): label to use for path in the verbose
+    """
+    label = label or dir_name
+    if os.path.exists(dir_name):
+        shutil.rmtree(dir_name)
+        print(f"💥 Removed {label} at {dir_name}")
+    else:
+        print(f"🫧 Nothing to clean: {label}")
+
+@task
+def run_figures(c, notebooks_path=None, figures_base=None):
+    """
+    Run figure notebooks, skipping any that already have output folders.
+    Notebooks directory and output location pulled from invoke.yaml.
+    """
+    import pathlib as lib
+
+    notebooks_path = Path(notebooks_path or c.config.get("notebooks_dir", "code/figures"))
+    figures_base = Path(figures_base or c.config.get("figures_dir", "output_data/Figures"))
+
+    if not notebooks_path.exists():
+        print(f"⚠️ Notebooks directory not found: {notebooks_path}")
+        return
+
+    notebooks = sorted(notebooks_path.glob("*.ipynb"))
+
+    if not notebooks:
+        print(f"⚠️ No notebooks found in {notebooks_path}/")
+        return
+
+    for nb in notebooks:
+        fig_name = nb.stem
+        fig_output_dir = figures_base / fig_name
+
+        if fig_output_dir.exists():
+            print(f"✅ Skipping {nb.name} (output exists at {fig_output_dir})")
+            continue
+
+        print(f"📈 Running {nb.name}...")
+        c.run(f"jupyter nbconvert --to notebook --execute --inplace {nb}")
+
+    print("🎉 All figure notebooks processed.")

airoh-0.1.2.dist-info/METADATA

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: airoh
+Version: 0.1.2
+Summary: A lightweight task library for reproducible workflows
+Project-URL: Homepage, https://github.com/simexp/airoh
+Project-URL: Issues, https://github.com/SIMEXP/airoh/issues
+Author-email: Lune Bellec <lune.bellec@umontreal.ca>
+License: MIT
+License-File: LICENSE
+Requires-Python: >=3.8
+Requires-Dist: invoke>=2.0
+Description-Content-Type: text/markdown
+
+# airoh
+_Because reproducible science takes clean tasks. And why don't you have a cup of relaxing jasmin tea?_
+
+**airoh** is a lightweight Python task library built with [`invoke`](https://www.pyinvoke.org/), designed for reproducible research workflows. It provides pre-written, modular task definitions that can be easily reused in your own `tasks.py` file — no boilerplate, just useful automation.
+
+## Installation
+Installation through PIP:
+```bash
+pip install airoh
+```
+
+For local deployment:
+
+```bash
+git clone https://github.com/simexp/airoh.git
+cd airoh
+pip install -e .
+```
+
+## Usage
+
+You can use `airoh` in your project simply by importing tasks in your `tasks.py` file.
+
+### Minimal Example
+
+```python
+# tasks.py
+from airoh.utils import run_figures, setup_env_python
+```
+
+Now you can call:
+
+```bash
+invoke run-figures
+invoke setup-env-python
+```
+
+## Available Tasks
+
+### From `containers.py`
+
+* `docker-build` — Build a Docker image from a Dockerfile
+* `docker-archive` — Archive the Docker image to `.tar.gz`
+* `docker-setup` — Download and load a Docker image from URL
+* `docker-run` — Run an `invoke` task inside Docker
+* `apptainer-archive` — Build an Apptainer image from Docker
+* `apptainer-run` — Run an `invoke` task inside Apptainer
+
+### From `utils.py`
+
+* `ensure_submodule` — Ensure a git submodule is present and up to date
+* `install_local` — Install a local Python package in editable mode using pip 
+* `setup-env-python` — Install Python dependencies from a file
+* `run-figures` — Run Jupyter notebooks to generate figures
+* `clean_folder` — Remove an entire directory recursively. Use with caution!!!
+
+### From `datalad.py`
+* `get-data` — Install and retrieve a subdataset listed in `invoke.yaml`
+* `import-archive` — Download a remote archive (e.g. from Zenodo), extract its contents with `datalad`, and optionally drop the original file
+* `import-file` — Download a remote file (e.g. from Zenodo), using `datalad`
+
+## Requirements
+
+* Python ≥ 3.8
+* [`invoke`](https://www.pyinvoke.org/) ≥ 2.0
+* Docker (for container tasks)
+* Apptainer (optional, for `.sif` support)
+* `jupyter` (if using `run-figures`)
+
+## Philosophy
+
+Inspired by Uncle Iroh from *Avatar: The Last Airbender*, `airoh` aims to bring simplicity, reusability, and clarity to research infrastructure — one well-structured task at a time. It is meant to support a concrete implementation of the [YODA principles](https://handbook.datalad.org/en/latest/basics/101-127-yoda.html).
+
+## License
+
+MIT © Lune Bellec

airoh-0.1.2.dist-info/RECORD

@@ -0,0 +1,8 @@
+airoh/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+airoh/containers.py,sha256=8fy7SbvcgvGB7zUz3XujZt0V1Mw3df5bu9tpPSGz5w8,5370
+airoh/datalad.py,sha256=UchPdADLLUIMWVpAaleFqVNxqzCCyMyCEs18NhYG3ms,2968
+airoh/utils.py,sha256=yjcxMiinRD79NTrGxHouQv0MEU6p0_lz7B7-6QUWDUs,2997
+airoh-0.1.2.dist-info/METADATA,sha256=mQsmrjvEyCI2SrXxS6dxD9CKWaYcTofzwdV5wvAqVZc,2944
+airoh-0.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+airoh-0.1.2.dist-info/licenses/LICENSE,sha256=g9RmS4EgjG249_mj6p16XgUnsqp4cpigQnI_pGyEKnk,1063
+airoh-0.1.2.dist-info/RECORD,,

airoh-0.1.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

airoh-0.1.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 SIMEXP
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.