junifer 0.0.3.dev188__py3-none-any.whl → 0.0.4__py3-none-any.whl

junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py

@@ -0,0 +1,192 @@
+"""Provide tests for GnuParallelLocalAdapter."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import pytest
+
+from junifer.api.queue_context import GnuParallelLocalAdapter
+
+
+def test_GnuParallelLocalAdapter_env_error() -> None:
+    """Test error for invalid env kind."""
+    with pytest.raises(ValueError, match="Invalid value for `env.kind`"):
+        GnuParallelLocalAdapter(
+            job_name="check_env",
+            job_dir=Path("."),
+            yaml_config_path=Path("."),
+            elements=["sub01"],
+            env={"kind": "jambalaya"},
+        )
+
+
+@pytest.mark.parametrize(
+    "elements, expected_text",
+    [
+        (["sub01", "sub02"], "sub01\nsub02"),
+        ([("sub01", "ses01"), ("sub02", "ses01")], "sub01,ses01\nsub02,ses01"),
+    ],
+)
+def test_GnuParallelLocalAdapter_elements(
+    elements: List[Union[str, Tuple]],
+    expected_text: str,
+) -> None:
+    """Test GnuParallelLocalAdapter elements().
+
+    Parameters
+    ----------
+    elements : list of str or tuple
+        The parametrized elements.
+    expected_text : str
+        The parametrized expected text.
+
+    """
+    adapter = GnuParallelLocalAdapter(
+        job_name="test_elements",
+        job_dir=Path("."),
+        yaml_config_path=Path("."),
+        elements=elements,
+    )
+    assert expected_text in adapter.elements()
+
+
+@pytest.mark.parametrize(
+    "pre_run, expected_text",
+    [
+        (None, "# Force datalad"),
+        ("# Check this out\n", "# Check this out"),
+    ],
+)
+def test_GnuParallelLocalAdapter_pre_run(
+    pre_run: Optional[str], expected_text: str
+) -> None:
+    """Test GnuParallelLocalAdapter pre_run().
+
+    Parameters
+    ----------
+    pre_run : str or None
+        The parametrized pre run text.
+    expected_text : str
+        The parametrized expected text.
+
+    """
+    adapter = GnuParallelLocalAdapter(
+        job_name="test_pre_run",
+        job_dir=Path("."),
+        yaml_config_path=Path("."),
+        elements=["sub01"],
+        pre_run=pre_run,
+    )
+    assert expected_text in adapter.pre_run()
+
+
+@pytest.mark.parametrize(
+    "pre_collect, expected_text",
+    [
+        (None, "# This script"),
+        ("# Check this out\n", "# Check this out"),
+    ],
+)
+def test_GnuParallelLocalAdapter_pre_collect(
+    pre_collect: Optional[str],
+    expected_text: str,
+) -> None:
+    """Test GnuParallelLocalAdapter pre_collect().
+
+    Parameters
+    ----------
+    pre_collect : str or None
+        The parametrized pre collect text.
+    expected_text : str
+        The parametrized expected text.
+
+    """
+    adapter = GnuParallelLocalAdapter(
+        job_name="test_pre_collect",
+        job_dir=Path("."),
+        yaml_config_path=Path("."),
+        elements=["sub01"],
+        pre_collect=pre_collect,
+    )
+    assert expected_text in adapter.pre_collect()
+
+
+def test_GnuParallelLocalAdapter_run() -> None:
+    """Test HTCondorAdapter run()."""
+    adapter = GnuParallelLocalAdapter(
+        job_name="test_run",
+        job_dir=Path("."),
+        yaml_config_path=Path("."),
+        elements=["sub01"],
+    )
+    assert "run" in adapter.run()
+
+
+def test_GnuParallelLocalAdapter_collect() -> None:
+    """Test HTCondorAdapter collect()."""
+    adapter = GnuParallelLocalAdapter(
+        job_name="test_run_collect",
+        job_dir=Path("."),
+        yaml_config_path=Path("."),
+        elements=["sub01"],
+    )
+    assert "collect" in adapter.collect()
+
+
+@pytest.mark.parametrize(
+    "env",
+    [
+        {"kind": "conda", "name": "junifer"},
+        {"kind": "venv", "name": "./junifer"},
+    ],
+)
+def test_GnuParallelLocalAdapter_prepare(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    env: Dict[str, str],
+) -> None:
+    """Test GnuParallelLocalAdapter prepare().
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    caplog : pytest.LogCaptureFixture
+        The pytest.LogCaptureFixture object.
+    env : dict
+        The parametrized Python environment config.
+
+    """
+    with monkeypatch.context() as m:
+        m.chdir(tmp_path)
+        with caplog.at_level(logging.DEBUG):
+            adapter = GnuParallelLocalAdapter(
+                job_name="test_prepare",
+                job_dir=tmp_path,
+                yaml_config_path=tmp_path / "config.yaml",
+                elements=["sub01"],
+                env=env,
+            )
+            adapter.prepare()
+
+            assert "GNU parallel" in caplog.text
+            assert f"Copying run_{env['kind']}" in caplog.text
+            assert "Writing pre_run.sh" in caplog.text
+            assert "Writing run_test_prepare.sh" in caplog.text
+            assert "Writing pre_collect.sh" in caplog.text
+            assert "Writing collect_test_prepare.sh" in caplog.text
+            assert "Shell scripts created" in caplog.text
+
+            assert adapter._exec_path.stat().st_size != 0
+            assert adapter._elements_file_path.stat().st_size != 0
+            assert adapter._pre_run_path.stat().st_size != 0
+            assert adapter._run_path.stat().st_size != 0
+            assert adapter._pre_collect_path.stat().st_size != 0
+            assert adapter._collect_path.stat().st_size != 0

junifer/api/queue_context/tests/test_htcondor_adapter.py

@@ -0,0 +1,257 @@
+"""Provide tests for HTCondorAdapter."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import pytest
+
+from junifer.api.queue_context import HTCondorAdapter
+
+
+def test_HTCondorAdapter_env_error() -> None:
+    """Test error for invalid env kind."""
+    with pytest.raises(ValueError, match="Invalid value for `env.kind`"):
+        HTCondorAdapter(
+            job_name="check_env",
+            job_dir=Path("."),
+            yaml_config_path=Path("."),
+            elements=["sub01"],
+            env={"kind": "jambalaya"},
+        )
+
+
+def test_HTCondorAdapter_collect_error() -> None:
+    """Test error for invalid collect option."""
+    with pytest.raises(ValueError, match="Invalid value for `collect`"):
+        HTCondorAdapter(
+            job_name="check_collect",
+            job_dir=Path("."),
+            yaml_config_path=Path("."),
+            elements=["sub01"],
+            collect="off",
+        )
+
+
+@pytest.mark.parametrize(
+    "pre_run, expected_text",
+    [
+        (None, "# Force datalad"),
+        ("# Check this out\n", "# Check this out"),
+    ],
+)
+def test_HTCondorAdapter_pre_run(
+    pre_run: Optional[str], expected_text: str
+) -> None:
+    """Test HTCondorAdapter pre_run().
+
+    Parameters
+    ----------
+    pre_run : str or None
+        The parametrized pre run text.
+    expected_text : str
+        The parametrized expected text.
+
+    """
+    adapter = HTCondorAdapter(
+        job_name="test_pre_run",
+        job_dir=Path("."),
+        yaml_config_path=Path("."),
+        elements=["sub01"],
+        pre_run=pre_run,
+    )
+    assert expected_text in adapter.pre_run()
+
+
+@pytest.mark.parametrize(
+    "pre_collect, expected_text, collect",
+    [
+        (None, "exit 1", "yes"),
+        (None, "# This script", "on_success_only"),
+        ("# Check this out\n", "# Check this out", "yes"),
+        ("# Check this out\n", "# Check this out", "on_success_only"),
+    ],
+)
+def test_HTCondorAdapter_pre_collect(
+    pre_collect: Optional[str], expected_text: str, collect: str
+) -> None:
+    """Test HTCondorAdapter pre_collect().
+
+    Parameters
+    ----------
+    pre_collect : str or None
+        The parametrized pre collect text.
+    expected_text : str
+        The parametrized expected text.
+    collect : str
+        The parametrized collect parameter.
+
+    """
+    adapter = HTCondorAdapter(
+        job_name="test_pre_collect",
+        job_dir=Path("."),
+        yaml_config_path=Path("."),
+        elements=["sub01"],
+        pre_collect=pre_collect,
+        collect=collect,
+    )
+    assert expected_text in adapter.pre_collect()
+
+
+@pytest.mark.parametrize(
+    "extra_preamble, expected_text",
+    [
+        (None, "universe = vanilla"),
+        ("# Check this out\n", "# Check this out"),
+    ],
+)
+def test_HTCondorAdapter_run_collect(
+    extra_preamble: Optional[str], expected_text: str
+) -> None:
+    """Test HTCondorAdapter run() and collect().
+
+    Parameters
+    ----------
+    extra_preamble : str or None
+        The parametrized extra preamble text.
+    expected_text : str
+        The parametrized expected text.
+
+    """
+    adapter = HTCondorAdapter(
+        job_name="test_run_collect",
+        job_dir=Path("."),
+        yaml_config_path=Path("."),
+        elements=["sub01"],
+        extra_preamble=extra_preamble,
+    )
+    assert expected_text in adapter.run()
+    assert expected_text in adapter.collect()
+
+
+@pytest.mark.parametrize(
+    "elements, collect, expected_text",
+    [
+        (["sub01"], "yes", "FINAL collect"),
+        (
+            [("sub01", "ses01")],
+            "yes",
+            'element="sub01,ses01" log_element="sub01-ses01"',
+        ),
+        (["sub01"], "on_success_only", "JOB collect"),
+        (
+            [("sub01", "ses01")],
+            "on_success_only",
+            'element="sub01,ses01" log_element="sub01-ses01"',
+        ),
+    ],
+)
+def test_HTCondor_dag(
+    elements: List[Union[str, Tuple]], collect: str, expected_text: str
+) -> None:
+    """Test HTCondorAdapter dag().
+
+    Parameters
+    ----------
+    elements : list of str or tuple
+        The parametrized elements.
+    collect : str
+       The parametrized collect parameter.
+    expected_text : str
+        The parametrized expected text.
+
+    """
+    adapter = HTCondorAdapter(
+        job_name="test_dag",
+        job_dir=Path("."),
+        yaml_config_path=Path("."),
+        elements=elements,
+        collect=collect,
+    )
+    assert expected_text in adapter.dag()
+
+
+@pytest.mark.parametrize(
+    "env",
+    [
+        {"kind": "conda", "name": "junifer"},
+        {"kind": "venv", "name": "./junifer"},
+    ],
+)
+def test_HTCondorAdapter_prepare(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+    env: Dict[str, str],
+) -> None:
+    """Test HTCondorAdapter prepare().
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    caplog : pytest.LogCaptureFixture
+        The pytest.LogCaptureFixture object.
+    env : dict
+        The parametrized Python environment config.
+
+    """
+    with monkeypatch.context() as m:
+        m.chdir(tmp_path)
+        with caplog.at_level(logging.DEBUG):
+            adapter = HTCondorAdapter(
+                job_name="test_prepare",
+                job_dir=tmp_path,
+                yaml_config_path=tmp_path / "config.yaml",
+                elements=["sub01"],
+                env=env,
+            )
+            adapter.prepare()
+
+            assert "Creating HTCondor job" in caplog.text
+            assert "Creating logs directory" in caplog.text
+            assert f"Copying run_{env['kind']}" in caplog.text
+            assert "Writing pre_run.sh" in caplog.text
+            assert "Writing run_test_prepare.submit" in caplog.text
+            assert "Writing pre_collect.sh" in caplog.text
+            assert "Writing collect_test_prepare.submit" in caplog.text
+            assert "HTCondor job files created" in caplog.text
+
+            assert adapter._exec_path.stat().st_size != 0
+            assert adapter._pre_run_path.stat().st_size != 0
+            assert adapter._submit_run_path.stat().st_size != 0
+            assert adapter._pre_collect_path.stat().st_size != 0
+            assert adapter._submit_collect_path.stat().st_size != 0
+            assert adapter._dag_path.stat().st_size != 0
+
+
+def test_HTCondorAdapter_prepare_submit_fail(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Test HTCondorAdapter prepare() with failed submit.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+
+    """
+    with monkeypatch.context() as m:
+        m.chdir(tmp_path)
+        adapter = HTCondorAdapter(
+            job_name="test_prepare_submit_fail",
+            job_dir=tmp_path,
+            yaml_config_path=tmp_path / "config.yaml",
+            elements=["sub01"],
+            submit=True,
+        )
+        with pytest.raises(RuntimeError, match="condor_submit_dag"):
+            adapter.prepare()

junifer/api/res/afni/run_afni_docker.sh

@@ -6,7 +6,7 @@ mounts=0
 for var in "$@"
 do
 if [ -d "${var}" ]; then
-        echo "$var is a directory"
+        echo "$var is a directory" >&2
         var=$(realpath "${var}")
         host_dir=$(dirname "${var}")
         ((mounts+=1))
@@ -15,10 +15,10 @@ if [ -d "${var}" ]; then
         var=${container_dir}
     elif [ -f "${var}" ] || [[ "${var}" == /* ]]; then
         if [ -f "${var}" ]; then
-            echo "$var is a file"
+            echo "$var is a file" >&2
             var=$(realpath "${var}")
         else
-            echo "$var is a prefix"
+            echo "$var is a prefix" >&2
         fi
         fname=$(basename "${var}")
         host_dir=$(dirname "${var}")
@@ -30,10 +30,10 @@ if [ -d "${var}" ]; then
     corrected_args+=("${var}")
 done
 
-echo "Docker args: ${docker_args[*]}"
-echo "Corrected args for afni: ${corrected_args[*]}"
+echo "Docker args: ${docker_args[*]}" >&2
+echo "Corrected args for afni: ${corrected_args[*]}" >&2
 
 cwd=$(pwd)
 cmd="docker run --rm ${docker_args[*]} -v ${cwd}:${cwd} -w ${cwd} afni/afni_make_build ${corrected_args[*]}"
-echo "Running command: ${cmd}"
+echo "Running command: ${cmd}" >&2
 ${cmd}

junifer/api/res/ants/ResampleImage

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+run_ants_docker.sh ResampleImage "$@"

junifer/api/res/ants/antsApplyTransforms

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+run_ants_docker.sh antsApplyTransforms "$@"

junifer/api/res/ants/antsApplyTransformsToPoints

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+run_ants_docker.sh antsApplyTransformsToPoints "$@"

junifer/api/res/ants/run_ants_docker.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+corrected_args=()
+docker_args=()
+mounts=0
+for var in "$@"
+do
+if [ -d "${var}" ]; then
+        echo "$var is a directory" >&2
+        var=$(realpath "${var}")
+        host_dir=$(dirname "${var}")
+        ((mounts+=1))
+        container_dir="/data/mount_${mounts}"
+        docker_args+=("-v ${host_dir}:${container_dir}")
+        var=${container_dir}
+    elif [ -f "${var}" ] || [[ "${var}" == /* ]]; then
+        if [ -f "${var}" ]; then
+            echo "$var is a file" >&2
+            var=$(realpath "${var}")
+        else
+            echo "$var is a prefix" >&2
+        fi
+        fname=$(basename "${var}")
+        host_dir=$(dirname "${var}")
+        ((mounts+=1))
+        container_dir="/data/mount_${mounts}"
+        docker_args+=("-v ${host_dir}:${container_dir}")
+        var=${container_dir}/${fname}
+    fi
+    corrected_args+=("${var}")
+done
+
+echo "Docker args: ${docker_args[*]}" >&2
+echo "Corrected args for fsl: ${corrected_args[*]}" >&2
+
+cwd=$(pwd)
+cmd="docker run --rm ${docker_args[*]} -v ${cwd}:${cwd} -w ${cwd} antsx/ants ${corrected_args[*]}"
+echo "Running command: ${cmd}" >&2
+${cmd}

junifer/api/res/fsl/applywarp

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+run_fsl_docker.sh applywarp "$@"

junifer/api/res/fsl/flirt

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+run_fsl_docker.sh flirt "$@"

junifer/api/res/fsl/img2imgcoord

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+run_fsl_docker.sh img2imgcoord "$@"

junifer/api/res/fsl/run_fsl_docker.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+corrected_args=()
+docker_args=()
+mounts=0
+for var in "$@"
+do
+if [ -d "${var}" ]; then
+        echo "$var is a directory" >&2
+        var=$(realpath "${var}")
+        host_dir=$(dirname "${var}")
+        ((mounts+=1))
+        container_dir="/data/mount_${mounts}"
+        docker_args+=("-v ${host_dir}:${container_dir}")
+        var=${container_dir}
+    elif [ -f "${var}" ] || [[ "${var}" == /* ]]; then
+        if [ -f "${var}" ]; then
+            echo "$var is a file" >&2
+            var=$(realpath "${var}")
+        else
+            echo "$var is a prefix" >&2
+        fi
+        fname=$(basename "${var}")
+        host_dir=$(dirname "${var}")
+        ((mounts+=1))
+        container_dir="/data/mount_${mounts}"
+        docker_args+=("-v ${host_dir}:${container_dir}")
+        var=${container_dir}/${fname}
+    fi
+    corrected_args+=("${var}")
+done
+
+echo "Docker args: ${docker_args[*]}" >&2
+echo "Corrected args for fsl: ${corrected_args[*]}" >&2
+
+cwd=$(pwd)
+cmd="docker run --rm ${docker_args[*]} -v ${cwd}:${cwd} -w ${cwd} brainlife/fsl ${corrected_args[*]}"
+echo "Running command: ${cmd}" >&2
+${cmd}

junifer/api/res/fsl/std2imgcoord

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+run_fsl_docker.sh std2imgcoord "$@"

junifer/api/res/run_conda.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
 if [ $# -lt 2 ]; then
-    echo "This script is meant to run a command within a python environment"
+    echo "This script is meant to run a command within a conda environment."
     echo "It needs at least 2 parameters."
     echo "The first one must be the environment name."
-    echo "The rest will be the command"
+    echo "The rest will be the command."
     exit 255
 fi
 
 eval "$(conda shell.bash hook)"
 env_name=$1
 echo "Activating ${env_name}"
-conda activate "$1"
+conda activate "${env_name}"
 shift 1
 
 if [ -f "pre_run.sh" ]; then
@@ -19,5 +19,5 @@ if [ -f "pre_run.sh" ]; then
     . ./pre_run.sh
 fi
 
-echo "Running ${*} in virtual environment"
+echo "Running ${*} in conda environment"
 "$@"

junifer/api/res/run_venv.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ $# -lt 2 ]; then
+    echo "This script is meant to run a command within a Python virtual environment."
+    echo "It needs at least 2 parameters."
+    echo "The first one must be the virtualenv path."
+    echo "The rest will be the command."
+    exit 255
+fi
+
+env_path=$1
+echo "Activating ${env_path}"
+source "${env_path}"/bin/activate
+shift 1
+
+if [ -f "pre_run.sh" ]; then
+    echo "Sourcing pre_run.sh"
+    . ./pre_run.sh
+fi
+
+echo "Running ${*} in Python virtual environment"
+"$@"

junifer/api/tests/data/partly_cloudy_agg_mean_tian.yml

@@ -0,0 +1,16 @@
+workdir: /tmp
+
+with: junifer.testing.registry
+
+datagrabber:
+  kind: PartlyCloudyTestingDataGrabber
+
+markers:
+  - kind: ParcelAggregation
+    parcellation: TianxS1x3TxMNInonlinear2009cAsym
+    method: mean
+    name: tian-s1-3T_mean
+
+storage:
+  kind: HDF5FeatureStorage
+  uri: /tmp/partly_cloudy_agg_mean_tian.hdf5