babs 0.3.2__tar.gz → 0.4.0__tar.gz

{babs-0.3.2 → babs-0.4.0}/.circleci/config.yml

@@ -2,55 +2,47 @@
 # See: https://circleci.com/docs/2.0/configuration-reference
 version: 2.1
 
-.dockersetup: &dockersetup
-  docker:
-    - image: pennlinc/slurm-docker-ci:0.11
+orbs:
+  codecov: codecov/codecov@5.2.1
 
 jobs:
-  download_test_data:
-    <<: *dockersetup
-    resource_class: medium
-    working_directory: /home/circleci/src/babs
-    steps:
-      - checkout:
-          path: /home/circleci/src/babs
-      - restore_cache:
-          keys:
-            - test-data-v2
-      - run:
-          name: Download test datasets
-          command: |
-            mkdir -p /home/circleci/test_data
-            cd /home/circleci/test_data
-            # Install datalad if not already in the image
-            pip install datalad-osf
-            # Download each dataset from origin_input_dataset.yaml
-            python3 /home/circleci/src/babs/tests/download_datasets.py
-            # Debug output
-            echo "Contents of test_data directory:"
-            ls -la /home/circleci/test_data/
-      - save_cache:
-          key: test-data-v2
-          paths:
-            - /home/circleci/test_data
-
-  pytest: # run pytest
-    <<: *dockersetup
-    resource_class: xlarge
+  pytest:
+    machine:
+      image: ubuntu-2204:current
+    resource_class: large
     working_directory: /home/circleci/src/babs
     steps:
       - checkout:
           path: /home/circleci/src/babs
-      - restore_cache:
-          keys:
-            - test-data-v2
       - run:
           name: pytest of BABS
           no_output_timeout: 1h
           command: |
-            micromamba install -y -c conda-forge shellcheck
-            pip install .[tests]
-            pytest -n 4 -sv --durations=0 --timeout=300
+            docker build \
+            -t pennlinc/slurm-docker-ci:unstable \
+            -f Dockerfile_testing .
+
+            # Make a directory that will hold the test artifacts
+            mkdir -p ${HOME}/e2e-testing
+            docker run -it \
+                -v ${PWD}:/tests \
+                -v ${HOME}/e2e-testing:/test-temp:rw \
+                -h slurmctl --cap-add sys_admin \
+                --privileged \
+                pennlinc/slurm-docker-ci:unstable \
+                pytest -n 4 -sv \
+                    --durations=0 \
+                    --timeout=300 \
+                    --junitxml=/test-temp/junit.xml \
+                    --cov-report term-missing \
+                    --cov-report xml:/test-temp/coverage.xml \
+                    --cov=babs \
+                    /babs
+
+      - store_test_results:
+          path: /home/circleci/e2e-testing/junit.xml
+      - codecov/upload:
+          files: /home/circleci/e2e-testing/coverage.xml
 
   e2e-slurm:
     machine:
@@ -61,9 +53,6 @@ jobs:
     steps:
       - checkout:
           path: /home/circleci/src/babs
-      - restore_cache:
-          keys:
-            - test-data-v2
       - run:
           name: pytest of BABS
           no_output_timeout: 1h
@@ -103,7 +92,9 @@ jobs:
           destination: test-artifacts
 
   deployable: # checkpoint of deployable: requires all success
-    <<: *dockersetup
+    resource_class: small
+    docker:
+      - image: busybox:latest
     steps:
       - run: echo Deploying!
 
@@ -128,19 +119,11 @@ workflows:
   # version: 2
   build_test_deploy:
     jobs:
-      - download_test_data:
-          filters:
-            tags:
-              only: /.*/
       - pytest:
-          requires:
-            - download_test_data
           filters:
             tags:
               only: /.*/
       - e2e-slurm:
-          requires:
-            - download_test_data
           filters:
             tags:
               only: /.*/

{babs-0.3.2 → babs-0.4.0}/Dockerfile

@@ -36,8 +36,9 @@ RUN git config --global user.name "CircleCI" \
 # Create toy bids app
 RUN mkdir -p /singularity_images \
     && apptainer build \
-    /singularity_images/toybidsapp_0.0.7.sif \
-    docker://pennlinc/toy_bids_app:0.0.7
+    /singularity_images/simbids_0.0.1.sif \
+    docker://pennlinc/simbids:0.0.1
+
 
 # # install BABS
 # COPY . $HOME/babs

{babs-0.3.2 → babs-0.4.0}/DockerfileSLURM

@@ -19,12 +19,11 @@ ENV PATH=/opt/conda/envs/babs/bin:$PATH
 RUN git config --global user.name "CircleCI" \
     && git config --global user.email "circleci@example.com"
 
-# Create toy bids app
+# Add Simbids
 RUN mkdir -p /singularity_images \
-    && /opt/conda/envs/babs/bin/apptainer build \
-    /singularity_images/toybidsapp_0.0.7.sif \
-    docker://pennlinc/toy_bids_app:0.0.7
-
+    && apptainer build \
+    /singularity_images/simbids_0.0.3.sif \
+    docker://pennlinc/simbids:0.0.3
 # # install BABS
 # COPY . /babs
 # WORKDIR /babs

babs-0.4.0/Dockerfile_testing

@@ -0,0 +1,6 @@
+FROM pennlinc/slurm-docker-ci:0.14
+
+# install BABS in development mode for proper coverage tracking
+COPY . /babs
+WORKDIR /babs
+RUN pip install -e .[tests]

{babs-0.3.2 → babs-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: babs
-Version: 0.3.2
+Version: 0.4.0
 Summary: BABS: BIDS App BootStrap
 Project-URL: Homepage, https://github.com/pennlinc/babs
 Project-URL: Documentation, https://pennlinc-babs.readthedocs.io/en/stable/
@@ -41,6 +41,7 @@ Requires-Dist: pyyaml>=6.0
 Requires-Dist: regex
 Requires-Dist: tqdm
 Provides-Extra: all
+Requires-Dist: coverage; extra == 'all'
 Requires-Dist: datalad-osf; extra == 'all'
 Requires-Dist: docutils<0.17; extra == 'all'
 Requires-Dist: fuzzywuzzy; extra == 'all'
@@ -48,7 +49,7 @@ Requires-Dist: jinja2<3.1.0; extra == 'all'
 Requires-Dist: myst-parser; extra == 'all'
 Requires-Dist: pre-commit; extra == 'all'
 Requires-Dist: pytest; extra == 'all'
-Requires-Dist: pytest-order; extra == 'all'
+Requires-Dist: pytest-cov; extra == 'all'
 Requires-Dist: pytest-xdist; extra == 'all'
 Requires-Dist: python-levenshtein; extra == 'all'
 Requires-Dist: recommonmark==0.7.1; extra == 'all'
@@ -73,7 +74,8 @@ Provides-Extra: maint
 Requires-Dist: fuzzywuzzy; extra == 'maint'
 Requires-Dist: python-levenshtein; extra == 'maint'
 Provides-Extra: tests
+Requires-Dist: coverage; extra == 'tests'
 Requires-Dist: datalad-osf; extra == 'tests'
 Requires-Dist: pytest; extra == 'tests'
-Requires-Dist: pytest-order; extra == 'tests'
+Requires-Dist: pytest-cov; extra == 'tests'
 Requires-Dist: pytest-xdist; extra == 'tests'

{babs-0.3.2 → babs-0.4.0}/README.rst

@@ -7,6 +7,9 @@ BABS: BIDS App Bootstrap
   :alt: Documentation Status
 .. image:: https://circleci.com/gh/PennLINC/babs/tree/main.svg?style=svg
   :target: https://circleci.com/gh/PennLINC/babs/tree/main
+.. image:: https://codecov.io/gh/PennLINC/babs/branch/main/graph/badge.svg
+  :target: https://codecov.io/gh/PennLINC/babs
+  :alt: Code Coverage
 .. image:: https://zenodo.org/badge/456981533.svg
    :target: https://zenodo.org/badge/latestdoi/456981533
    :alt: DOI

babs-0.4.0/babs/__init__.py

@@ -0,0 +1,13 @@
+"""Top-level package for BABS."""
+
+try:
+    from ._version import __version__
+except ImportError:
+    __version__ = '0+unknown'
+
+from .bootstrap import BABSBootstrap
+from .check_setup import BABSCheckSetup
+from .interaction import BABSInteraction
+from .merge import BABSMerge
+
+__all__ = ['BABSBootstrap', 'BABSCheckSetup', 'BABSInteraction', 'BABSMerge']

{babs-0.3.2 → babs-0.4.0}/babs/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.2'
-__version_tuple__ = version_tuple = (0, 3, 2)
+__version__ = version = '0.4.0'
+__version_tuple__ = version_tuple = (0, 4, 0)

babs-0.4.0/babs/base.py

@@ -0,0 +1,207 @@
+"""This is the main module."""
+
+import os.path as op
+import subprocess
+from pathlib import Path
+from urllib.parse import urlparse
+
+from babs.input_datasets import InputDatasets
+from babs.system import validate_queue
+from babs.utils import (
+    read_yaml,
+    validate_processing_level,
+)
+
+CONFIG_SECTIONS = ['processing_level', 'queue', 'input_datasets', 'container']
+
+
+class BABS:
+    """The BABS base class holds common attributes and methods for all BABS classes."""
+
+    def __init__(self, project_root):
+        """The BABS class is for babs projects of BIDS Apps.
+
+        The constructor only initializes the attributes.
+
+        Parameters
+        ----------
+        project_root: Path
+            absolute path to the root of this babs project
+
+        Attributes
+        ----------
+        project_root: str
+            absolute path to the root of this babs project
+        processing_level : {'subject', 'session'}
+            whether processing is done on a subject-wise or session-wise basis
+        queue: str
+            the type of job scheduling system, "sge" or "slurm"
+        config_path: str
+            path to the config yaml file
+        analysis_path: str
+            path to the `analysis` folder.
+        analysis_datalad_handle: datalad dataset
+            the `analysis` datalad dataset
+        input_ria_path: str
+            Path to the input RIA store, the sibling of `analysis`.
+            The computation of each job will start with a clone from this input RIA store.
+        output_ria_path: str
+            Path to the output RIA store, the sibling of `analysis`.
+            The results of jobs will be pushed to this output RIA store.
+        input_ria_url: str
+            URL of input RIA store, starting with "ria+file://".
+        output_ria_url: str
+            URL of output RIA store, starting with "ria+file://".
+        output_ria_data_dir: str
+            Path to the output RIA's data directory.
+            Example: /full/path/to/project_root/output_ria/238/da2f2-2fc4-4b88-a2c5-aa6e754b5d0b
+        analysis_dataset_id: str
+            The ID of DataLad dataset `analysis`.
+            This will be used to get the full path to the dataset in input RIA.
+            Example: '238da2f2-2fc4-4b88-a2c5-aa6e754b5d0b'
+        list_sub_path_rel: str
+            Path to the list of final included subjects (and sessions) CSV file.
+            This is relative to `analysis` folder.
+        list_sub_path_abs: str
+            Absolute path of `list_sub_path_rel`.
+            Example: '/path/to/analysis/code/processing_inclusion.csv'
+        job_status_path_rel: str
+            Path to the `job_status.csv` file.
+            This is relative to `analysis` folder.
+        job_status_path_abs: str
+            Absolute path of `job_status_path_abs`.
+            Example: '/path/to/analysis/code/job_status.csv'
+        job_submit_path_abs: str
+            Absolute path of `job_submit_path_abs`.
+            Example: '/path/to/analysis/code/job_submit.csv'
+        """
+
+        # validation:
+
+        # attributes:
+        self.project_root = str(project_root)
+
+        self.analysis_path = op.join(self.project_root, 'analysis')
+        self.analysis_datalad_handle = None
+
+        self.config_path = op.join(self.analysis_path, 'code/babs_proj_config.yaml')
+
+        self.input_ria_path = op.join(self.project_root, 'input_ria')
+        self.output_ria_path = op.join(self.project_root, 'output_ria')
+
+        self.input_ria_url = 'ria+file://' + self.input_ria_path
+        self.output_ria_url = 'ria+file://' + self.output_ria_path
+
+        self.output_ria_data_dir = None  # not known yet before output_ria is created
+        self.analysis_dataset_id = None  # to update later
+
+        self.list_sub_path_rel = 'code/processing_inclusion.csv'
+        self.list_sub_path_abs = op.join(self.analysis_path, self.list_sub_path_rel)
+
+        self.job_status_path_rel = 'code/job_status.csv'
+        self.job_status_path_abs = op.join(self.analysis_path, self.job_status_path_rel)
+        self.job_submit_path_abs = op.join(self.analysis_path, 'code/job_submit.csv')
+        self._apply_config()
+
+    def _apply_config(self):
+        """Apply the configuration to the BABS project.
+
+        The following attributes are set in this function:
+
+          - processing_level
+          - queue
+          - container
+          - input_datasets
+
+        """
+        # Sanity check: the path `project_root` exists:
+        if not op.exists(self.project_root):
+            raise Exception(
+                f'`project_root` does not exist! Requested `project_root` was: {self.project_root}'
+            )
+        if not op.exists(self.analysis_path):
+            raise FileNotFoundError(
+                f'Missing: {self.analysis_path}\n'
+                f'{self.project_root} is not a valid BABS project.\n\n'
+                'Please run `babs init` first.'
+            )
+        if not op.exists(self.config_path):
+            raise FileNotFoundError(
+                f'Missing: {self.config_path}\n'
+                f'{self.project_root} is not a valid BABS project.\n\n'
+                'Please run `babs init` first.'
+            )
+
+        config_yaml = read_yaml(self.config_path)
+        for section in CONFIG_SECTIONS:
+            if section not in config_yaml:
+                raise ValueError(f'Section {section} not found in {self.config_path}')
+
+        self.processing_level = validate_processing_level(config_yaml['processing_level'])
+        self.queue = validate_queue(config_yaml['queue'])
+        self.container = config_yaml['container']
+
+        # Check the output RIA:
+        self.wtf_key_info(flag_output_ria_only=True)
+
+        self.input_datasets = InputDatasets(self.processing_level, config_yaml['input_datasets'])
+        self.input_datasets.update_abs_paths(Path(self.project_root) / 'analysis')
+
+    def wtf_key_info(self, flag_output_ria_only=False):
+        """
+        This is to get some key information on DataLad dataset `analysis`,
+        and assign to `output_ria_data_dir` and `analysis_dataset_id`.
+        This function relies on `git` and `datalad wtf`
+        This needs to be done after the output RIA is created.
+
+        Parameters
+        ----------
+        flag_output_ria_only: bool
+            if only to get information on output RIA.
+            This may expedite the process as other information requires
+            calling `datalad` in terminal, which would takes several seconds.
+        """
+
+        # Get the `self.output_ria_data_dir`:
+        # e.g., /full/path/output_ria/238/da2f2-2fc4-4b88-a2c5-aa6e754b5d0b
+        analysis_git_path = op.join(self.analysis_path, '.git')
+        proc_output_ria_data_dir = subprocess.run(
+            [
+                'git',
+                '--git-dir',
+                analysis_git_path,
+                'remote',
+                'get-url',
+                '--push',
+                'output',
+            ],
+            stdout=subprocess.PIPE,
+        )
+        # ^^: for `analysis`: git remote get-url --push output
+        # ^^ another way to change the wd temporarily: add `cwd=self.xxx` in `subprocess.run()`
+        # if success: no output; if failed: will raise CalledProcessError
+        proc_output_ria_data_dir.check_returncode()
+        self.output_ria_data_dir = urlparse(proc_output_ria_data_dir.stdout.decode('utf-8')).path
+        if self.output_ria_data_dir[-1:] == '\n':
+            # remove the last 2 characters
+            self.output_ria_data_dir = self.output_ria_data_dir[:-1]
+
+        if not flag_output_ria_only:  # also want other information:
+            # Get the dataset ID of `analysis`, i.e., `analysis_dataset_id`:
+            # way #2: command line of datalad:
+            proc_analysis_dataset_id = subprocess.run(
+                ['datalad', '-f', "'{infos[dataset][id]}'", 'wtf', '-S', 'dataset'],
+                cwd=self.analysis_path,
+                stdout=subprocess.PIPE,
+            )
+            # datalad -f '{infos[dataset][id]}' wtf -S dataset
+            proc_analysis_dataset_id.check_returncode()
+            self.analysis_dataset_id = proc_analysis_dataset_id.stdout.decode('utf-8')
+            # remove the `\n`:
+            if self.analysis_dataset_id[-1:] == '\n':
+                # remove the last 2 characters
+                self.analysis_dataset_id = self.analysis_dataset_id[:-1]
+            # remove the double quotes:
+            if (self.analysis_dataset_id[0] == "'") & (self.analysis_dataset_id[-1] == "'"):
+                # if first and the last characters are quotes: remove them
+                self.analysis_dataset_id = self.analysis_dataset_id[1:-1]