babs 0.3.0__tar.gz → 0.3.2__tar.gz

babs-0.3.2/.circleci/config.yml

@@ -0,0 +1,163 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+
+.dockersetup: &dockersetup
+  docker:
+    - image: pennlinc/slurm-docker-ci:0.11
+
+jobs:
+  download_test_data:
+    <<: *dockersetup
+    resource_class: medium
+    working_directory: /home/circleci/src/babs
+    steps:
+      - checkout:
+          path: /home/circleci/src/babs
+      - restore_cache:
+          keys:
+            - test-data-v2
+      - run:
+          name: Download test datasets
+          command: |
+            mkdir -p /home/circleci/test_data
+            cd /home/circleci/test_data
+            # Install datalad if not already in the image
+            pip install datalad-osf
+            # Download each dataset from origin_input_dataset.yaml
+            python3 /home/circleci/src/babs/tests/download_datasets.py
+            # Debug output
+            echo "Contents of test_data directory:"
+            ls -la /home/circleci/test_data/
+      - save_cache:
+          key: test-data-v2
+          paths:
+            - /home/circleci/test_data
+
+  pytest: # run pytest
+    <<: *dockersetup
+    resource_class: xlarge
+    working_directory: /home/circleci/src/babs
+    steps:
+      - checkout:
+          path: /home/circleci/src/babs
+      - restore_cache:
+          keys:
+            - test-data-v2
+      - run:
+          name: pytest of BABS
+          no_output_timeout: 1h
+          command: |
+            micromamba install -y -c conda-forge shellcheck
+            pip install .[tests]
+            pytest -n 4 -sv --durations=0 --timeout=300
+
+  e2e-slurm:
+    machine:
+      image: ubuntu-2204:current
+    # docker:
+    #   - image: pennlinc/slurm-docker-ci:0.8
+    working_directory: /home/circleci/src/babs
+    steps:
+      - checkout:
+          path: /home/circleci/src/babs
+      - restore_cache:
+          keys:
+            - test-data-v2
+      - run:
+          name: pytest of BABS
+          no_output_timeout: 1h
+          command: |
+            docker build \
+            -t pennlinc/slurm-docker-ci:unstable \
+            -f Dockerfile_testing .
+
+            # Make a directory that will hold the test artifacts
+            mkdir -p ${HOME}/e2e-testing
+            docker run -it \
+                -v ${PWD}:/tests \
+                -v ${HOME}/e2e-testing:/test-temp:rw \
+                -h slurmctl --cap-add sys_admin \
+                --privileged \
+                pennlinc/slurm-docker-ci:unstable \
+                /tests/tests/e2e-slurm/container/walkthrough-tests.sh
+
+      - run:
+          name: clean up test artifacts
+          when: always
+          command: |
+            TEST_ARTIFACTS_DIR=${HOME}/e2e-testing/test_project/analysis
+            sudo chmod -R 777 $TEST_ARTIFACTS_DIR
+            # Find and remove all .git directories recursively
+            sudo find $TEST_ARTIFACTS_DIR -type d -name ".git" -exec rm -rf {} +
+            # Also remove any .git files
+            sudo find $TEST_ARTIFACTS_DIR -type f -name ".git" -exec rm -f {} +
+            # Remove any .gitignore files
+            sudo find $TEST_ARTIFACTS_DIR -type f -name ".gitignore" -exec rm -f {} +
+            # Remove any .gitattributes files
+            sudo find $TEST_ARTIFACTS_DIR -type f -name ".gitattributes" -exec rm -f {} +
+            sudo find $TEST_ARTIFACTS_DIR/inputs/data -type d -name "sub-*" -exec rm -rf {} +
+
+      - store_artifacts:
+          path: ${HOME}/e2e-testing/test_project/analysis
+          destination: test-artifacts
+
+  deployable: # checkpoint of deployable: requires all success
+    <<: *dockersetup
+    steps:
+      - run: echo Deploying!
+
+  deploy_pypi: # deploy to pypi
+    resource_class: small
+    docker:
+      - image: cimg/python:3.11.11
+    working_directory: /tmp/src/babs
+    steps:
+      - checkout
+      - run:
+          name: Update build tools
+          command: pip install --upgrade build twine
+      - run:
+          name: Build BABS
+          command: python -m build
+      - run:
+          name: Upload packages to PyPI
+          command: python -m twine upload -u __token__ -p ${PYPI_PASS} dist/babs*
+
+workflows:
+  # version: 2
+  build_test_deploy:
+    jobs:
+      - download_test_data:
+          filters:
+            tags:
+              only: /.*/
+      - pytest:
+          requires:
+            - download_test_data
+          filters:
+            tags:
+              only: /.*/
+      - e2e-slurm:
+          requires:
+            - download_test_data
+          filters:
+            tags:
+              only: /.*/
+      - deployable:
+          requires:
+            - pytest
+            - e2e-slurm
+          filters:
+            branches:
+              only: main
+            tags:
+              only: /.*/
+      - deploy_pypi:
+          requires:
+            - deployable
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /.*/

babs-0.3.2/.codespellrc

@@ -0,0 +1,2 @@
+[codespell]
+skip = versioneer.py,_version.py,.git,*html

babs-0.3.2/.gitattributes

@@ -0,0 +1 @@
+babs/_version.py export-subst

babs-0.3.2/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,37 @@
+---
+name: Bug report
+about: Something not working as described? Missing/incorrect documentation? This is the place.
+title: ''
+labels: 'bug'
+assignees: ''
+
+---
+## Summary
+<!--What is the nature of the bug?-->
+
+## Additional details
+<!--Please fill in the following details-->
+- BABS version (`pip show babs`):
+- DataLad version (`datalad --version`):
+- Git version (`git --version`):
+- `git-annex` version (`git-annex version`; pasting the first printed line is sufficient):
+- `datalad-container` version (`datalad containers-add --version`):
+- Singularity version (`singularity --version`):
+- Please copy and paste the printed messages from `babs check-setup`:
+
+### What were you trying to do?
+
+### What did you expect to happen?
+
+### What actually happened?
+
+## Reproducing the bug
+<!--Please share any steps you performed that revealed the bug-->
+<!--Please include any code snippets.
+Enclose them in triple back-ticks (```)
+Like this:
+
+```
+<code>
+```
+-->

babs-0.3.2/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,16 @@
+---
+name: Feature request
+about: Got an idea for a new feature, or changing an existing one? This is the place.
+title: ''
+labels: 'enhancement'
+assignees: ''
+
+---
+## Summary
+<!--What would you like changed/added and why?-->
+
+## Additional details
+<!--What would be the benefit?-->
+
+## Next steps
+<!--Do you have any ideas about the implementation?-->

babs-0.3.2/.github/release.yml

@@ -0,0 +1,20 @@
+changelog:
+  exclude:
+    labels:
+      - ignore-for-release
+  categories:
+    - title: 🛠 Breaking Changes
+      labels:
+        - breaking-change
+    - title: 🎉 Exciting New Features
+      labels:
+        - enhancement
+    - title: 👎 Deprecations
+      labels:
+        - deprecation
+    - title: 🐛 Bug Fixes
+      labels:
+        - bug
+    - title: Other Changes
+      labels:
+        - "*"

babs-0.3.2/.github/workflows/codespell.yml

@@ -0,0 +1,14 @@
+---
+name: Codespell
+
+on: [push, pull_request]
+# ref: https://docs.github.com/en/actions/using-workflows/triggering-a-workflow
+
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: codespell-project/actions-codespell@master

babs-0.3.2/.github/workflows/e2e-slurm.yml

@@ -0,0 +1,35 @@
+---
+name: Slurm
+
+# Temporarily disabled
+on:
+  push:
+    branches-ignore:
+      - '**'  # This prevents the workflow from running on any branch
+
+jobs:
+    e2e-slurm:
+      name: Test e2e with SLURM
+      runs-on: ubuntu-latest
+      steps:
+        - name: checkout our repo
+          uses: actions/checkout@v4
+        - name: Install apptainer
+          uses: eWaterCycle/setup-apptainer@v2
+          with:
+            apptainer-version: 1.1.2
+        - name: Install Conda
+          uses: conda-incubator/setup-miniconda@v3
+          with:
+            activate-environment: babs
+            auto-update-conda: true
+            python-version: 3.9
+        - name: Conda info
+          shell: bash -el {0}
+          run: conda info
+        - name: Install Babs
+          shell: bash -el {0}
+          run: ./tests/e2e-slurm/install-babs.sh
+        - name: Execute e2e with SLURM
+          shell: bash -el {0}
+          run: ./tests/e2e-slurm/main.sh

babs-0.3.2/.github/workflows/ruff.yml

@@ -0,0 +1,22 @@
+name: Ruff
+
+on: [push, pull_request]
+
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff
+      - name: Run Ruff
+        run: |
+          ruff check .
+          ruff format --check .

babs-0.3.2/.github/workflows/shellcheck.yml

@@ -0,0 +1,18 @@
+name: Shellcheck scripts
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up system
+        shell: bash
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install shellcheck
+      - uses: actions/checkout@v4
+      - name: Run shellcheck
+        run: |
+          git grep -l -E '^#!/bin/(ba|)sh' | grep -v jinja | xargs shellcheck -x

babs-0.3.2/.readthedocs.yaml

@@ -0,0 +1,17 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true
+
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - doc

babs-0.3.2/CITATION.cff

@@ -0,0 +1,102 @@
+# This CITATION.cff file was generated with cffinit.
+# Visit https://bit.ly/cffinit to generate yours today!
+
+cff-version: 1.2.0
+title: >-
+  BABS : BIDS App Bootstrap
+message: >-
+  If you use this software, please cite it using the
+  metadata from this file, as well as the
+  Imaging Neuroscience (doi:10.1162/imag_a_00074) and
+  Biological Psychiatry (doi:10.1016/j.biopsych.2024.02.056) papers.
+type: software
+authors:
+  - given-names: Chenying
+    family-names: Zhao
+    orcid: "https://orcid.org/0000-0001-7878-6259"
+  - given-names: Dorota
+    family-names: Jarecka
+  - given-names: Sydney
+    family-names: Covitz
+    affiliation: Stanford University
+    orcid: "https://orcid.org/0000-0002-7430-4125"
+  - given-names: Yibei
+    family-names: Chen
+  - given-names: Simon B.
+    family-names: Eickhoff
+  - given-names: Damien
+    family-names: Fair
+    affiliation: University of Minnesota
+    orcid: "https://orcid.org/0000-0001-8602-393X"
+  - given-names: Alexandre R.
+    family-names: Franco
+  - given-names: Yaroslav O.
+    family-names: Halchenko
+  - given-names: Timothy J.
+    family-names: Hendrickson
+  - given-names: Felix
+    family-names: Hoffstaedter
+  - given-names: Audrey
+    family-names: Houghton
+  - given-names: Gregory
+    family-names: Kiar
+  - given-names: Austin
+    family-names: Macdonald
+  - given-names: Kahini
+    family-names: Mehta
+    email: kahini.mehta@pennmedicine.upenn.edu
+    affiliation: Columbia University
+    orcid: "https://orcid.org/0000-0001-9466-100X"
+  - given-names: Michael P.
+    family-names: Milham
+  - given-names: Taylor
+    family-names: Salo
+    email: salot@pennmedicine.upenn.edu
+    affiliation:Penn Lifespan Informatics and Neuroimaging Center, University of Pennsylvania, Philadelphia, PA, USA; Department of Psychiatry, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA, USA
+    orcid: "https://orcid.org/0000-0001-9813-3167"
+  - given-names: Michael
+    family-names: Hanke
+  - given-names: Satrajit S.
+    family-names: Ghosh
+  - given-names: Brooke L.
+    family-names: Sevchik
+    affiliation: Penn Lifespan Informatics and Neuroimaging Center, University of Pennsylvania, Philadelphia, PA, USA; Department of Psychiatry, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA, USA.
+  - given-names: S. Parker
+    family-names: Singleton
+    affiliation: Penn Lifespan Informatics and Neuroimaging Center, University of Pennsylvania, Philadelphia, PA, USA; Department of Psychiatry, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA, USA
+    orcid: "https://orcid.org/0000-0002-7102-7820"
+  - given-names: Tien
+    family-names: Tong
+    affiliation: Penn Lifespan Informatics and Neuroimaging Center, University of Pennsylvania, Philadelphia, PA, USA; Department of Psychiatry, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA, USA
+    orcid: 'https://orcid.org/0000-0002-1940-3198'
+  - given-names: Matthew
+    family-names: Cieslak
+    email: matthew.cieslak@pennmedicine.upenn.edu
+    affiliation: Penn Lifespan Informatics and Neuroimaging Center, University of Pennsylvania, Philadelphia, PA, USA; Department of Psychiatry, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA, USA
+    orcid: "https://orcid.org/0000-0002-1931-4734"
+  - given-names: Steven
+    family-names: Meisler
+    orcid: "https://orcid.org/0000-0002-8888-1572"
+    affiliation: Penn Lifespan Informatics and Neuroimaging Center, University of Pennsylvania, Philadelphia, PA, USA; Department of Psychiatry, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA, USA
+  - given-names: Theodore
+    family-names: Satterthwaite
+    affiliation: Penn Lifespan Informatics and Neuroimaging Center, University of Pennsylvania, Philadelphia, PA, USA; Department of Psychiatry, Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA, USA
+    orcid: "https://orcid.org/0000-0001-7072-9399"
+identifiers:
+  - type: doi
+    value: 10.5281/zenodo.8184642
+    description: The Zenodo DOI
+repository-code: "https://github.com/PennLINC/babs"
+url: "https://pennlinc-babs.readthedocs.io/"
+abstract: >-
+  BIDS App Bootstrap (BABS) is a reproducible, generalizable, and scalable
+  Python package for BIDS App analysis of large datasets.
+  It uses DataLad and adopts the FAIRly big framework.
+  Currently, BABS supports jobs submissions and audits on SLURM
+  high performance computing (HPC) clusters.
+keywords:
+  - BIDS-App
+  - Neuroimaging
+license: MIT
+version: 0.0.10
+date-released: "2025-03-24"

babs-0.3.2/Dockerfile

@@ -0,0 +1,48 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    ca-certificates \
+    bzip2 \
+    curl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+
+# Create a shared $HOME directory
+RUN useradd -m -s /bin/bash -G users babs
+WORKDIR /home/babs
+ENV HOME="/home/babs"
+
+WORKDIR /
+RUN echo "2024.04.11"
+RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
+
+COPY docker/environment.yml /tmp/environment.yml
+ENV MAMBA_ROOT_PREFIX="/opt/conda" \
+    MAMBA_NO_LOW_SPEED_LIMIT=1 \
+    PIP_DEFAULT_TIMEOUT=100
+
+RUN micromamba config set extract_threads 1 \
+    && micromamba create -vv -y -f /tmp/environment.yml \
+    && micromamba clean -y -a
+
+ENV PATH=/opt/conda/envs/babs/bin:$PATH
+
+# Configure the git user name and email
+RUN git config --global user.name "CircleCI" \
+    && git config --global user.email "circleci@example.com"
+
+# Create toy bids app
+RUN mkdir -p /singularity_images \
+    && apptainer build \
+    /singularity_images/toybidsapp_0.0.7.sif \
+    docker://pennlinc/toy_bids_app:0.0.7
+
+# # install BABS
+# COPY . $HOME/babs
+# WORKDIR $HOME/babs
+# RUN pip install .[tests]
+
+# # pytest BABS
+# RUN pytest ~/babs

babs-0.3.2/DockerfileSLURM

@@ -0,0 +1,31 @@
+FROM giovtorres/docker-centos7-slurm:latest
+
+WORKDIR /
+RUN echo "2024.04.11"
+RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
+
+COPY docker/environment.yml /tmp/environment.yml
+ENV MAMBA_ROOT_PREFIX="/opt/conda" \
+    MAMBA_NO_LOW_SPEED_LIMIT=1 \
+    PIP_DEFAULT_TIMEOUT=100
+
+RUN micromamba config set extract_threads 1 \
+    && micromamba create -vv -y -f /tmp/environment.yml \
+    && micromamba clean -y -a
+
+ENV PATH=/opt/conda/envs/babs/bin:$PATH
+
+# Configure the git user name and email
+RUN git config --global user.name "CircleCI" \
+    && git config --global user.email "circleci@example.com"
+
+# Create toy bids app
+RUN mkdir -p /singularity_images \
+    && /opt/conda/envs/babs/bin/apptainer build \
+    /singularity_images/toybidsapp_0.0.7.sif \
+    docker://pennlinc/toy_bids_app:0.0.7
+
+# # install BABS
+# COPY . /babs
+# WORKDIR /babs
+# RUN pip install .[tests]

babs-0.3.2/Dockerfile_testing

@@ -0,0 +1,6 @@
+FROM pennlinc/slurm-docker-ci:0.11
+
+# # install BABS
+COPY . /babs
+WORKDIR /babs
+RUN pip install .[tests]

babs-0.3.2/MANIFEST.in

@@ -0,0 +1,11 @@
+#Misc
+# include CHANGES.rst
+include LICENSE
+
+# necessary files in babs/:
+#   by default it only includes *.py
+recursive-include babs *.yaml
+recursive-include babs/templates *.jinja2
+
+# versioneer
+include babs/_version.py

babs-0.3.2/Makefile

@@ -0,0 +1,15 @@
+install:
+	./tests/e2e-slurm/install-babs.sh
+
+setup-user:
+	./tests/e2e-slurm/setup-user.sh
+
+e2e: clean
+	./tests/e2e-slurm/main.sh
+
+clean:
+	@ podman stop slurm 2>/dev/null || true
+	@ podman rm slurm 2>/dev/null || true
+	@[ -e .testdata/babs_test_project/toybidsapp-container ] && \
+		datalad remove -d .testdata/babs_test_project/toybidsapp-container --reckless kill || :
+	rm -rf .testdata

{babs-0.3.0 → babs-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: babs
-Version: 0.3.0
+Version: 0.3.2
 Summary: BABS: BIDS App BootStrap
 Project-URL: Homepage, https://github.com/pennlinc/babs
 Project-URL: Documentation, https://pennlinc-babs.readthedocs.io/en/stable/
@@ -34,10 +34,10 @@ Requires-Dist: datalad-container>=1.1.6
 Requires-Dist: datalad>=0.17.2
 Requires-Dist: filelock>=3.8.0
 Requires-Dist: nibabel>=2.2.1
+Requires-Dist: niworkflows
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: pyyaml>=6.0
-Requires-Dist: qstat>=0.0.5
 Requires-Dist: regex
 Requires-Dist: tqdm
 Provides-Extra: all

babs-0.3.2/README.rst

@@ -0,0 +1,72 @@
+
+BABS: BIDS App Bootstrap
+========================
+
+.. image:: https://readthedocs.org/projects/pennlinc-babs/badge/?version=latest
+  :target: http://pennlinc-babs.readthedocs.io/en/latest/?badge=latest
+  :alt: Documentation Status
+.. image:: https://circleci.com/gh/PennLINC/babs/tree/main.svg?style=svg
+  :target: https://circleci.com/gh/PennLINC/babs/tree/main
+.. image:: https://zenodo.org/badge/456981533.svg
+   :target: https://zenodo.org/badge/latestdoi/456981533
+   :alt: DOI
+
+Full documentation at https://pennlinc-babs.readthedocs.io
+
+
+About
+-----
+
+BIDS App Bootstrap (BABS) is a reproducible, generalizable, and
+scalable Python package for BIDS App analysis of large datasets.
+It uses `DataLad <https://www.datalad.org/>`_ and adopts
+the `FAIRly big framework <https://doi.org/10.1038/s41597-022-01163-2>`_.
+Currently, BABS supports job submissions and audits on SLURM
+high performance computing (HPC) clusters.
+
+
+Citing BABS
+-----------
+
+If you use BABS, please cite it using the metadata from the `CITATION.cff <CITATION.cff>`_ file,
+as well as the following papers:
+
+  Zhao, C., Jarecka, D., Covitz, S., Chen, Y., Eickhoff, S. B., Fair, D. A., ... & Satterthwaite, T. D. (2024).
+  A reproducible and generalizable software workflow for analysis of large-scale neuroimaging data collections
+  using BIDS Apps. *Imaging Neuroscience*, 2, 1-19. doi:10.1162/imag_a_00074.
+
+  Zhao, C., Chen, Y., Jarecka, D., Ghosh, S., Cieslak, M., & Satterthwaite, T. D. (2024).
+  BABS in Action: Merging Reproducibility and Scalability for Large-Scale Neuroimaging Analysis With BIDS Apps.
+  *Biological Psychiatry*, 95(10), S21-S22. doi:10.1016/j.biopsych.2024.02.056.
+
+
+BABS programs
+-------------
+
+.. image:: https://github.com/PennLINC/babs/raw/main/docs/_static/babs_cli.png
+.. Note: this image is taken from the main branch, so it's normal that docs built from branches is not up-to-date.
+..  If using relative path, e.g., `_static/babs_cli.png`, although readthedocs front page would look good, GitHub front page cannot find that image!!!
+
+
+Schematic of BABS workflow
+--------------------------
+
+.. image:: https://github.com/PennLINC/babs/raw/main/docs/_static/babs_workflow.png
+.. Note: this image is taken from the main branch, so it's normal that docs built from branches is not up-to-date.
+
+
+Background and Significance
+---------------------------
+
+Neuroimaging research faces a crisis of reproducibility.
+With massive sample sizes and greater data complexity, this problem becomes more acute.
+The BIDS Apps - the software operating on BIDS data - have provided a substantial advance.
+However, even using BIDS Apps, a full audit trail of data processing is a necessary prerequisite for fully reproducible research.
+Obtaining a faithful record of the audit trail is challenging - especially for large datasets.
+Recently, the `FAIRly big framework <https://doi.org/10.1038/s41597-022-01163-2>`_
+was introduced as a way to facilitate reproducible processing of large-scale data
+by leveraging `DataLad <https://www.datalad.org/>`_ - a version control system for data management.
+However, the current implementation of this framework remains challenging to general users.
+
+BABS was developed to address these challenges
+and to facilitate the reproducible application of BIDS Apps to large-scale datasets.

babs-0.3.2/babs/__init__.py

@@ -0,0 +1,6 @@
+"""Top-level package for BABS."""
+
+try:
+    from ._version import __version__
+except ImportError:
+    __version__ = '0+unknown'