boldqc-bu 0.0.1__tar.gz

boldqc_bu-0.0.1/LICENSE

@@ -0,0 +1,26 @@
+Copyright 2021 The President and Fellows of Harvard College. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, 
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this 
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, 
+   this list of conditions and the following disclaimer in the documentation 
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors 
+   may be used to endorse or promote products derived from this software without 
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

boldqc_bu-0.0.1/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: boldqc-bu
+Version: 0.0.1
+Summary: BOLD QC
+Home-page: https://github.com/kakurk/boldqc
+Author: Kyle Kurkela
+Author-email: kkurkela@bu.edu
+License-File: LICENSE
+Requires-Dist: selfie
+Requires-Dist: executors
+Requires-Dist: yaxil
+Requires-Dist: matplotlib
+Requires-Dist: nibabel
+Requires-Dist: scipy
+Dynamic: author
+Dynamic: author-email
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary

boldqc_bu-0.0.1/README.md

@@ -0,0 +1,11 @@
+# BOLD Quality Control (beta)
+BOLDQC is an automated quality control pipeline for functional MRI scans. BOLDQC
+is built on top of the excellent
+[`dcm2niix`](https://github.com/rordenlab/dcm2niix), 
+[`FreeSurfer`](https://surfer.nmr.mgh.harvard.edu/),
+and
+[`FSL`](https://fsl.fmrib.ox.ac.uk/fsl)
+software packages.
+
+For the latest documentation please head over to [boldqc.readthedocs.io](https://boldqc.readthedocs.io).
+

boldqc_bu-0.0.1/boldqc/__init__.py

@@ -0,0 +1,35 @@
+import os
+import json
+import tarfile
+import logging
+from . import __version__
+
+logger = logging.getLogger()
+
+def version():
+   return __version__.__version__
+ 
+def archive(indir, output):
+    with tarfile.open(output, 'w:gz') as tar:
+        tar.add(indir, os.path.basename(indir))
+
+def get_mask_threshold(sidecar):
+    with open(sidecar, 'r') as fo:
+        js = json.load(fo)
+    bits_stored = js.get('BitsStored', None)
+    receive_coil = js.get('ReceiveCoilName', None)
+    if bits_stored == 12:
+        logger.info(f'scan has "{bits_stored}" bits and receive coil "{receive_coil}", setting mask threshold to 150.0')
+        return 150.0
+    if bits_stored == 16:
+        if receive_coil in ['Head_32']:
+            logger.info(f'scan has "{bits_stored}" bits and receive coil "{receive_coil}", setting mask threshold to 1500.0')
+            return 1500.0
+        if receive_coil in ['Head_64', 'HeadNeck_64']:
+            logger.info(f'scan has "{bits_stored}" bits and receive coil "{receive_coil}", setting mask threshold to 3000.0')
+            return 3000.0
+    raise MaskThresholdError(f'unexpected bits stored "{bits_stored}" + receive coil "{receive_coil}"')
+
+class MaskThresholdError(Exception):
+    pass
+

boldqc_bu-0.0.1/boldqc/__version__.py

@@ -0,0 +1,6 @@
+__title__ = 'boldqc-bu'
+__description__ = 'BOLD QC'
+__url__ = 'https://github.com/kakurk/boldqc'
+__version__ = '0.0.1'
+__author__ = 'Kyle Kurkela'
+__author_email__ = 'kkurkela@bu.edu'

boldqc_bu-0.0.1/boldqc/bids/__init__.py

@@ -0,0 +1,153 @@
+import os
+import re
+import sys
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+class BIDS(object):
+    def __init__(self, base, sub, ses=None):
+        self._base = base
+        self._sub = sub
+        self._ses = ses
+
+    @staticmethod
+    def basename(sub, mod, ses=None, acq=None, ce=None, rec=None, run=None, defacemask=False):
+        # add sub (required)
+        b = 'sub-' + sub.replace('sub-', '')
+        if ses:
+            b += '_ses-' + ses.replace('ses-', '')
+        if acq:
+            b += '_acq-' + acq.replace('acq-', '') 
+        if ce:
+            b += '_ce-' + ce.replace('ce-', '')
+        if rec:
+            b += '_rec-' + rec.replace('rec-', '')
+        if run:
+            b += '_run-' + str(run).replace('run-', '')
+        if defacemask:
+            b += '_mod-' + mod.replace('mod-', '')
+            b += '_defacemask'
+        else:
+            b += '_' + mod.replace('mod-', '')
+        return b
+    
+    @staticmethod
+    def sidecar_for_image(image):
+        logger.info(image)
+        base = re.sub('\.nii.*$', '', image)
+        logger.info(base)
+        return base + '.json'
+
+    def derivatives_dir(self, name):
+        parts = [
+            self._sub
+        ]
+        if self._ses:
+            parts.append(self._ses)
+        return os.path.join(
+            self._base,
+            'derivatives',
+            name,
+            *parts
+        )
+
+    def raw_anat(self, mod, sourcedata=False, **kwargs):
+        parts = [
+            self._sub
+        ]
+        if self._ses:
+            parts.append(self._ses)
+        if sourcedata:
+            dirname = os.path.join(self._base, 'sourcedata', *parts, 'anat')
+        else:
+            dirname = os.path.join(self._base, *parts, 'anat')
+        acq = kwargs.get('acq', None)
+        ce = kwargs.get('ce', None)
+        rec = kwargs.get('rec', None)
+        run = kwargs.get('run', None)
+        brainmask = kwargs.get('brainmask', None)
+        if acq:
+            parts.append('acq-' + acq)
+        if ce:
+            parts.append('ce-' + ce)
+        if rec:
+            parts.append('rec-' + rec)
+        if run:
+            parts.append('run-' + str(run))
+        if brainmask:
+            parts.append('mod-' + mod + '_brainmask')
+        else:
+            parts.append(mod)
+        basename = '_'.join(parts)
+        return dirname, basename
+
+    def raw_bold(self, mod, sourcedata=False, **kwargs):
+        parts = [
+            self._sub
+        ]
+        if self._ses:
+            parts.append(self._ses)
+        if sourcedata:
+            dirname = os.path.join(self._base, 'sourcedata', *parts, 'func')
+        else:
+            dirname = os.path.join(self._base, *parts, 'func')
+        acq = kwargs.get('acq', None)
+        ce = kwargs.get('ce', None)
+        rec = kwargs.get('rec', None)
+        run = kwargs.get('run', None)
+        brainmask = kwargs.get('brainmask', None)
+        if acq:
+            parts.append('acq-' + acq)
+        if ce:
+            parts.append('ce-' + ce)
+        if rec:
+            parts.append('rec-' + rec)
+        if run:
+            parts.append('run-' + str(run))
+        if brainmask:
+            parts.append('mod-' + mod + '_brainmask')
+        else:
+            parts.append(mod)
+        basename = '_'.join(parts)
+        return dirname, basename
+
+    @staticmethod
+    def parse(filename):
+        result = {
+            'filename': filename
+        }
+        # get extension
+        match = re.match('^.*(.dicom|.json|.nii|.nii.gz)$', filename)
+        result['ext'] = match.group(1) if match else None
+        # get sub component
+        match = re.match('^sub-([^\W_]+).*\.(dicom|json|nii|nii.gz)$', filename)
+        result['sub'] = match.group(1) if match else None
+        # get ses component
+        match = re.match('.*_ses-([^\W_]+).*\.(dicom|json|nii|nii.gz)$', filename)
+        result['ses'] = match.group(1) if match else None
+        # get acq component
+        match = re.match('.*_acq-([^\W_]+).*\.(dicom|json|nii|nii.gz)$', filename)
+        result['acq'] = match.group(1) if match else None
+        # get run component
+        match = re.match('.*_run-(\d+).*\.(dicom|json|nii|nii.gz)$', filename)
+        result['run'] = match.group(1) if match else None    
+        # get rec component
+        match = re.match('.*_rec-(\d+).*\.(dicom|json|nii|nii.gz)$', filename)
+        result['rec'] = match.group(1) if match else None
+        # get ce component
+        match = re.match('.*_ce-(\d+).*\.(dicom|json|nii|nii.gz)$', filename)
+        result['ce'] = match.group(1) if match else None
+        # get defacemask component
+        match = re.match('.*(_defacemask)\.(dicom|json|nii|nii.gz)$', filename)
+        result['defacemask'] = True if match else None
+        # get modality component
+        if result['defacemask']:
+            match = re.match('.*_mod-([^\W_]+)_defacemask\.(dicom|json|nii|nii.gz)$', filename)
+            result['mod'] = match.group(1) if match else None
+        else:
+            match = re.match('.*_([^\W_]+)\.(dicom|json|nii|nii.gz)$', filename)
+            result['mod'] = match.group(1) if match else None
+        return result
+

boldqc_bu-0.0.1/boldqc/cli/__init__.py

@@ -0,0 +1,3 @@
+from . import get
+from . import process
+from . import tandem

boldqc_bu-0.0.1/boldqc/cli/get.py

@@ -0,0 +1,83 @@
+import os
+import re
+import sys
+import json
+import yaml
+import yaxil
+import logging
+import argparse as ap
+import subprocess as sp
+import collections as col
+
+logger = logging.getLogger(__name__)
+
+def do(args):
+    if args.insecure:
+        logger.warning('disabling ssl certificate verification')
+        yaxil.CHECK_CERTIFICATE = False
+
+    # load authentication data and set environment variables for ArcGet.py
+    auth = yaxil.auth2(
+        args.xnat_alias,
+        args.xnat_host,
+        args.xnat_user,
+        args.xnat_pass
+    )
+    os.environ['XNAT_HOST'] = auth.url
+    os.environ['XNAT_USER'] = auth.username
+    os.environ['XNAT_PASS'] = auth.password
+
+    # query BOLD scans
+    with yaxil.session(auth) as ses:
+        scans = col.defaultdict(dict)
+        for scan in ses.scans(label=args.label, project=args.project):
+            note = scan['note']
+            bold_match = re.match('.*(^|\s)#BOLD(?P<run>_\d+)?(\s|$).*', note, flags=re.IGNORECASE)
+            if bold_match:
+                run = bold_match.group('run')
+                run = re.sub('[^0-9]', '', run or '1')
+                run = int(run)
+                scans[run]['bold'] = scan['id']
+    logger.info(json.dumps(scans, indent=2))
+    for run,scansr in scans.items():
+        logger.info('getting bold run=%s, scan=%s', run, scansr['bold'])
+        get_bold(args, auth, run, scansr['bold'], verbose=args.verbose)
+
+def get_bold(args, auth, run, scan, verbose=False):
+    config = {
+        'func': {
+            'bold': [
+                {
+                    'run': int(run),
+                    'scan': scan
+                }
+            ]
+        }
+    }
+    config = yaml.safe_dump(config)
+    cmd = [
+        'ArcGet.py',
+        '--label', args.label,
+        '--output-dir', args.bids_dir,
+        '--output-format', 'bids',
+    ]
+    if args.project:
+        cmd.extend([
+            '--project', args.project
+        ])
+    if args.insecure:
+        cmd.extend([
+            '--insecure'
+        ])
+    if args.in_mem:
+        cmd.extend([
+            '--in-mem'
+        ])
+    cmd.extend([
+        '--config', '-'
+    ])
+    if verbose:
+        cmd.append('--debug')
+    logger.info(sp.list2cmdline(cmd))
+    sp.check_output(cmd, input=config.encode('utf-8'))
+

boldqc_bu-0.0.1/boldqc/cli/process.py

@@ -0,0 +1,110 @@
+import os
+import re
+import sys
+import json
+import yaml
+import yaxil
+import glob
+import math
+import boldqc
+import logging
+import tarfile
+import executors
+import tempfile as tf
+import subprocess as sp
+import boldqc.tasks.stackcheck_ext as stackcheck_ext
+import boldqc.tasks.niftiqa_wrapper as niftiqa_wrapper
+from executors.models import Job, JobArray
+from boldqc.bids import BIDS
+from boldqc.xnat import Report
+from boldqc.state import State
+
+logger = logging.getLogger(__name__)
+LIBEXEC = os.path.realpath(os.path.join(os.path.dirname(boldqc.__file__), 'libexec'))
+os.environ['PATH'] = LIBEXEC + ':' + os.environ['PATH']
+
+def do(args):
+    if args.insecure:
+        logger.warning('disabling ssl certificate verification')
+        yaxil.CHECK_CERTIFICATE = False
+
+    # create job executor and job array
+    if args.scheduler:
+        E = executors.get(args.scheduler, partition=args.partition)
+    else:
+        E = executors.probe(args.partition)
+    jarray = JobArray(E)
+
+    # create BIDS
+    B = BIDS(args.bids_dir, args.sub, ses=args.ses)
+    raw = B.raw_bold('bold', run=args.run)
+    logger.debug('BOLD raw: %s', raw)
+
+    # get repetition time from T1w sidecar for vNav processing
+    sidecar = os.path.join(*raw) + '.json'
+    logger.debug('sidecar: %s', sidecar)
+    with open(sidecar) as fo:
+        js = json.load(fo)
+        tr = js['RepetitionTime']
+    logger.debug('TR: %s', tr)
+
+    boldqc_outdir = None
+    infile = os.path.join(*raw) + '.nii.gz'
+    boldqc_outdir = B.derivatives_dir('boldqc')
+    boldqc_outdir = os.path.join(boldqc_outdir, 'func', raw[1])
+    
+    # niftiqa job
+    task = niftiqa_wrapper.Task(
+        infile,
+        boldqc_outdir
+    )
+    logger.info(json.dumps(task.command, indent=1))
+    jarray.add(task.job)
+
+    # stackcheck_ext job
+    task = stackcheck_ext.Task(
+        infile,
+        boldqc_outdir
+    )
+    logger.info(json.dumps(task.command, indent=1))
+    jarray.add(task.job)
+
+    # submit jobs and wait for them to finish
+    if not args.dry_run:
+        logger.info('submitting jobs')
+        jarray.submit(limit=1)
+        logger.info('waiting for all jobs to finish')
+        jarray.wait()
+        numjobs = len(jarray.array)
+        failed = len(jarray.failed)
+        complete = len(jarray.complete)
+        if failed:
+            logger.info('%s/%s jobs failed', failed, numjobs)
+            for pid,job in iter(jarray.failed.items()):
+                logger.error('%s exited with returncode %s', job.name, job.returncode)
+                with open(job.output, 'r') as fp:
+                    logger.error('standard output\n%s', fp.read())
+                with open(job.error, 'r') as fp:
+                    logger.error('standard error\n%s', fp.read())
+        logger.info('%s/%s jobs completed', complete, numjobs)
+        if failed > 0:
+            sys.exit(1)
+
+    # artifacts directory
+    if not args.artifacts_dir:
+        args.artifacts_dir = os.path.join(
+            boldqc_outdir,
+            'xnat-artifacts'
+        )
+
+    # build data to upload to XNAT
+    R = Report(args.bids_dir, args.sub, args.ses, args.run)
+    logger.info('building xnat artifacts to %s', args.artifacts_dir)
+    R.build_assessment(args.artifacts_dir)
+
+    # upload data to xnat over rest api
+    if args.xnat_upload:
+        logger.info('Uploading artifacts to XNAT')
+        auth = yaxil.auth2(args.xnat_alias)
+        yaxil.storerest(auth, args.artifacts_dir, 'boldqc-resource')
+

boldqc_bu-0.0.1/boldqc/cli/tandem.py

@@ -0,0 +1,66 @@
+import os
+import re
+import json
+import yaml
+import yaxil
+import logging
+import yaxil.bids
+import argparse as ap
+import subprocess as sp
+import boldqc.cli.get
+import boldqc.cli.process
+import collections as col
+import yaxil.bids
+
+logger = logging.getLogger(__name__)
+
+def do(args):
+    if args.insecure:
+        logger.warning('disabling ssl certificate verification')
+        yaxil.CHECK_CERTIFICATE = False
+
+    # load authentication data and set environment variables for ArcGet.py
+    auth = yaxil.auth2(
+        args.xnat_alias,
+        args.xnat_host,
+        args.xnat_user,
+        args.xnat_pass
+    )
+    os.environ['XNAT_HOST'] = auth.url
+    os.environ['XNAT_USER'] = auth.username
+    os.environ['XNAT_PASS'] = auth.password
+
+    # query BOLD scans
+    with yaxil.session(auth) as ses:
+        scans = col.defaultdict(dict)
+        for scan in ses.scans(label=args.label, project=args.project):
+            note = scan['note']
+            bold_match = re.match('.*(^|\s)#BOLD(?P<run>_\d+)?(\s|$).*', note, flags=re.IGNORECASE)
+            if bold_match:
+                run = bold_match.group('run')
+                run = re.sub('[^0-9]', '', run or '1')
+                run = int(run)
+                scans[run]['bold'] = scan['id']
+
+    subject_label = scan['subject_label']
+    logger.info(json.dumps(scans, indent=2))
+
+    for run,scansr in scans.items():
+        if run != args.run:
+            continue
+        logger.info('getting bold run=%s, scan=%s', run, scansr['bold'])
+        boldqc.cli.get.get_bold(
+            args,
+            auth,
+            run,
+            scansr['bold'],
+            verbose=args.verbose
+        )
+        args.run = int(run)
+        args.run = int(run)
+        bids_ses_label = yaxil.bids.legal.sub('', args.label)
+        bids_sub_label = yaxil.bids.legal.sub('', subject_label)
+        args.sub = 'sub-' + bids_sub_label
+        args.ses = 'ses-' + bids_ses_label
+        logger.debug('sub=%s, ses=%s', args.sub, args.ses)
+        boldqc.cli.process.do(args)

boldqc_bu-0.0.1/boldqc/libexec/helpers.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+function isInt {
+	num="$1"
+	if ! [[ "$num" =~ ^-?[0-9]+$ ]]; then
+		return 0
+	fi
+	return 1
+}
+
+function isFloat {
+	num="$1"
+	if ! [[ "$num" =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then
+		return 0
+	fi
+	return 1
+}
+
+function checkMode {
+	file="$1"
+	mode="$2"
+	if [ ! -e "$file" -a "$mode" != "c" ]; then
+		error "file does not exist '$file'"
+	fi
+	case $mode in
+		r)
+			if [ ! -r "$file" ]; then
+				error "file is unreadable '$file'"
+			fi
+			;;
+		w)
+			if [ ! -r "$file" ]; then
+				error "file is not writable '$file'"
+			fi
+			;;
+		x)
+			if [ ! -x "$file" ]; then
+				error "file is not executable '$file'"
+			fi
+			;;
+		c)
+			parent=`dirname "$file"`
+			if [ ! -w "$parent" ]; then
+				error "cannot create file in '$parent'"
+			fi
+			;;
+	esac
+	return 1;
+}
+
+function dirBackup {
+	dir="$1"
+	if [ -e "$dir" ]; then
+        	newdir="$dir-`date +'%Y-%M-%d_%H.%M.%S'`"
+        	echo -n "Moving directory $dir to $newdir... "
+        	if [ ! -w "$dir" ]; then
+                	error "permission denied."
+		else
+                	mv "$dir" "$newdir" || error "failed."
+        	fi
+	else
+		return 0
+	fi
+    echo "Done." && return 1
+}
+
+function error {
+	echo "[ERROR]: $1"
+	exit 1
+}
+
+function warn {
+	caller=`caller 1`
+	if [ $caller ]; then
+		echo -n "$caller ";
+	fi
+	echo "[WARN]: $1"
+}
+
+function check {
+	if [ -z  `which "$1"` ]; then
+		[ -n "$2" ] && error "Could not find command $1 (hint: $2)"
+		error "Could not find command $1"
+	fi
+}
+
+function execute {
+	cmd="$1"
+	verbose="$2"
+	[ -n "$cmd" ] || error "No command supplied to execute function."
+	if [ "$verbose" = "0" ]; then
+		cmd="$cmd &> /dev/null"
+	else
+		echo "$cmd"
+	fi
+	eval "$cmd" && return $?
+}