cnda_dl 1.0.0__tar.gz → 1.1.0__tar.gz

{cnda_dl-1.0.0 → cnda_dl-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cnda_dl
-Version: 1.0.0
+Version: 1.1.0
 Summary: A command line utility for downloading fMRI data from CNDA
 Author-Email: Ramone Agard <agardr@wustl.edu>, Joey Scanga <joeyscanga92@gmail.com>
 Requires-Python: <4.0,>=3.9
@@ -10,20 +10,12 @@ Requires-Dist: progressbar2>=4.4.2
 Requires-Dist: pyxnat>=1.6.2
 Description-Content-Type: text/markdown
 
-# Dev installation
+# Installation
 
-Ensure `uv` is installed:
+We recommend installing cnda-dl with [pipx](https://github.com/pypa/pipx), which ensures that the cnda-dl binary is installed in an isolated environment. Installation instructions can be found [here](https://github.com/pypa/pipx?tab=readme-ov-file#install-pipx).
 
-```
-which uv
-```
-
-If not, this can be installed following instructions [here](https://docs.astral.sh/uv/getting-started/installation/).
-
-Then run:
+With `pipx` installed:
 
 ```
-git clone https://github.com/washu-ocean/cnda-dl.git
-cd cnda-dl
-uv sync
+pipx install cnda_dl
 ```

cnda_dl-1.1.0/README.md

@@ -0,0 +1,9 @@
+# Installation
+
+We recommend installing cnda-dl with [pipx](https://github.com/pypa/pipx), which ensures that the cnda-dl binary is installed in an isolated environment. Installation instructions can be found [here](https://github.com/pypa/pipx?tab=readme-ov-file#install-pipx).
+
+With `pipx` installed:
+
+```
+pipx install cnda_dl
+```

{cnda_dl-1.0.0 → cnda_dl-1.1.0}/cnda_dl/cli.py

@@ -6,11 +6,12 @@ Authors:
     Joey Scanga (scanga@wustl.edu)
     Ramone Agard (rhagard@wustl.edu)
 '''
-
+from .formatters import ParensOnRightFormatter1, Colors
 from glob import glob
 from matplotlib.ticker import EngFormatter
 from pathlib import Path
 from pyxnat import Interface
+import concurrent.futures
 import pyxnat
 import argparse
 import logging
@@ -25,8 +26,11 @@ import xml.etree.ElementTree as et
 import zipfile
 import datetime
 
+CONNECTION_POOL_SIZE = 10
+
 default_log_format = "%(levelname)s:%(funcName)s: %(message)s"
 sout_handler = logging.StreamHandler(stream=sys.stdout)
+sout_handler.setFormatter(ParensOnRightFormatter1())
 logging.basicConfig(level=logging.INFO,
                     handlers=[sout_handler],
                     format=default_log_format)
@@ -49,16 +53,16 @@ def handle_dir_creation(dir_path: Path):
         ans = ans.lower()
 
         if len(ans) != 1 or ans not in 'yn':
-            logger.info("Invalid response.")
+            logger.info("Invalid response")
         elif ans == 'y':
             dir_path.mkdir(parents=True)
             prompt_chosen = True
-            logger.info(f"new directory created at {dir_path}.")
+            logger.info(f"new directory created at {dir_path}")
         elif ans == 'n':
-            logger.info("Chose to not create a new directory. Aborting...")
+            logger.info("Chose to not create a new directory Aborting")
             sys.exit(0)
         else:
-            logger.info("Invalid response.")
+            logger.info("Invalid response")
 
 
 def download_xml(central: Interface,
@@ -66,7 +70,7 @@ def download_xml(central: Interface,
                  project_id: str,
                  file_path: Path):
 
-    logger.info("Downloading session xml...")
+    logger.info("Downloading session xml")
     sub = central.select(f"/projects/{project_id}/subjects/{subject_id}")
     with open(file_path, "w") as f:
         f.write(sub.get().decode())
@@ -109,7 +113,6 @@ def download_experiment_dicoms(session_experiment: pyxnat.jsonutil.JsonTable,
                                xml_file_path: Path,
                                scan_number_start: str = None,
                                skip_unusable: bool = False):
-
     project_id = session_experiment["project"]
     exp_id = session_experiment['ID']
 
@@ -135,53 +138,77 @@ def download_experiment_dicoms(session_experiment: pyxnat.jsonutil.JsonTable,
         logger.info(f"The following scans were marked 'unusable' and will not be downloaded: \n\t {[s for s,q in quality_pairs.items() if q=='unusable']}")
 
     # Get total number of files
-    total_file_count, cur_file_count = 0, 0
-    for s in scans:
-        files = central.select(f"/projects/{project_id}/experiments/{exp_id}/scans/{s}/resources/files").get("")
-        total_file_count += len(files)
+    def _get_file_objects_in_scan(scan: str) -> dict:
+        file_objects = central.select(f"/projects/{project_id}/experiments/{exp_id}/scans/{scan}/resources/files").get("")
+        return {file_obj: scan for file_obj in file_objects}
+
+    all_scan_file_objects = {}
+    with concurrent.futures.ThreadPoolExecutor(max_workers=CONNECTION_POOL_SIZE) as executor:
+        future_dicts = executor.map(_get_file_objects_in_scan, scans)
+        for future_dict in future_dicts:
+            all_scan_file_objects.update(future_dict)
+
+    total_file_count = len(all_scan_file_objects.keys())
     logger.info(f"Total number of files: {total_file_count}")
 
+    # Make DICOM directories for each series number
+    for scan in scans:
+        series_path = session_dicom_dir / scan / "DICOM"
+        series_path.mkdir(parents=True, exist_ok=True)
+
     # So log message does not interfere with format of the progress bar
     logger.removeHandler(sout_handler)
-    downloaded_files = set()
     zero_size_files = set()
     fmt = EngFormatter('B')
 
+    # Function assigned to threads
+    def _download_session_file(f, scan):
+        file_attrs = {}
+        series_path = session_dicom_dir / scan / "DICOM"
+        assert series_path.is_dir()
+        file_attrs = {
+            "name": series_path / f._uri.split("/")[-1],
+            "size": fmt(int(f.size())) if f.size() else fmt(0),
+            "isempty": True if not f.size() else False,
+            "isdownloaded": False
+        }
+        if file_attrs["isempty"] and file_attrs["name"].exists():
+            return file_attrs
+        f.get(file_attrs["name"])
+        file_attrs["isdownloaded"] = True
+        return file_attrs
+
     # Download the session files
-    with progressbar.ProgressBar(max_value=total_file_count, redirect_stdout=True) as bar:
-        for s in scans:
-            logger.info(f"  Downloading scan {s}...")
-            print(f"Downloading scan {s}...")
-            series_path = session_dicom_dir / s / "DICOM"
-            series_path.mkdir(parents=True, exist_ok=True)
-            files = central.select(f"/projects/{project_id}/experiments/{exp_id}/scans/{s}/resources/files").get("")
-            for f in files:
+    with (
+        progressbar.ProgressBar(max_value=total_file_count, redirect_stdout=True) as bar,
+        concurrent.futures.ThreadPoolExecutor(max_workers=CONNECTION_POOL_SIZE) as executor
+    ):
+        cur_file_count = 0
+        zero_size_files = []
+        futures = [executor.submit(_download_session_file, f, scan) for (f, scan) in all_scan_file_objects.items()]
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                file_attrs = future.result()
                 cur_file_count += 1
-                add_file = True
-                file_name = series_path / f._uri.split("/")[-1]
-                file_size = fmt(int(f.size())) if f.size() else fmt(0)
-                file_info = f"File {f.attributes()['Name']}, {file_size} ({cur_file_count} out of {total_file_count})"
-                print("\t" + file_info)
-                logger.info("\t" + file_info)
-                if not f.size():
-                    msg = "\t-- File is empty"
-                    if file_name in downloaded_files:
-                        msg += " -- another copy was already downloaded, skipping download of this file"
-                        add_file = False
-                    else:
-                        zero_size_files.add(file_name)
-                    print(msg)
-                    logger.info(msg)
-                elif file_name in zero_size_files:
-                    zero_size_files.remove(file_name)
-                if add_file:
-                    f.get(file_name)
-                    downloaded_files.add(file_name)
                 bar.update(cur_file_count)
+                if file_attrs['isempty']:
+                    zero_size_files.append(file_attrs)
+                if file_attrs['isdownloaded']:
+                    msg = f"Downloaded file {file_attrs['name']}, {file_attrs['size']} ({cur_file_count} out of {total_file_count})"
+                    if len(msg) > (tsize := os.get_terminal_size()[0]) - tsize // 5:
+                        msg = msg[:tsize // 2 - 4] + f"{Colors.DARK_GREY}.......{Colors.RESET}" + msg[tsize // 2 + tsize // 5:]
+                    # if len(msg) > (tsize := os.get_terminal_size()[0]) and tsize % 2 == 1:
+                    #     msg = msg[:tsize // 2 - 4] + f"{Colors.DARK_GREY}.......{Colors.RESET}" + msg[tsize // 2 + tsize // 5:]
+                    logger.info(msg)
+                    # logger.info(f"\tDownloaded file {file_attrs['name']}, {file_attrs['size']} ({cur_file_count} out of {total_file_count})")
+                    print(msg)
+            except Exception as exc:
+                print(f"Task ended with an exception {exc}")
+
     logger.addHandler(sout_handler)
-    logger.info("Dicom download complete \n")
+    logger.info("DICOM download complete!")
     if len(zero_size_files) > 0:
-        logger.warning(f"The following downloaded files contained no data:\n{[f.label() for f in zero_size_files]} \nCheck these files for unintended missing data!")
+        logger.warning(msg := f"The following downloaded files contained no data:\n{[file_attrs['name'] for file_attrs in zero_size_files]} \nCheck these files for unintended missing data!")
 
 
 def download_nordic_zips(session: str,
@@ -192,8 +219,8 @@ def download_nordic_zips(session: str,
     project_id = session_experiment["project"]
     exp_id = session_experiment['ID']
 
-    def __digests_identical(zip_path: Path,
-                            cnda_file: pyxnat.core.resources.File):
+    def _digests_identical(zip_path: Path,
+                           cnda_file: pyxnat.core.resources.File):
         if zip_path.is_file():  # Compare digests of zip on CNDA to see if we need to redownload
             with zip_path.open("rb") as f:
                 if hashlib.md5(f.read()).hexdigest() == cnda_file.attributes()['digest']:  # digests match
@@ -205,12 +232,12 @@ def download_nordic_zips(session: str,
     logger.info(f"Found {len(nordic_volumes)} 'NORDIC_VOLUMES' for this session")
     for nv in nordic_volumes:
         zip_path = session_dicom_dir / nv._uri.split("/")[-1]
-        if not __digests_identical(zip_path, nv):
-            logger.info(f"Downloading {zip_path.name}...")
+        if not _digests_identical(zip_path, nv):
+            logger.info(f"Downloading {zip_path.name}")
             nv.get(zip_path)
         unzip_path = zip_path.parent / zip_path.stem
         with zipfile.ZipFile(zip_path, "r") as zip_ref:
-            logger.info(f"Unzipping to {unzip_path}...")
+            logger.info(f"Unzipping to {unzip_path}")
             zip_ref.extractall(unzip_path)
         dat_dir_list.append(unzip_path)
 
@@ -286,8 +313,8 @@ def dat_dcm_to_nifti(session: str,
                     logger.warning("Could not find the mismatched dicom")
 
         # run the dcmdat2niix subprocess
-        logger.info(f"Running dcmdat2niix on series {series_id}...")
-        dcmdat2niix_cmd = shlex.split(f"dcmdat2niix -ba n -z o -w 1 -o {nifti_path} {series_path}")
+        logger.info(f"Running dcmdat2niix on series {series_id}")
+        dcmdat2niix_cmd = shlex.split(f"dcmdat2niix -ba y -z o -w 1 -o {nifti_path} {series_path}")
         with subprocess.Popen(dcmdat2niix_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
             while p.poll() is None:
                 for line in p.stdout:
@@ -318,7 +345,7 @@ def dat_dcm_to_nifti(session: str,
 def main():
     parser = argparse.ArgumentParser(
         prog="cnda-dl",
-        description="download cnda data directly to wallace",
+        description="A command-line utility for downloading fMRI data from CNDA",
     )
     parser.add_argument('session_list',
                         nargs="+",
@@ -376,7 +403,7 @@ def main():
     logger.addHandler(file_handler)
     logger.addHandler(sout_handler)
 
-    logger.info("Starting cnda-dl...")
+    logger.info("Starting cnda-dl")
     logger.info(f"Log will be stored at {log_path}")
 
     # set up data paths
@@ -399,6 +426,7 @@ def main():
 
     # main loop
     for session in session_list:
+        download_success = False
         xml_file_path = xml_path / f"{session}.xml"
         session_dicom_dir = dicom_path / session
 
@@ -415,10 +443,11 @@ def main():
                                  skip_short_runs=args.skip_short_runs)
             except Exception:
                 logger.exception(f"Error moving the .dat files to the appropriate scan directories and converting to NIFTI for session: {session}")
+                download_success = False
             continue
 
         # download the experiment data
-        logger.info(f"Starting download of session {session}...")
+        logger.info(f"Starting download of session {session}")
 
         # try to retrieve the experiment corresponding to this session
         exp = None
@@ -434,6 +463,26 @@ def main():
 
         except Exception:
             logger.exception("Error retrieving the experiment from the given parameters. Double check your inputs or enter more specific parameters.")
+            download_success = False
+            continue
+
+        # download the experiment data
+        logger.info(f"Starting download of session {session}")
+
+        # try to retrieve the experiment corresponding to this session
+        exp = None
+        try:
+            exp = retrieve_experiment(central=central,
+                                      session=session,
+                                      experiment_id=args.experiment_id,
+                                      project_id=args.project_id)
+            if len(exp) == 0:
+                raise RuntimeError("ERROR: CNDA query returned JsonTable object of length 0, meaning there were no results found with the given search parameters.")
+            elif len(exp) > 1:
+                raise RuntimeError("ERROR: CNDA query returned JsonTable object of length >1, meaning there were multiple results returned with the given search parameters.")
+
+        except Exception:
+
             continue
 
         # download the xml for this session
@@ -452,29 +501,33 @@ def main():
                                            skip_unusable=args.skip_unusable)
             except Exception:
                 logger.exception(f"Error downloading the experiment data from CNDA for session: {session}")
+                download_success = False
                 continue
 
-        # if we are not skipping the NORDIC files
-        if not args.ignore_nordic_volumes:
-            # try to download NORDIC related files and convert raw data to NIFTI
-            try:
-                nordic_dat_dirs = download_nordic_zips(session=session,
-                                                       central=central,
-                                                       session_experiment=exp,
-                                                       session_dicom_dir=session_dicom_dir)
-                nifti_path = dicom_path / f"{session}_nii"
-                for nordic_dat_path in nordic_dat_dirs:
-                    dat_dcm_to_nifti(session=session,
-                                     dat_directory=nordic_dat_path,
-                                     xml_file_path=xml_file_path,
-                                     session_dicom_dir=session_dicom_dir,
-                                     nifti_path=nifti_path,
-                                     skip_short_runs=args.skip_short_runs)
-            except Exception:
-                logger.exception(f"Error downloading 'NORDIC_VOLUMES' and converting to NIFTI for session: {session}")
-                continue
+        # exit if skipping the NORDIC files
+        if args.ignore_nordic_volumes:
+            continue
+        # try to download NORDIC related files and convert raw data to NIFTI
+        try:
+            nordic_dat_dirs = download_nordic_zips(session=session,
+                                                   central=central,
+                                                   session_experiment=exp,
+                                                   session_dicom_dir=session_dicom_dir)
+            nifti_path = dicom_path / f"{session}_nii"
+            for nordic_dat_path in nordic_dat_dirs:
+                dat_dcm_to_nifti(session=session,
+                                 dat_directory=nordic_dat_path,
+                                 xml_file_path=xml_file_path,
+                                 session_dicom_dir=session_dicom_dir,
+                                 nifti_path=nifti_path,
+                                 skip_short_runs=args.skip_short_runs)
+        except Exception:
+            logger.exception(f"Error downloading 'NORDIC_VOLUMES' and converting to NIFTI for session: {session}")
+            download_success = False
+            continue
 
-    logger.info("\n...Downloads Complete")
+    if download_success:
+        logger.info("\nDownloads Complete")
 
 
 if __name__ == "__main__":

cnda_dl-1.1.0/cnda_dl/formatters.py

@@ -0,0 +1,65 @@
+import logging
+import os
+
+
+class Colors:
+    RESET = "\033[0m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+
+    BLACK = "\033[30m"
+    RED = "\033[31m"
+    GREEN = "\033[32m"
+    YELLOW = "\033[33m"
+    BLUE = "\033[34m"
+    MAGENTA = "\033[35m"
+    CYAN = "\033[36m"
+    WHITE = "\033[37m"
+    DARK_GREY = "\033[90m"
+    LIGHT_RED = "\033[91m"
+    LIGHT_GREEN = "\033[92m"
+    LIGHT_YELLOW = "\033[93m"
+    LIGHT_BLUE = "\033[94m"
+    LIGHT_MAGENTA = "\033[95m"
+    LIGHT_CYAN = "\033[96m"
+    LIGHT_WHITE = "\033[97m"
+
+    # Background colors
+    BACK_BLACK = "\033[40m"
+    BACK_RED = "\033[41m"
+    BACK_GREEN = "\033[42m"
+    BACK_YELLOW = "\033[43m"
+    BACK_BLUE = "\033[44m"
+    BACK_MAGENTA = "\033[45m"
+    BACK_CYAN = "\033[46m"
+    BACK_WHITE = "\033[47m"
+    BACK_DARK_GREY = "\033[100m"
+    BACK_LIGHT_RED = "\033[101m"
+    BACK_LIGHT_GREEN = "\033[102m"
+    BACK_LIGHT_YELLOW = "\033[103m"
+    BACK_LIGHT_BLUE = "\033[104m"
+    BACK_LIGHT_MAGENTA = "\033[105m"
+    BACK_LIGHT_CYAN = "\033[106m"
+    BACK_LIGHT_WHITE = "\033[107m"
+
+
+class ParensOnRightFormatter1(logging.Formatter):
+    def format(self, record):
+        log_message = f"{record.msg}"
+        log_level = f"{record.levelname}"
+        func_name = (f"{record.funcName}")
+        if func_name[-1] == '.':
+            func_name[-1] = f"{Colors.DARK_GREY}.{Colors.RESET}"
+        # Determine total width of terminal window
+        terminal_width = os.get_terminal_size()[0]
+        # Calculate the right margin position for the log level and function name
+        if log_level == "INFO":
+            right_margin_text = f"{Colors.LIGHT_GREEN}({log_level}, {func_name}){Colors.RESET}"
+        elif log_level == "WARNING":
+            right_margin_text = f"{Colors.YELLOW}({log_level}, {func_name}){Colors.RESET}"
+        elif log_level == "ERROR":
+            right_margin_text = f"{Colors.RED}({log_level}, {func_name}){Colors.RESET}"
+        necessary_padding = terminal_width - len(log_message) - len(right_margin_text)
+        # Ensure padding is non-negative
+        padding = f'{Colors.DARK_GREY}.{Colors.RESET}' * max(0, necessary_padding)
+        return f"{log_message}{padding}{right_margin_text}"

{cnda_dl-1.0.0 → cnda_dl-1.1.0}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
     "pyxnat>=1.6.2",
 ]
 name = "cnda_dl"
-version = "1.0.0"
+version = "1.1.0"
 description = "A command line utility for downloading fMRI data from CNDA"
 readme = "README.md"
 
@@ -34,5 +34,6 @@ cnda-dl = "cnda_dl.cli:main"
 
 [dependency-groups]
 dev = [
+    "pudb>=2025.1",
     "pytest<9.0.0,>=8.2.2",
 ]

cnda_dl-1.0.0/README.md

@@ -1,17 +0,0 @@
-# Dev installation
-
-Ensure `uv` is installed:
-
-```
-which uv
-```
-
-If not, this can be installed following instructions [here](https://docs.astral.sh/uv/getting-started/installation/).
-
-Then run:
-
-```
-git clone https://github.com/washu-ocean/cnda-dl.git
-cd cnda-dl
-uv sync
-```