qmenta-sdk-lib 2.0.1.dev3450__tar.gz

qmenta_sdk_lib-2.0.1.dev3450/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.1
+Name: qmenta_sdk_lib
+Version: 2.0.1.dev3450
+Summary: QMENTA SDK for tool development.
+Author: QMENTA
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: PySimpleGui (>=1.0)
+Requires-Dist: enum-compat (>=0.0.1)
+Requires-Dist: future (>=0.18.2)
+Requires-Dist: humanfriendly (>=1.0)
+Requires-Dist: qmenta-client (>=1.0)
+Requires-Dist: requests (>=1.0)

qmenta_sdk_lib-2.0.1.dev3450/pyproject.toml

@@ -0,0 +1,39 @@
+[tool.poetry]
+name = "qmenta_sdk_lib"
+version = "2.0.1.dev3450"
+description = "QMENTA SDK for tool development."
+authors = ["QMENTA"]
+packages = [
+    {include = "qmenta", from = "python"}
+]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+qmenta-client = ">=1.0"
+enum-compat = ">=0.0.1"
+future = ">=0.18.2"
+requests = ">=1.0"
+humanfriendly = ">=1.0"
+PySimpleGui = ">=1.0"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.setuptools.packages.find]
+where = ["templates_tool_maker"]
+
+[tool.setuptools.package-data]
+mypkg = ["*_schema*", "*.png"]
+
+[project]
+authors = [
+  {name = "Tim Peeters", email = "tim@qmenta.com"}
+]
+maintainers = [
+  {name = "Marc Ramos", email = "marc@qmenta.com"},
+  {name = "Vicente Ferrer", email = "vicente.ferrer@qmenta.com"}
+]
+readme = {file = "README.md", content-type = "text/markdown"}

qmenta_sdk_lib-2.0.1.dev3450/python/qmenta/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

qmenta_sdk_lib-2.0.1.dev3450/python/qmenta/sdk/__init__.py

@@ -0,0 +1 @@
+__version__ = "{version}"  # This is filled by CI at build time (also used for documentation)

qmenta_sdk_lib-2.0.1.dev3450/python/qmenta/sdk/bids/make_entrypoint.py

@@ -0,0 +1,24 @@
+import sys
+from argparse import ArgumentParser
+
+entry_point_contents = """#!/bin/bash -eu
+# Add your configuration here:
+# ...
+
+# Tool start:
+exec {executable} -m qmenta.sdk.executor "$@"
+"""
+
+
+def make_entrypoint():
+
+    parser = ArgumentParser()
+    parser.add_argument("target")
+    options = parser.parse_args()
+
+    with open(options.target, "w") as fp:
+        fp.write(entry_point_contents.format(executable=sys.executable))
+
+
+if __name__ == "__main__":
+    make_entrypoint()

qmenta_sdk_lib-2.0.1.dev3450/python/qmenta/sdk/bids/wrapper.py

@@ -0,0 +1,228 @@
+import json
+import logging
+import os
+import shutil
+from subprocess import check_output, CalledProcessError, STDOUT
+
+from qmenta.sdk.directory_utils import mkdirs, TemporaryDirectory
+
+
+def tag_files(files):
+    # TODO: improve tagging
+    tfiles = []
+
+    for f in files:
+        tags = []
+
+        # Modality
+        if "T1w" in f:
+            tags.append("m:T1")
+        elif "T2w" in f:
+            tags.append("m:T2")
+
+        # Tags
+        if "brainmask" in f:
+            tags.append("mask")
+        elif "labels" in f:
+            tags.append("labels")
+            if "tissue_labels" in f:
+                tags.append("tissue_segmentation")
+        elif "biasfield" in f:
+            tags.append("bf")
+        elif "restore" in f:
+            tags.append("bfc")
+        elif "anat2" in f or "2anat" in f:
+            tags.append("warp")
+
+        tfiles.append((f, tags))
+
+    return tfiles
+
+
+def generate_file_path(original_name, sub_name, modality_name, cat_dir):
+    """
+    Returns a valid BIDS name (full path) for the given file name.
+
+    Parameters
+    ----------
+    original_name : str
+        Original name of the file.
+    sub_name : str
+        QMENTA platform subject name (SUB-XXX_...).
+    modality_name : str
+        Modality string in the BIDS format (may be part of the name).
+    cat_dir : str
+        Path to the directory where the file will be stored.
+
+    Returns
+    -------
+    str
+        The full BIDS-compatible path where the file should be downloaded.
+    """
+
+    # Try to detect if the file name is already in BIDS format
+    if original_name.startswith("sub-") and modality_name in original_name:
+
+        # Use the original name (replacing the subject ID)
+        target_name = "_".join([sub_name] + original_name.split("_")[1:])
+
+    else:
+
+        # Automatically generate a BIDS compatible name for the file
+        extension = "." + ".".join(original_name.split(".")[1:])
+        if extension in [".bval", ".bvec"]:  # In BIDS, bval and bvec use dwi as modality
+            target_name = "{}_dwi{}".format(sub_name, extension)
+        else:
+            target_name = "{}_{}{}".format(sub_name, modality_name, extension)
+
+    return os.path.join(cat_dir, target_name)
+
+
+def format_input_data(context, subject, session, path):
+    """
+    Format input data followings the BIDS specification: http://bids.neuroimaging.io/bids_spec.pdf
+
+    :param context: AnalysisContext of the running analysis
+    :param subject: Subject name (from analysis_data)
+    :param session: Session ID (from analysis_data)
+    :param path: Destination directory where the files will be placed
+
+    TODO: support group analyses (multiple input containers).
+    """
+
+    logger = logging.getLogger(__name__)
+
+    dataset_description = {"Name": "QMENTA_ANALYSIS", "BIDSVersion": "1.1.0"}
+
+    # The input container specification must include file filters following the bids_ff_names nomenclature
+    bids_ff_names = {
+        "anat": [
+            "T1w",
+            "T2w",
+            "T1rho",
+            "T1map",
+            "T2map",
+            "T2star",
+            "FLAIR",
+            "FLASH",
+            "PD",
+            "PDmap",
+            "PDT2",
+            "inplaneT1",
+            "inplaneT2",
+            "angio",
+        ],
+        "func": ["bold", "events", "physio", "sbref"],
+        "dwi": ["dwi", "bval", "bvec"],
+        "fmap": ["phasediff, magnitude1"],
+    }
+
+    supported_extensions = [".nii", ".nii.gz", ".bval", ".bvec", ".tsv"]
+
+    # BIDS spec does not allow underscores or dashes
+    sub_name = "sub-{}".format(subject.replace("-", "").replace("_", ""))
+    sub_path = os.path.join(path, sub_name)
+    mkdirs(sub_path)
+
+    with open(os.path.join(path, "dataset_description.json"), "w") as fp:
+        json.dump(dataset_description, fp)
+
+    for category in list(bids_ff_names.keys()):
+        for modality_name in bids_ff_names[category]:
+            try:
+                file_handlers = context.get_files("input", file_filter_condition_name="c_{}".format(modality_name))
+            except:  # noqa: E731,E123,E722
+                continue
+
+            for fh in file_handlers:
+
+                # NIFTI IS A MUST FOR BIDS
+                if any([fh.name.endswith(ext) for ext in supported_extensions]):
+                    cat_dir = os.path.join(sub_path, category)
+                    mkdirs(cat_dir)
+
+                    target_path = generate_file_path(fh.name, subject, modality_name, cat_dir)
+
+                    if os.path.isfile(target_path):
+                        logger.warning(
+                            "Multiple files with same name ({}). Only the first will be used".format(target_path)
+                        )
+                        break
+
+                    with TemporaryDirectory() as tmp_dir:
+                        download_path = fh.download(tmp_dir)
+                        shutil.move(download_path, target_path)
+                        logger.info("File: {}".format(target_path))
+
+                    metadata = fh.get_file_info()
+                    if metadata:
+                        metadata_target_name = "{}_{}{}".format(sub_name, modality_name, ".json")
+                        metadata_target_path = os.path.join(cat_dir, metadata_target_name)
+                        with open(metadata_target_path, "w") as fp:
+                            json.dump(metadata, fp)
+                else:
+                    logger.warning("File {} extension not supported ({})".format(fh.name, supported_extensions))
+
+
+def format_settings(settings):
+    # Bypass from JSON schema
+    # See docs.qmenta.com for more information
+
+    # FIXME: Better define inputs, for now, we'll skip common settings
+    ignore_param_list = ["input", "age_months"]
+
+    options = []
+    for param in settings:
+        if param not in ignore_param_list:
+            options += ["--" + param, str(settings[param])]
+    return options
+
+
+def run(context):
+    logger = logging.getLogger(__name__)
+
+    # Get information from the platform such as patient name, user_id, ssid...
+    analysis_data = context.fetch_analysis_data()
+    context.set_progress(message="Preparing data...")
+
+    # Create in/out dirs
+    input_path = "/analysis/input/"
+    output_path = "/analysis/output/"
+    mkdirs(input_path)
+    mkdirs(output_path)
+
+    # Download data from the platform and store following the BIDS specification
+    format_input_data(
+        context, subject=analysis_data["patient_secret_name"], session=analysis_data["ssid"], path=input_path
+    )
+
+    # Select type of analysis (TODO: add support for group analysis)
+    analysis_type = "participant"
+
+    # Prepare a list of arguments from the settings
+    options = format_settings(context.get_settings())
+
+    # Run the original BIDS-APP entrypoint
+    app_path = os.environ["BIDS_APP_ENTRYPOINT"]
+    context.set_progress(message="Running {}".format(os.path.basename(app_path)))
+    try:
+        args = [app_path, input_path, output_path, analysis_type] + options
+        logger.info("Calling: {}".format(" ".join(args)))
+        ret = check_output(args, stderr=STDOUT)
+        logger.info(ret)
+    except CalledProcessError as e:
+        raise RuntimeError("Program exited with error: {}\nOutput:\n{}".format(e, e.output))
+    except Exception:
+        raise
+
+    # Upload output files
+    context.set_progress(message="Saving output...")
+    output_files = []
+    for dirpath, _, filenames in os.walk(output_path):
+        for f in filenames:
+            file_path = os.path.abspath(os.path.join(dirpath, f))
+            output_files.append(file_path)
+
+    for f, tags in tag_files(output_files):
+        output_container_path = os.path.relpath(f, output_path)
+        context.upload_file(f, output_container_path, tags=tags)

qmenta_sdk_lib-2.0.1.dev3450/python/qmenta/sdk/client.py

@@ -0,0 +1,103 @@
+import importlib
+import inspect
+import logging
+
+from enum import Enum
+
+from qmenta.sdk.context import NoFilesError
+
+
+class AnalysisState(Enum):
+    RUNNING = "running"
+    COMPLETED = "completed"
+    EXCEPTION = "exception"
+    NO_FILES = "no_files"
+
+
+class ExecClient(object):
+    def __init__(self, analysis_id, comm, context, log_path):
+        self.__analysis_id = analysis_id
+        self.__comm = comm
+        self.__context = context
+        self.__log_path = log_path
+
+    def __call__(self, user_script_path):
+        """
+        Run an analysis provided by the user
+
+        Parameters
+        ----------
+        user_script_path : str
+            Should look like 'path.to.module:object', where 'object' should be an importable analysis tool class
+            or a function taking one 'context' argument.
+        """
+
+        logger = logging.getLogger(__name__)
+        logger.info("Launching: {}".format(user_script_path))
+
+        analysis_state = None
+        try:
+            # noinspection PyTypeChecker
+            self.set_state(AnalysisState.RUNNING, log_path=self.__log_path)
+            class_or_func = load_user_analysis(user_script_path)
+            self.__context.fetch_analysis_data()
+
+            # run the analysis
+            # may allow classes to be used later
+            assert inspect.isfunction(class_or_func)
+            self.__context.set_progress(value=0, message="Running")
+            class_or_func(self.__context)
+        except NoFilesError:
+            logger.warning("No files found")
+            analysis_state = AnalysisState.NO_FILES
+        except Exception as e:
+            logger.exception(e)
+            self.__context.set_progress(message="Error")
+            analysis_state = AnalysisState.EXCEPTION
+            raise e
+        else:
+            self.__context.set_progress(message="Completed", value=100)
+            analysis_state = AnalysisState.COMPLETED
+        finally:
+            self.upload_log()
+            # noinspection PyTypeChecker
+            self.set_state(analysis_state)
+
+    def set_state(self, state, **kwargs):
+        """
+        Set analysis state
+
+        Parameters
+        ----------
+        state : AnalysisState
+            One of AnalysisState.RUNNING, AnalysisState.COMPLETED, AnalysisState.EXCEPTION or AnalysisState.NO_FILES
+        kwargs : dict
+        """
+        assert isinstance(state, AnalysisState), "'state' is {}, not an 'AnalysisState'".format(state)
+        data_to_send = {"analysis_id": self.__analysis_id, "state": state.value}
+        data_to_send.update(kwargs)
+        logging.getLogger(__name__).info("State = {state!r} for analysis {analysis_id}".format(**data_to_send))
+        self.__comm.send_request("analysis_manager/set_analysis_state", data_to_send)
+
+    def upload_log(self):
+
+        data_to_send = {"analysis_id": self.__analysis_id}
+        log_contents = ""
+        try:
+            with open(self.__log_path, "r") as log_file:
+                log_contents = log_file.read()
+        except IOError:
+            log_contents = "Log file error ({})".format(self.__log_path)
+        finally:
+            # Send file in one single request
+            # FIXME Implement chuncked upload
+            self.__comm.send_files(
+                "analysis_manager/upload_log", files={"file": ("exec.log", log_contents)}, req_data=data_to_send
+            )
+
+
+def load_user_analysis(user_script_path):
+    module_name, class_or_func_name = user_script_path.split(":")
+    user_module = importlib.import_module(module_name)
+    class_or_func = getattr(user_module, class_or_func_name)
+    return class_or_func

qmenta_sdk_lib-2.0.1.dev3450/python/qmenta/sdk/communication.py

@@ -0,0 +1,273 @@
+import json
+import logging
+import time
+
+import requests
+
+try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urlparse
+
+
+class CommunicationObject(object):
+    def __init__(self, url, project_id, token, timeout=900.0, chunk_size=256, verify=True, critical=False):
+        """
+        Communications with the platform
+
+        Parameters
+        ----------
+        url : str
+            Platform URL
+        project_id : int
+            Unique project identifier
+        token : str
+            Unique token that authenticates the communication
+        timeout : float or tuple, optional
+            How many seconds to wait for the server to send data before giving up, as a float, or a
+            (connect timeout, read timeout) tuple. Defaults to
+        chunk_size : int
+            Size of chunks in bytes (1MB = 1024KB, 1KB = 1024B)
+        verify : bool, optional
+            Controls whether the server's TLS certificate is verified.
+            Defaults to True.
+        critical: bool, optional
+            Overrides total_retry seting in requests by making themm fail after the first error if set.
+        """
+        self.timeout = timeout
+        self.chunk_size = chunk_size
+
+        self.__req_url = url
+        self.__project_id = project_id
+        self.__token = token
+        self.__verify = verify
+        self.__critical = critical
+
+        self.__session = requests.Session()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        ret = self.send_request("logout")
+        if not ret["success"]:
+            # Not being able to logout is a security issue and must be a fatal error
+            raise RuntimeError("Unable to logout")
+
+    def send_request(
+        self,
+        access_point=None,
+        req_data=None,
+        req_headers=None,
+        return_json=True,
+        timeout=None,
+        total_retry=5,
+        stream=False,
+        url=None,
+        method="POST",
+        **kwargs
+    ):
+        """
+        Send a request to the platform to the given API endpoint.
+
+        Parameters
+        ----------
+        access_point : str, optional
+            API endpoint. Will be joined with the request URL provided at construction time to create the
+            full ("absolute") url
+        req_data : dict or str, optional
+            Request data
+        req_headers : dict, optional
+            Request headers
+        return_json : bool, optional
+            Whether the response should be a JSON object or a requests.Response instance.
+            Defaults to True.
+        timeout : float, optional
+            Timeout in seconds for this request.
+            Defaults to the timeout set at construction time.
+        total_retry : int, optional
+            Number of connection retries.
+            Defaults to 5.
+        stream : bool, optional
+            If False, the response content will be immediately downloaded.
+        url : str, optional
+            When no access_point is provided, a full url must be provided through this argument
+        kwargs : dict
+            Additional arguments to be passed to `requests.Session.post`
+        method: str
+            HTTP verb (for instance POST or GET)
+
+        Returns
+        -------
+        requests.Response or dict
+            JSON response encoded in a dictionary if return_json is True (default),
+            otherwise requests.Response object.
+        """
+        logger = logging.getLogger(__name__)
+
+        # Default parameters
+        req_data = req_data or dict()
+        req_headers = req_headers or dict()
+        timeout = timeout or self.timeout
+
+        if access_point:
+            # Add authentication for the platform
+            req_headers["Cookie"] = "AUTH_COOKIE={}; MINT_AP={}".format(self.__token, self.__project_id)
+            req_headers["Mint-Api-Call"] = "1"
+            url = urlparse.urljoin(self.__req_url, access_point)
+
+        retry_counter = 0
+        while True:
+            try:
+                response = self.__session.request(
+                    method=method,
+                    url=url,
+                    data=req_data,
+                    headers=req_headers,
+                    timeout=timeout,
+                    verify=self.__verify,
+                    stream=stream,
+                    **kwargs
+                )
+
+                # log & raise error if we get a HTTP error code
+                if not response.ok:
+                    logger.debug("{!r} HTTP response: {!r}".format(url, response.text))
+                response.raise_for_status()
+
+            except (requests.HTTPError, requests.Timeout, requests.ConnectionError):
+                retry_counter += 1
+                if self.__critical or total_retry == retry_counter:
+                    raise
+                time.sleep(retry_counter * 5)
+
+            else:
+                return json.loads(response.text) if return_json else response
+
+    def send_files(
+        self,
+        access_point,
+        files,
+        req_data=None,
+        req_headers=None,
+        return_json=True,
+        timeout=None,
+        total_retry=5,
+        stream=False,
+        **kwargs
+    ):
+        """
+        Send a POST request to the platform to the given API endpoint.
+
+        Parameters
+        ----------
+        access_point : str
+            API endpoint. Will be joined with the request URL provided at construction time to create the
+            full ("absolute") url
+        files : dict
+            Request files
+        req_data : dict or str, optional
+            Request data
+        req_headers : dict, optional
+            Request headers
+        return_json : bool, optional
+            Whether the response should be a JSON object or a requests.Response instance.
+            Defaults to True.
+        timeout : float, optional
+            Timeout in seconds for this request.
+            Defaults to the timeout set at construction time.
+        total_retry : int, optional
+            Number of connection retries.
+            Defaults to 5.
+        stream : bool, optional
+            If False, the response content will be immediately downloaded.
+        kwargs : dict
+            Additional arguments to be passed to `requests.Session.post`
+
+        Returns
+        -------
+        requests.Response or dict
+            JSON response encoded in a dictionary if return_json is True (default),
+            otherwise requests.Response object.
+        """
+        logger = logging.getLogger(__name__)
+
+        # Default parameters
+        req_data = req_data or dict()
+        req_headers = req_headers or dict()
+        timeout = timeout or self.timeout
+
+        # Add authentication for the platform
+        req_headers["Cookie"] = "AUTH_COOKIE={}; MINT_AP={}".format(self.__token, self.__project_id)
+        req_headers["Mint-Api-Call"] = "1"
+
+        retry_counter = 0
+        while True:
+            try:
+                url = urlparse.urljoin(self.__req_url, access_point)
+                response = self.__session.post(
+                    url,
+                    files=files,
+                    data=req_data,
+                    headers=req_headers,
+                    timeout=timeout,
+                    verify=self.__verify,
+                    stream=stream,
+                    **kwargs
+                )
+
+                # Raise error if we get a HTTP error code
+                response.raise_for_status()
+                result = json.loads(response.text) if return_json else response
+                logger.info("{!r} HTTP response: {!r}".format(url, result))
+                return result
+
+            except (requests.HTTPError, requests.Timeout, requests.ConnectionError):
+                retry_counter += 1
+                if self.__critical or total_retry == retry_counter:
+                    raise
+                time.sleep(retry_counter * 5)
+
+
+class DummyCommunicationObject(object):
+    def __init__(self, url, chunk_size=1, mock_response=None):
+        self.__req_url = url
+        self.chunk_size = chunk_size
+        self.mock_response = mock_response
+        self.sent_contents = None
+
+    def send_request(
+        self,
+        access_point=None,
+        req_data=None,
+        req_headers=None,
+        return_json=True,
+        timeout=None,
+        total_retry=5,
+        stream=False,
+        url=None,
+        method="POST",
+        **kwargs
+    ):
+        self.sent_contents = req_data
+        if stream and method == "GET":
+            return DummmyResponse()
+        elif self.mock_response is None or stream:
+            return DummmyResponse()
+        else:
+            return self.mock_response
+
+
+class DummmySession(object):
+
+    def get(self, url, stream=False):
+        if stream:
+            return DummmyResponse()
+        else:
+            return "{}"
+
+
+class DummmyResponse(object):
+
+    def iter_content(self, *args, **kwargs):
+        return []