cellmaps-utils 0.10.0__py3-none-any.whl

cellmaps_utils/logutils.py

@@ -0,0 +1,214 @@
+
+import os
+import time
+import json
+import getpass
+import platform
+import logging
+import logging.config
+import argparse
+from cellmaps_utils import constants
+
+
+logger = logging.getLogger(__name__)
+
+
+def setup_cmd_logging(args):
+    """
+    Sets up logging based on parsed command line arguments.
+    If **args.logconf** is set use that configuration otherwise look
+    at **args.verbose** and set logging for this module
+
+    This function assumes the following:
+
+    * **args.logconf** exists and is ``None`` or set to :py:class:`str`
+      containing path to logconf file
+
+    * **args.verbose** exists and is set to :py:class:`int` to one of
+      these values:
+
+      * ``0`` = no logging
+      * ``1`` = critical
+      * ``2`` = error
+      * ``3`` = warning
+      * ``4`` = info
+      * ``5`` = debug
+
+    :param args: parsed command line arguments from argparse
+    :type args: :py:class:`argparse.Namespace`
+    :raises AttributeError: If args is ``None`` or
+                            if **args.logconf** is None or missing or
+                            if **args.verbose** is None or missing
+    """
+
+    if args.logconf is None:
+        level = (50 - (10 * args.verbose))
+        logging.basicConfig(format=constants.LOG_FORMAT,
+                            level=level)
+        logger.setLevel(level)
+        return
+
+    # logconf was set use that file
+    logging.config.fileConfig(args.logconf,
+                              disable_existing_loggers=False)
+
+
+def write_task_start_json(outdir=None, start_time=None,
+                          data=None,
+                          version=None):
+    """
+    Writes :py:const:`~cellmaps_utils.constants.TASK_FILE_PREFIX` **##** :py:const:`~cellmaps_utils.constants.TASK_START_FILE_SUFFIX`
+    file with information about
+    what is to be run. The **##** in name is value of **start_time**
+
+    .. code-block::
+
+        from cellmaps_utils import logutils
+        import time
+
+        logutils.write_task_start_json(outdir='./mydir', start_time=int(time.time()),
+                                       data={'someparam': 'some value'},
+                                       version='1.0.0')
+
+    :param outdir: directory to write file
+    :type outdir: str
+    :param start_time: time in seconds since epoch
+                       If ``-1`` or ``None`` then value will
+                       be set to current time
+    :type start_time: int
+    :param data: additional data to persist in
+    :type data: dict
+    :param version: Version of software
+    :type version: str
+    """
+    login = ''
+    try:
+        login = getpass.getuser()
+    except Exception as e:
+        logger.error('Unable to get login for user: ' + str(e))
+
+    if start_time is None or start_time == -1:
+        start_time = int(time.time())
+
+    task = {'start_time': start_time,
+            'version': str(version),
+            'pid': str(os.getpid()),
+            'outdir': outdir,
+            'login': login,
+            'cwd': str(os.getcwd()),
+            'platform': str(platform.platform()),
+            'python': str(platform.python_version()),
+            'system': str(platform.system()),
+            'uname': str(platform.uname())
+            }
+    if data is not None:
+        task.update(data)
+
+    with open(os.path.join(outdir,
+                           constants.TASK_FILE_PREFIX +
+                           str(start_time) +
+                           constants.TASK_START_FILE_SUFFIX), 'w') as f:
+        json.dump(task, f, indent=2)
+
+
+def write_task_finish_json(outdir=None, start_time=None,
+                           end_time=None, status=None):
+        """
+        Writes :py:const:`~cellmaps_utils.constants.TASK_FILE_PREFIX` **##** :py:const:`~cellmaps_utils.constants.TASK_FINISH_FILE_SUFFIX`
+        file in **outdir** directory where **##** is the **start_time** value
+
+        .. code-block::
+
+            from cellmaps_utils import logutils
+            import time
+
+            logutils.write_task_finish_json(outdir='./mydir', start_time=int(time.time())-10,
+                                            end_time=int(time.time()),
+                                            status=0)
+
+        :param outdir: directory to write file
+        :type outdir: str
+        :param start_time: time in seconds since epoch
+                           if set to ``-1`` or ``None`` value will be set
+                           to current time
+        :type start_time: int
+        :param end_time: time in seconds since epoch
+                         if set to ``-1`` or ``None`` value will be set to
+                         current time
+        :type end_time: int
+        :param status: status of task, ``0`` means success, otherwise error
+        :type status: int
+        """
+        if outdir is None or not os.path.isdir(outdir):
+            logger.error('Output directory is not set or not a '
+                         'directory, cannot write'
+                         'task finish json file')
+            return
+
+        if start_time is None or start_time == -1:
+            start_time = int(time.time())
+
+        if end_time is None or end_time == -1:
+            end_time = int(time.time())
+        task = {'end_time': end_time,
+                'elapsed_time': int(end_time - start_time),
+                'status': str(status)
+                }
+        with open(os.path.join(outdir,
+                               constants.TASK_FILE_PREFIX +
+                               str(start_time) +
+                               constants.TASK_FINISH_FILE_SUFFIX), 'w') as f:
+            json.dump(task, f, indent=2)
+
+
+def setup_filelogger(outdir=None, handlerprefix='cellmaps'):
+    """
+    Sets up a logger to write all debug and higher logs
+    to output **outdir**/:py:const:`~cellmaps_utils.constants.OUTPUT_LOG_FILE`
+    and all error level log messages and higher
+    to output **outdir**/:py:const:`~cellmaps_utils.constants.ERROR_LOG_FILE`
+
+    :param outdir: directory where to store :py:const:`~cellmaps_utils.constants.OUTPUT_LOG_FILE`
+                   and :py:const:`~cellmaps_utils.constants.ERROR_LOG_FILE` files
+    :type outdir: str
+    :param handlerprefix: prefix of name to give to handlers and formatters,
+                          if ``None`` code will set value to ``cellmaps``
+    :type handlerprefix: str
+    """
+    if handlerprefix is None:
+        handlerprefix = 'cellmaps'
+
+    err_log_file = os.path.join(outdir, constants.ERROR_LOG_FILE)
+    out_log_file = os.path.join(outdir, constants.OUTPUT_LOG_FILE)
+    logging.config.dictConfig({'version': 1,
+                               'propagate': True,
+                               'disable_existing_loggers': False,
+                               'loggers': {
+                                 '': {
+                                     'level': 'NOTSET',
+                                     'handlers': [handlerprefix + '_file_handler',
+                                                  handlerprefix + '_error_file_handler']
+                                 }
+                               },
+                               'handlers': {
+                                   handlerprefix + '_file_handler': {
+                                       'level': 'DEBUG',
+                                       'class': 'logging.FileHandler',
+                                       'formatter': handlerprefix + '_formatter',
+                                       'filename': out_log_file,
+                                       'mode': 'a'
+                                   },
+                                   handlerprefix + '_error_file_handler': {
+                                       'level': 'ERROR',
+                                       'class': 'logging.FileHandler',
+                                       'formatter': handlerprefix + '_formatter',
+                                       'filename': err_log_file,
+                                       'mode': 'a'
+                                   }
+                               },
+                               'formatters': {
+                                   handlerprefix + '_formatter': {
+                                       'format': constants.LOG_FORMAT
+                                   }
+                               }
+                               })

cellmaps_utils/music_utils.py

@@ -0,0 +1,246 @@
+import numpy as np
+import pandas as pd
+import pickle
+import dill
+from sklearn.metrics.pairwise import manhattan_distances, euclidean_distances, cosine_similarity
+from scipy.spatial.distance import canberra
+
+
+def upper_tri_values(df):
+    """
+    Return array with values of upper triangle of the DataFrame
+
+    :param df: Symmetric DataFrame
+    :type df: :py:class:`pandas.DataFrame`
+    :return:
+    :rtype: :py:func:`numpy.array`
+    """
+    m = df.values
+    return m[np.triu_indices(df.shape[0], k=1)]
+
+
+def znorm(df):
+    """
+    Z-transform within each column.
+
+    :param df:
+    :return:
+    :rtype: :py:class:`pandas.DataFrame`
+    """
+    norm_df = pd.DataFrame(index=df.index, columns=df.columns)
+    for c in df.columns:
+        value = df[c]
+        norm_df[c] = (value - value.mean()) / value.std()
+    return norm_df
+
+
+def cosine_similarity_scaled(df):
+    """
+    Calculate Cosine similarity between each pair of rows in a DataFrame.
+    Similarity scaled into [0, 1]
+
+    :param df:
+    :return:
+    :rtype: :py:class:`pandas.DataFrame`
+    """
+    sim = cosine_similarity(df)
+    shift = sim.min()
+    sim -= shift
+    scale = sim.max()
+    sim /= scale
+    return pd.DataFrame(sim, index=df.index.values, columns=df.index.values)
+
+
+def manhattan_similarity(df):
+    """
+    Calculate Manhattan similarity between each pair of rows in a DataFrame.
+    Similarity scaled into [0, 1]
+
+    :param df:
+    :return:
+    :rtype: :py:class:`pandas.DataFrame`
+    """
+    # Get manhattan distance
+    dist = manhattan_distances(df)
+    # Convert distance to similarity by max-minus
+    sim = dist.max() - dist
+    # Scale into [0,1]
+    sim /= sim.max()
+    return pd.DataFrame(sim, index=df.index.values, columns=df.index.values)
+
+
+def euclidean_similarity(df):
+    """
+    Calculate Euclidean similarity between each pair of rows in a DataFrame.
+    Similarity scaled into [0, 1]
+
+    :param df:
+    :return:
+    :rtype: :py:class:`pandas.DataFrame`
+    """
+    # Get euclidean distance
+    dist = euclidean_distances(df)
+    # Convert distance to similarity by max-minus
+    sim = dist.max() - dist
+    # Scale into [0,1]
+    sim /= sim.max()
+    return pd.DataFrame(sim, index=df.index.values, columns=df.index.values)
+
+
+def canberra_similarity(df):
+    """
+    Calculate Canberra similarity between each pair of rows in a DataFrame.
+    Similarity scaled into [0, 1]
+
+    :param df:
+    :return:
+    :rtype: :py:class:`pandas.DataFrame`
+    """
+    index = df.index.values
+    dist = pd.DataFrame(0, index=index, columns=index, dtype=float)
+    for i in range(len(index) - 1):
+        a = df.loc[index[i]].values
+        for j in range(i + 1, len(index)):
+            b = df.loc[index[j]].values
+            d = canberra(a, b)
+            dist.at[index[i], index[j]] = d
+            dist.at[index[j], index[i]] = d
+    dist = dist.values
+    # Convert distance to similarity by max-minus
+    sim = dist.max() - dist
+    # Scale into [0,1]
+    sim /= sim.max()
+    return pd.DataFrame(sim, index=index, columns=index)
+
+
+def pearson_scaled(df):
+    """
+    Calculate Pearson correlation between each pair of rows in a DataFrame.
+    Correlation scaled into [0, 1]
+
+    :param df:
+    :return:
+    :rtype: :py:class:`pandas.DataFrame`
+    """
+    corr = df.T.corr(method='pearson')
+    shift = corr.min().min()
+    corr -= shift
+    scale = corr.max().max()
+    corr /= scale
+    return corr
+
+
+def spearman_scaled(df):
+    """
+    Calculate Spearman correlation between each pair of rows in a DataFrame.
+    Correlation scaled into [0, 1]
+
+    :param df:
+    :return:
+    """
+    corr = df.T.corr(method='spearman')
+    shift = corr.min().min()
+    corr -= shift
+    scale = corr.max().max()
+    corr /= scale
+    return corr
+
+
+def kendall_scaled(df):
+    """
+    Calculate Kendall correlation between each pair of rows in a DataFrame.
+    Correlation scaled into [0, 1]
+
+    :param df:
+    :return:
+    """
+    corr = df.T.corr(method='kendall')
+    shift = corr.min().min()
+    corr -= shift
+    scale = corr.max().max()
+    corr /= scale
+    return corr
+
+
+def check_symmetric(a, rtol=1e-05, atol=1e-08):
+    """
+    Check if the given numpy matrix is symmetric or not.
+
+    :param a:
+    :param rtol:
+    :param atol:
+    :return:
+    """
+    return np.allclose(a, a.T, rtol=rtol, atol=atol)
+
+
+def save_obj(obj, fname, method='pickle'):
+    """
+
+    :param obj: object that want to be saved
+    :param fname: path to saved file
+    :type fname: str
+    :param method: {pickle, dill} specify package used for compressing
+    :type method: str
+    :raises ValueError: if **method** is not set to ``pickle`` or ``dill``
+    """
+    with open(fname, 'wb') as f:
+        if method == 'pickle':
+            pickle.dump(obj, f)
+        elif method == 'dill':
+            dill.dump(obj, f)
+        else:
+            raise ValueError('Please select method from {pickle, dill}!')
+    return
+
+
+def load_obj(fname, method='pickle'):
+    """
+    Loading object that was saved in pickle format
+
+    :param fname: path to file
+    :type fname: str
+    :param method: {pickle, dill} specify package used for compressing
+    :type method: str
+    :raises ValueError: if **method** is not set to ``pickle`` or ``dill``
+    """
+    with open(fname, 'rb') as f:
+        if method == 'pickle':
+            return pickle.load(f)
+        elif method == 'dill':
+            return dill.load(f)
+        else:
+            raise ValueError('Please select method from {pickle, dill}!')
+        return
+
+
+def jaccard(setA, setB):
+    """
+    Calculates jaccard
+
+    :param setA:
+    :param setB:
+    :return:
+    """
+    return len(setA.intersection(setB)) / len(setA.union(setB))
+
+
+def scaled_P_to_nm(scaled_P):
+    """
+    TODO: Add doc here
+
+    :param scaled_P:
+    :return:
+    """
+    power = -3.968 * scaled_P + 4.326
+    return 10 ** power
+
+
+def num_comb(x):
+    """
+    TODO: Add doc here
+
+    :param x:
+    :return:
+    """
+    return x * (x - 1) / 2

cellmaps_utils/ndexupload.py

@@ -0,0 +1,193 @@
+import os
+import time
+import uuid
+
+import ndex2
+import logging
+import copy
+from cellmaps_utils import constants
+
+from ndex2.cx2 import RawCX2NetworkFactory, CX2Network
+from requests import RequestException
+
+from cellmaps_utils.exceptions import CellMapsError
+
+logger = logging.getLogger(__name__)
+
+
+class NDExHierarchyUploader(object):
+    """
+    Base class for uploading hierarchy and its parent network to NDEx.
+    """
+
+    def __init__(self, ndexserver, ndexuser, ndexpassword, visibility=None):
+        """
+        Constructor
+
+        :param ndexserver:
+        :type ndexserver: str
+        :param ndexuser:
+        :type ndexuser: str
+        :param ndexpassword:
+        :type ndexpassword: str
+        :param visibility: If set to ``public``, ``PUBLIC`` or ``True`` sets hierarchy and interactome to
+                           publicly visibility on NDEx, otherwise they are left as private
+        :type visibility: str or bool
+        """
+        self._server = ndexserver
+        self._user = ndexuser
+        if ndexpassword is not None and os.path.isfile(ndexpassword):
+            with open(ndexpassword, 'r') as file:
+                self._password = file.readline().strip()
+        else:
+            self._password = ndexpassword
+        self._visibility = None
+        if visibility is not None:
+            if isinstance(visibility, bool):
+                if visibility is True:
+                    self._visibility = 'PUBLIC'
+            elif isinstance(visibility, str):
+                if visibility.lower() == 'public':
+                    self._visibility = 'PUBLIC'
+        self._ndexclient = None
+        self._initialize_ndex_client()
+
+    def _initialize_ndex_client(self):
+        """
+        Creates NDEx client
+        :raises CellmapsGenerateHierarchyError: If the NDEx server URL or user credentials are not specified.
+        """
+        logger.debug('Connecting to NDEx server: ' + str(self._server) +
+                     ' with user: ' + str(self._user))
+        if self._server is None or self._user is None or self._password is None:
+            raise CellMapsError('NDEx server or credentials were not specified. '
+                                'NDEx client connection cannot be established.')
+        self._ndexclient = ndex2.client.Ndex2(host=self._server,
+                                              username=self._user,
+                                              password=self._password,
+                                              skip_version_check=True)
+
+    def _save_network(self, network, max_retries=3, retry_wait=10):
+        """
+        This method saves a network to the NDEx server and returns the unique NDEx UUID for the network
+        along with its URL. The visibility of the saved network is determined by the variable `self._visibility`.
+
+        :param network: Network to save
+        :type network: :py:class:`~ndex2.cx2.CX2Network`
+        :return: NDEX UUID of network
+        :rtype: str
+        """
+        logger.debug('Saving network named: ' + str(network.get_name()) +
+                     ' to NDEx with visibility set to: ' + str(self._visibility))
+        retry_num = 1
+        while retry_num <= max_retries:
+            try:
+                res = self._ndexclient.save_new_cx2_network(network.to_cx2(),
+                                                            visibility=self._visibility)
+                if not isinstance(res, str):
+                    raise CellMapsError('Expected a str, but got this: ' + str(res))
+
+                ndexuuid = res[res.rfind('/') + 1:]
+
+                return ndexuuid, res.replace('v3', 'viewer')
+            except RequestException as re:
+                if retry_num == max_retries:
+                    raise CellMapsError(str(max_retries) + 'attempts to save the network failed.')
+                logger.debug(str(re.response.text))
+                retry_num += 1
+                time.sleep(retry_wait)
+            except Exception as e:
+                raise CellMapsError('An error occurred while saving the network to NDEx: ' + str(e))
+
+    def get_cytoscape_url(self, ndexurl):
+        """
+        Generates a Cytoscape URL for a given NDEx network URL.
+
+        :param ndexurl: The URL of the NDEx network.
+        :type ndexurl: str
+        :return: The URL pointing to the network's view on the Cytoscape platform.
+        :rtype: str
+        """
+        ndexuuid = ndexurl[ndexurl.rfind('/') + 1:]
+        network_url = (f"https://{self._server.replace('https://', '').replace('http://', '')}"
+                       f"/cytoscape/0/networks/{ndexuuid}")
+        return network_url
+
+    def _update_hcx_annotations(self, hierarchy, interactome_id):
+        """
+        This method updates the given network hierarchy with specific HCX annotations. These annotations
+        are associated with the interactome ID and the NDEx server where the interactome resides.
+
+        :param hierarchy: The network hierarchy that needs to be updated with HCX annotations.
+        :type hierarchy: `~ndex2.cx2.CX2Network`
+        :param interactome_id: The unique ID (UUID) of the interactome that is associated with the hierarchy.
+        :type interactome_id: str
+        :return: The updated hierarchy with the HCX annotations.
+        :rtype: `~ndex2.cx2.CX2Network`
+        """
+        hierarchy_copy = copy.deepcopy(hierarchy)
+        hierarchy_copy.add_network_attribute('HCX::interactionNetworkUUID', str(interactome_id))
+        hierarchy_copy.remove_network_attribute('HCX::interactionNetworkName')
+        return hierarchy_copy
+
+    def save_hierarchy_and_parent_network(self, hierarchy, parent_ppi):
+        """
+        Saves both the hierarchy and its parent network to the NDEx server. This method first saves the parent
+        network, then updates the hierarchy with HCX annotations based on the parent network's UUID, and
+        finally saves the updated hierarchy. It returns the UUIDs and URLs for both the hierarchy and
+        the parent network.
+
+        :param hierarchy: The hierarchy network to be saved.
+        :type hierarchy: :py:class:`~ndex2.cx2.CX2Network`
+        :param parent_ppi: The parent protein-protein interaction network associated with the hierarchy.
+        :type parent_ppi: :py:class:`~ndex2.cx2.CX2Network`
+        :return: UUIDs and URLs for both the parent network and the hierarchy.
+        :rtype: tuple
+        """
+        parent_url = None
+        if isinstance(parent_ppi, CX2Network):
+            parent_uuid, parent_url = self._save_network(parent_ppi)
+        else:
+            try:
+                _ = uuid.UUID(parent_ppi, version=4)
+                parent_uuid = parent_ppi
+            except ValueError:
+                raise CellMapsError(f'Invalid UUID format for parent_ppi: {parent_ppi}')
+
+        hierarchy_for_ndex = self._update_hcx_annotations(hierarchy, parent_uuid)
+        hierarchy_uuid, hierarchy_url = self._save_network(hierarchy_for_ndex)
+        return parent_uuid, parent_url, hierarchy_uuid, hierarchy_url
+
+    def upload_hierarchy_and_parent_network_from_files(self, hier_dir=None, hierarchy_path=None, parent_path=None):
+        """
+        Uploads hierarchy and parent network to NDEx from CX2 files.
+        It first checks if hierarchy_path and parent_path are provided.
+        If not provided, it tries to get them from hier_dir directory.
+        If none is specified or cannot find hierarchy and parent in hier_dir, it raises an exception.
+
+        :param hier_dir: The directory where the hierarchy and parent network files are located.
+        :type hier_dir: str
+        :param hierarchy_path: The path to the hierarchy network file.
+        :type hierarchy_path: str, optional
+        :param parent_path: The path to the parent network file.
+        :type parent_path: str, optional
+        :return: UUIDs and URLs for both the hierarchy and parent network.
+        :rtype: tuple
+        :raises CellMapsError: If the required hierarchy or parent network files do not exist.
+        """
+        if not hierarchy_path and hier_dir:
+            hierarchy_path = os.path.join(hier_dir, constants.HIERARCHY_NETWORK_PREFIX + constants.CX2_SUFFIX)
+        if not parent_path and hier_dir:
+            parent_path = os.path.join(hier_dir, constants.HIERARCHY_PARENT_NETWORK_PREFIX + constants.CX2_SUFFIX)
+
+        if not hierarchy_path or not os.path.exists(hierarchy_path):
+            raise CellMapsError(f'Hierarchy network file does not exist at {hierarchy_path}.')
+
+        if not parent_path or not os.path.exists(parent_path):
+            raise CellMapsError(f'Parent network file does not exist at {parent_path}.')
+
+        cx_factory = RawCX2NetworkFactory()
+        hierarchy_network = cx_factory.get_cx2network(hierarchy_path)
+        parent_network = cx_factory.get_cx2network(parent_path)
+
+        return self.save_hierarchy_and_parent_network(hierarchy_network, parent_network)