hogprof 0.3.0__py3-none-any.whl

hogprof/utils/goautils.py

@@ -0,0 +1,229 @@
+
+from __future__ import print_function
+
+from goatools import semantic
+from pyoma.browser.models import ProteinEntry
+from goatools.obo_parser import GODag
+from . import hashutils
+import pickle
+from goatools.go_enrichment import GOEnrichmentStudy
+from pyoma.browser import db
+import multiprocessing as mp
+from tables import *
+import time
+##############enrichment##############################################
+
+def return_enrichment_study_obj(gaf_taxfiltered, obo = None):
+    '''
+    Generate go enrichment study object with a background dataset.
+    '''
+    if obo is None:
+        obodag = GODag(config_utils.datadir+"/GOData/go-basic.obo")
+    else:
+        obodag = GODag(obo)
+
+    goeaobj = GOEnrichmentStudy(
+        gaf_taxfiltered.keys(), #
+        gaf_taxfiltered, # geneid/GO associations possible with tree used for DB
+        obodag, # Ontologies
+        propagate_counts = False,
+        alpha = 0.15, # default significance cut-off
+        methods = ['fdr_bh']) # defult multipletest correction method
+    return goeaobj
+
+
+def buildGAF(gaf_file , universe= None):
+
+    gaf_filtered = {}
+    with open(gaf_file, mode='r') as gafin:
+        for line in gafin:
+            words = line.split()
+            if words[0] not in gaf_filtered:
+                gaf_filtered[words[0]]=set([words[1]])
+            else:
+                gaf_filtered[words[0]].add(words[1])
+
+    if universe:
+        gaf_filtered = { prot:gaf_filtered[prot] for prot in universe}
+
+
+    return gaf_filtered
+
+
+def return_hogs_timeout( result, retq):
+    print('started')
+    with open_file(config_utils.config['dir']['omadir']+'OmaServer.h5' , mode="r") as h5_oma:
+        db_obj = db.Database(h5_oma)
+        res =  [ ProteinEntry(db_obj, e).omaid for  e in db_obj.member_of_fam(int(result)) ]
+        retq.put(res)
+
+def run_GOEA_onresults(results, db_obj, goeaobj, outname = None):
+    '''
+        Perform enrichment analysis on returned results
+        grabs all member protein of all hogs in result
+        returns goe results and HOG composition
+    '''
+    hogids =[ "HOG:" + (7-len(str(fam_id))) * '0' + str(fam_id) for fam_id in results ]
+    HOGS={}
+    print('compiling hogs')
+    prots = []
+
+    print('mod13')
+    retq = mp.Queue()
+
+    for i,result in enumerate(results):
+        if i %10 ==0:
+            print(i)
+        print(result)
+        HOGS[result]=[]
+        p = mp.Process( target= return_hogs_timeout , args= (result, retq))
+        p.start()
+
+        t0 = time.time()
+        timeout = False
+
+        while time.time()-t0 < 10 :
+            time.sleep(.1)
+            if p.is_alive() == False:
+                print('done')
+                break
+        if time.time()-t0 > 10:
+            timeout = True
+            print('Dead')
+        p.terminate()
+        del p
+
+
+        if retq.empty() == False:
+            iterobj = retq.get(10)
+            #retq get
+            for k,member in enumerate(iterobj):
+                if k < 1:
+                    print(member)
+                if k > 500:
+                    break
+                HOGS[result].append(member)
+                prots.append(member)
+
+    print('done')
+    print('running GO enrichment study')
+    goea_results_all = goeaobj.run_study(prots )
+    print('done')
+    if outname:
+        with open( config_utils.datadir + outname + 'Hogs2Prots.pkl' , 'wb' ) as save:
+            save.write(pickle.dumps(HOGS,2))
+        goeaobj.wr_txt(config_utils.datadir+ str(outname)+"enrichment.txt", goea_results_all)
+    print('DONE!')
+    return goea_results_all, HOGS
+
+
+def run_GOEA_onresults_tar(results, tar, goeaobj, outname = None):
+    '''
+        Perform enrichment analysis on returned results
+        grabs all member protein of all hogs in result
+        returns goe results and HOG composition
+    '''
+    ## TODO: finish this function with tar hog to list of prot IDS
+    #print(db_obj.member_of_hog_id(int(results[0])))
+    #hogids =[ "HOG:" + (7-len(fam_id)) * '0' + fam_id for fam_id in results ]
+    #print( db_obj.member_of_hog_id(hogids[0]) )
+
+
+    HOGS={}
+    print('compiling hogs')
+    prots = []
+    for i,result in enumerate(hogids):
+        if i %10 ==0:
+            print(i)
+        HOGS[result]=[]
+        for member in db_obj.iter_members_of_hog_id(result):
+            HOGS[result].append(member.omaid)
+            prots.append(member.omaid)
+    print('done')
+    print('running GO enrichment study')
+
+    goea_results_all = goeaobj.run_study(prots )
+    print('done')
+    with open( config_utils.datadir + outname + 'Hogs2Prots.pkl' , 'wb' ) as save:
+       save.write(pickle.dumps(HOGS,2))
+
+    goeaobj.wr_txt(config_utils.datadir+ str(outname)+"enrichment.txt", goea_results_all)
+    print('DONE!')
+    return goea_results_all, HOGS
+
+
+######################resnik semsim ###################################################
+
+def resnik_sim_hdf5(go_id1, go_id2, godag, termcounts, hdf5):
+    '''
+        Computes Resnik's similarity measure.
+    '''
+    try:
+        msca_goid = deepest_common_ancestor_hdf5([goterm2id(go_id1), goterm2id(go_id2)], godag, hdf5)
+        score = semantic.get_info_content(msca_goid, termcounts)
+    except:
+        score = -1
+    return score
+
+
+def deepest_common_ancestor_hdf5(go_ids, godag, hdf5):
+    '''
+        Gets the nearest common ancestor
+        using the above function.
+        Only returns single most specific - assumes unique exists.
+    '''
+    # Take the element at maximum depth.
+    return max(common_parent_go_ids_hdf5(go_ids, hdf5), key=lambda t: godag[t].depth)
+
+def common_parent_go_ids_hdf5(go_ids, hdf5_set):
+    '''
+        Finds the common ancestors in the GO
+        tree of the list of goids in the input.
+    '''
+    candidates = set(hdf5_set[go_ids[0]].tolist())
+    for go_id in go_ids[1:]:
+        candidates_to_add = set(hdf5_set[go_id].tolist())
+        candidates.intersection_update(candidates_to_add)
+    corrected_candidates = [id2goterm(c) for c in candidates]
+    return corrected_candidates
+
+def resnik_sim_pandas(tup, df , termcounts):
+    '''
+        Computes Resnik's similarity measure.
+    '''
+    go_id1, go_id2 = tup
+    #print(df.head())
+    if go_id1 == go_id2:
+        return semantic.get_info_content(go_id1, termcounts)
+
+    elif go_id2 in df.index and go_id1 in df.index:
+
+        ancestors = df.loc[str(go_id2)].parents
+        ancestors += df.loc[str(go_id1)].parents
+        terms = df.loc[ancestors]
+        ancestors_set = terms.parents.tolist()
+        intersection = set(ancestors_set[0]).intersection(* ancestors_set[1:])
+        common_ancestors = df.loc[list(intersection)]
+        common_ancestors = common_ancestors.sort_values('depth', ascending= False)
+        msca_goid = common_ancestors.index.tolist()[0]
+        return semantic.get_info_content(msca_goid, termcounts)
+
+    else:
+        return -1
+
+
+def get_go_terms_gaf(hog_id, pyoma_dbobj, gaf , genomes = None):
+    '''
+        iterate over hog members and get the go information from a gaf in memory
+    '''
+    fam = hashutils.hogid2fam(hog_id)
+    go_terms = { mr.omaid:gaf[mr.omaid] for mr in pyoma_dbobj.iter_members_of_hog_id(hog_id) if mr.omaid in gaf  }
+    return go_terms
+
+
+def goterm2id(go_term_to_modif):
+
+    return int(go_term_to_modif.split(':')[1])
+
+def id2goterm(go_term_to_modif):
+    return 'GO:{:07d}'.format(go_term_to_modif)

hogprof/utils/hashutils.py

@@ -0,0 +1,190 @@
+import datasketch
+import copy
+import logging
+import numpy as np
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+def generate_treeweights( mastertree, taxaIndex ,  taxfilter= None, taxmask=None ):
+    #weighing function for tax level, masking levels etc. sets all weights to 1 if they are in taxmask or filter
+    #custom weights can also be used here
+    """
+    Generate the weights of each taxonomic level to be applied during the
+    constructin of weighted minhashes
+    :param mastertree: full corrected ncbi taxonomy
+    :param taxaIndex: dict mapping taxa to columns
+    :param taxfilter: list of branches to delete
+    :param taxmask: if this is not NONE taxmask, the DB is constructed with this subtree
+    :return: weights: a vector of weights for each tax level
+    """
+    #get max of taxa index
+    taxmax = max(taxaIndex.values())+1
+    weights = np.zeros((3*taxmax,1))
+    logger.info('making tree weights with %d taxa', len(taxaIndex))
+    newtree = copy.deepcopy(mastertree)
+    if taxmask:
+        for node in newtree.traverse():
+            if taxmask:
+                if str(node.name) == str(taxmask):
+                    logger.info('Masking tree to %s', taxmask)
+                    newtree = node
+                    break
+        logger.info(
+            'Masked tree contains %d taxa',
+            len([n.name for n in newtree.traverse()]),
+        )
+    
+    if taxfilter:
+        for n in newtree.traverse():
+            if n.name in taxfilter:            
+                n.delete()
+                
+    for i in range(3):
+        for n in newtree.traverse():
+            weights[taxmax*i + taxaIndex[n.name] ] = 1
+    return weights
+
+def hash_tree(tp , taxaIndex , treeweights , wmg , lossonly = False , duplonly = False ):
+    """
+    Generate a weighted minhash and binary matrix row for a tree profile
+
+    :param tp: a pyham tree profile
+    :param taxaIndex: dict mapping taxa to columns
+    :param treeweights: a vector of weights for each tax levels
+    :param wmg: Datasketch weighted minhash generator
+    :return hog_matrix: a vector of weights for each tax level
+    :return weighted_hash: a weighted minhash of a HOG
+
+    """
+    if not tp:
+        return None, None
+
+    taxaIndex_max = max(taxaIndex.values())+1
+    hog_matrix_weighted = np.zeros((1, 3*taxaIndex_max))
+    hog_matrix_binary = np.zeros((1, 3*taxaIndex_max))
+
+    losses = [ taxaIndex[n.name]  for n in tp.traverse() if n.lost and n.name in taxaIndex  ]
+    dupl = [ taxaIndex[n.name]  for n in tp.traverse() if n.dupl  and n.name in taxaIndex  ]
+    presence = [ taxaIndex[n.name]  for n in tp.traverse() if n.nbr_genes > 0  and n.name in taxaIndex ]
+    indices = dict(zip (['presence', 'loss', 'dup'],[presence,losses,dupl] ) )
+    ### Athina comment:
+    ### change things here to do permutation test!!!!
+    for i,event in enumerate(indices):
+        if len(indices[event])>0:
+            try:
+                hogindex = np.asarray(indices[event])+i*taxaIndex_max
+                
+                hog_matrix_weighted[:,hogindex] = treeweights[hogindex , : ].ravel()
+                
+                if lossonly == True and event != 'loss':
+                    hog_matrix_weighted[:,hogindex] = 0
+                if duplonly == True and event != 'dup':
+                    hog_matrix_weighted[:,hogindex] = 0
+                if lossonly == False and duplonly == False:
+                    hog_matrix_binary[:,hogindex] = 1
+
+            except Exception:
+                logger.exception(
+                    'Error hashing event %s (indices=%s, hogindex=%s)',
+                    event,
+                    indices[event],
+                    hogindex,
+                )
+
+    input_vec = list(hog_matrix_weighted.flatten())
+
+    if wmg.dim == len(input_vec) and sum(input_vec) > 0:
+        weighted_hash = wmg.minhash(input_vec)
+        return  hog_matrix_binary , weighted_hash
+
+    else:
+        return None, None
+
+
+
+
+def hash_trees_subhogs(hog_tps , taxaIndex , treeweights , wmg , lossonly = False , duplonly = False ):
+    """
+    Generate a weighted minhash and binary matrix row for multiple tree profiles within a HOG
+
+    :param tp: a pyham tree profile
+    :param taxaIndex: dict mapping taxa to columns
+    :param treeweights: a vector of weights for each tax levels
+    :param wmg: Datasketch weighted minhash generator
+    :return hog_matrix: a vector of weights for each tax level
+    :return weighted_hash: a weighted minhash of a HOG
+
+    """
+    #print(hog_tps.keys())
+    #print(hog_tps[['tree_dicts']].to_dict())
+    if hog_tps is None or 'tree_dicts' not in hog_tps or hog_tps['tree_dicts'] is None:
+        return {}
+
+    hashes_subhogs = {}
+    for key, tp in hog_tps['tree_dicts'].items():
+        matrix, hashed = hash_tree(tp, taxaIndex, treeweights, wmg,
+                                   lossonly=lossonly, duplonly=duplonly)
+        if hashed is not None:
+            hashes_subhogs[key] = (matrix, hashed)
+    return hashes_subhogs
+
+
+def tree2str_DCA(tp , taxaIndex ):
+    """
+    Generate a string where each column is a tax level
+    each letter code corresponds to an event type
+    each row is a protein family. for use with DCA pipelines
+
+    :param tp: a pyham tree profile
+    :param taxaIndex: dict mapping taxa to columns
+    :return dcaMat: a weighted minhash of a HOG
+    """
+    #convert a tree profile to a weighted minhash
+    losses = [ taxaIndex[n.name]  for n in tp.traverse() if n.lost and n.name in taxaIndex  ]
+    dupl = [ taxaIndex[n.name]  for n in tp.traverse() if n.dupl  and n.name in taxaIndex  ]
+    presence = [ taxaIndex[n.name]  for n in tp.traverse() if n.nbr_genes > 0  and n.name in taxaIndex ]
+    Ds = list( set(dupl).intersection(set(presence)))
+    Ps=  list(set(presence).difference(set(dupl)))
+    Ls=  list(set(losses))
+    charar = np.chararray(len(taxaIndex) )
+    #set to absent
+    charar.fill( 'A')
+    charar[Ds] = 'D'
+    charar[Ls] = 'L'
+    charar[Ps] = 'P'
+    return charar
+
+def row2hash(row , taxaIndex , treeweights , wmg , lossonly = False , duplonly = False):
+    """
+    turn a dataframe row with an orthoxml file to hash and matrix row
+    :param row: lsh builder dataframe row
+    :param taxaIndex: dict mapping taxa to columnsfam
+    :param treeweights: a vector of weights for each tax levels
+    :param wmg: Datasketch weighted minhash generator
+    :return: hog_matrix: a vector of weights for each tax level
+    :return: weighted_hash: a weighted minhash of a HOG
+    """
+    #convert a dataframe row to a weighted minhash
+    fam, treemap = row.tolist()
+    #print(f'Processing row in row2hash: fam={fam}, treemap={treemap}')
+    hog_matrix,weighted_hash = hash_tree(treemap , taxaIndex , treeweights , wmg , lossonly = lossonly , duplonly = duplonly)
+    #print(f'Generated hog_matrix: {hog_matrix}, weighted_hash: {weighted_hash}')
+    
+    return  pd.Series([weighted_hash,hog_matrix], index=['hash','rows'])
+
+def fam2hash_hdf5(fam, hdf5, dataset=None, nsamples=128):
+    """
+    Read the stored hash values and return a weighted minhash
+    :param fam: hog id
+    :param hdf5: h5py object of the hashvalues
+    :param dataset: which dataset to use when constructing the hash
+    :return: minhash1: the weighted hash of your HOG
+    """
+    if dataset is None:
+        dataset = next(iter(hdf5.keys()))
+
+    hash_values = np.asarray(hdf5[dataset][fam, :].reshape(nsamples,2 )).astype('int64')
+    minhash1 = datasketch.WeightedMinHash(seed = 1, hashvalues=hash_values)
+    return minhash1

hogprof/utils/orthoxml.py

@@ -0,0 +1,83 @@
+import collections
+import xml.etree.ElementTree as ET
+from io import StringIO
+
+
+class OrthoXMLBuilder:
+    """
+    This is a function for adding subHOG IDs to the OMA orthoxml files (from Adrian).
+    Will become redundant with the next version of OMA
+    """
+
+    def __init__(self):
+        self.NS = "http://orthoXML.org/2011/"
+        ET.register_namespace('', self.NS)  # Register default namespace
+
+    def add_loft_ids(self, orthoxml: str | ET.Element) -> str:
+        """Annotate XML text or an already parsed root, modifying it in place."""
+
+        def encode_paralog_cluster_id(prefix, nr):
+            letters = []
+            while nr // 26 > 0:
+                letters.append(chr(97 + (nr % 26)))
+                nr = nr // 26 - 1
+            letters.append(chr(97 + (nr % 26)))
+            return prefix + ''.join(letters[::-1])
+
+        def next_sub_hog_id(idx):
+            dups[idx] += 1
+            return dups[idx]
+
+        def rec_annotate(node, og, idx=0):
+            if node.tag == f"{{{self.NS}}}orthologGroup":
+                taxon_id = node.get('taxonId')
+                if taxon_id is not None:
+                    node.set('id', og + "_" + taxon_id)
+                else:
+                    # If no taxonId, just set the id to the og
+                    node.set('id', og )
+                for child in list(node):
+                    rec_annotate(child, og, idx)
+            elif node.tag == f"{{{self.NS}}}paralogGroup":
+                idx += 1
+                next_og = f"{og}.{next_sub_hog_id(idx)}"
+                for i, child in enumerate(list(node)):
+                    rec_annotate(child, encode_paralog_cluster_id(next_og, i), idx)
+            elif node.tag == f"{{{self.NS}}}geneRef":
+                # Set the id to the geneRef id
+                node.set('LOFT', og)
+
+        xml = ET.fromstring(orthoxml) if isinstance(orthoxml, str) else orthoxml
+        doc = ET.ElementTree(xml)
+        for i, el in enumerate(xml.findall(".//groups/orthologGroup", {"": self.NS})):
+            og = el.get('id', f"HOG:{i+1:08d}")
+            dups = collections.defaultdict(int)
+            rec_annotate(el, og)
+        tree_str = StringIO()
+        ### addition by Athina:
+        tree_str.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+        doc.write(tree_str, encoding='unicode')
+        return tree_str.getvalue()
+
+
+def get_root_hog_id(orthoxml: str | ET.Element) -> str:
+    xml = ET.fromstring(orthoxml) if isinstance(orthoxml, str) else orthoxml
+    groups = xml.findall(
+        '{http://orthoXML.org/2011/}groups/{http://orthoXML.org/2011/}orthologGroup'
+    )
+    if len(groups) != 1:
+        raise ValueError(f'Expected one root HOG, found {len(groups)}')
+    return groups[0].attrib['id']
+
+
+def count_species(ortho_fam: str | ET.Element) -> int:
+    if isinstance(ortho_fam, ET.Element):
+        return len(ortho_fam.findall('{http://orthoXML.org/2011/}species'))
+    return ortho_fam.count('<species name=')
+
+
+def iter_species(filename):
+    root = ET.parse(filename).getroot()
+    for child in root:
+        if child.tag.rsplit("}", 1)[-1] == "species":
+            yield child