Oncodrive3D 1.0.4__py3-none-any.whl

scripts/run/communities.py

@@ -0,0 +1,53 @@
+"""
+Contains function to initialize a network for communities detection.
+"""
+
+import networkx as nx
+import numpy as np
+
+
+def get_network(nodes, mut_count_v, cmap):
+    """
+    Generate a network with significative pos as nodes 
+    and ratio of shared mutation (Jaccard score) as edges. 
+    """
+    
+    G = nx.Graph()
+    G.add_nodes_from(nodes)
+
+    # Iterate through each hit
+    for i, ipos in enumerate(nodes):
+        for j, jpos in enumerate(nodes):
+            if i > j:
+                
+                # Add an edge if they are in contact
+                ix, jx = ipos-1, jpos-1
+                if cmap[ix, jx] == 1:
+
+                    # Get the common res and their mut
+                    neigh_vec_i, neigh_vec_j = cmap[ix], cmap[jx]
+                    common_neigh = neigh_vec_j * neigh_vec_i
+                    num_mut = np.dot(common_neigh, mut_count_v)
+
+                    # Get the sum of the union of the mut
+                    all_neigh = (neigh_vec_i + neigh_vec_j != 0).astype(int)
+                    union_num_mut = np.dot(all_neigh, mut_count_v)
+
+                    # Compute the Jaccard score or avg ratio between ij shared mut    
+                    jaccard = np.round(num_mut/union_num_mut, 3)
+                    G.add_edge(ipos, jpos, weight = jaccard)
+    return G
+
+
+def get_community_index_nx(pos_hits, communities):
+    
+    """
+    Parse the labels returned by communities detection algorithms from NetworkX.
+    """
+    
+    communities_mapper = {}
+    for ic, c in enumerate(communities):
+        for p in c:
+            communities_mapper[p] = ic
+
+    return pos_hits.map(communities_mapper).values

scripts/run/miss_mut_prob.py

@@ -0,0 +1,206 @@
+"""
+Contain functions to compute the per-residues probability of missense 
+mutation of any protein given the mutation profile of the cohort.
+"""
+
+
+from itertools import product
+
+import daiquiri
+import numpy as np
+
+from scripts.run.mutability import Mutabilities
+from scripts import __logger_name__
+
+logger = daiquiri.getLogger(__logger_name__ + ".run.miss_mut_prob")
+
+
+def get_unif_gene_miss_prob(size):
+    """
+    Get a uniformly distributed gene missense mutation 
+    probability vector.
+    """
+    
+    vector = np.ones(size)
+    vector[0] = 0
+    
+    return vector / sum(vector)
+
+
+def mut_rate_vec_to_dict(mut_rate):
+    """
+    Convert the vector of mut mut_rate of 96 channels to a dictionary of 192 
+    items: the keys are mutations in trinucleotide context (e.g., "ACA>A") 
+    and values are the corresponding mut rate (frequency of mut normalized 
+    for the nucleotide content).
+    """
+    
+    cb  = dict(zip('ACGT', 'TGCA'))
+    mut_rate_dict = {}
+    i = 0
+    for ref in ['C', 'T']:
+        for alt in cb.keys():
+            if ref == alt:
+                continue
+            else:
+                for p in product(cb.keys(), repeat=2):
+                    mut = f"{p[0]}{ref}{p[1]}>{alt}"
+                    cmut = f"{cb[p[1]]}{cb[ref]}{cb[p[0]]}>{cb[alt]}"
+                    mut_rate_dict[mut] = mut_rate[i]
+                    mut_rate_dict[cmut] = mut_rate[i]
+                    i +=1
+                    
+    return mut_rate_dict
+
+
+def get_codons(dna_seq):
+    """
+    Get the list of codons from a DNA sequence.
+    """
+    
+    return [dna_seq[i:i+3] for i in [n*3 for n in range(int(len(dna_seq) / 3))]]
+
+
+def translate_dna_to_prot(dna_seq, gencode):
+    """
+    Translate a DNA sequence into amino acid sequence.
+    """
+    
+    return "".join([gencode[codon] for codon in get_codons(dna_seq)])
+
+
+def codons_trinucleotide_context(lst_contexts):
+    
+    return list(zip(lst_contexts[::3], lst_contexts[1::3], lst_contexts[2::3]))
+
+
+# TODO: doc function
+
+def get_miss_mut_prob(dna_seq, 
+                      dna_tricontext, 
+                      mut_rate_dict, 
+                      mutability=False, 
+                      get_probability=True, 
+                      mut_start_codon=False):
+    """
+    Generate a list including the probabilities that the 
+    codons can mutate resulting into a missense mutations.
+    
+    Arguments
+    ---------
+    dna_seq: str
+        Sequence of DNA
+    mut_rate_dict: dict
+        Mutation rate probability as values and the 96 possible
+        trinucleotide contexts as keys
+    gencode: dict
+        Nucleotide as values and codons as keys
+        
+    Returns
+    -------
+    missense_prob_vec: list
+        List of probabilities (one for each codon or prot res) 
+        of a missense mutation  
+    """
+
+    # Initialize
+    gencode = {
+        'ATA':'I', 'ATC':'I', 'ATT':'I', 'ATG':'M',
+        'ACA':'T', 'ACC':'T', 'ACG':'T', 'ACT':'T',
+        'AAC':'N', 'AAT':'N', 'AAA':'K', 'AAG':'K',
+        'AGC':'S', 'AGT':'S', 'AGA':'R', 'AGG':'R',
+        'CTA':'L', 'CTC':'L', 'CTG':'L', 'CTT':'L',
+        'CCA':'P', 'CCC':'P', 'CCG':'P', 'CCT':'P',
+        'CAC':'H', 'CAT':'H', 'CAA':'Q', 'CAG':'Q',
+        'CGA':'R', 'CGC':'R', 'CGG':'R', 'CGT':'R',
+        'GTA':'V', 'GTC':'V', 'GTG':'V', 'GTT':'V',
+        'GCA':'A', 'GCC':'A', 'GCG':'A', 'GCT':'A',
+        'GAC':'D', 'GAT':'D', 'GAA':'E', 'GAG':'E',
+        'GGA':'G', 'GGC':'G', 'GGG':'G', 'GGT':'G',
+        'TCA':'S', 'TCC':'S', 'TCG':'S', 'TCT':'S',
+        'TTC':'F', 'TTT':'F', 'TTA':'L', 'TTG':'L',
+        'TAC':'Y', 'TAT':'Y', 'TAA':'_', 'TAG':'_',
+        'TGC':'C', 'TGT':'C', 'TGA':'_', 'TGG':'W'}
+
+    # Get all codons of the seq
+    #logger.debug("Getting codons of seq..")
+    codons = get_codons(dna_seq)
+    missense_prob_vec = []
+    
+    # Get the trinucleotide context as list of tuples of 3 elements corresponding to each codon   
+    #logger.debug("Getting tri context seq..")                               
+    tricontext = codons_trinucleotide_context(dna_tricontext.split(","))
+    
+    # Iterate through codons and get prob of missense based on context
+    for c in range(len(codons)):
+        missense_prob = 0
+        codon = codons[c]
+        aa = gencode[codon]         
+        trinucl0, trinucl1, trinucl2  = tricontext[c]
+
+        # Iterate through the possible contexts of a missense mut
+        for i, trinucl in enumerate([trinucl0, trinucl1, trinucl2]):
+            ref = trinucl[1]
+            aa = gencode[codon]
+
+            # Iterate through the possible alt 
+            for alt in "ACGT":
+                if alt != ref:         
+                    alt_codon = [n for n in codon]
+                    alt_codon[i] = alt
+                    alt_codon = "".join(alt_codon)
+                    alt_aa = gencode[alt_codon]  
+                    # If there is a missense mut, get prob from context and sum it
+                    if alt_aa != aa and alt_aa != "_":
+                        if not mutability:
+                            mut = f"{trinucl}>{alt}"    # query using only the trinucleotide change
+                            mut_prob = mut_rate_dict[mut] if mut in mut_rate_dict else 0                                                                 
+                            missense_prob += mut_prob
+
+                        else:
+                            # TODO this has not been tested
+                            cdna_pos = (c * 3) + i  # compute the cDNA position of the residue
+                            if cdna_pos in mut_rate_dict:
+                                missense_prob += mut_rate_dict[cdna_pos].get(alt, 0)
+                            else:
+                                missense_prob += 0
+
+        missense_prob_vec.append(missense_prob)
+
+    # Assign 0 prob to the first residue
+    if mut_start_codon == False:
+        missense_prob_vec[0] = 0
+        
+    # Convert into probabilities
+    if get_probability:
+        missense_prob_vec = np.array(missense_prob_vec) / sum(missense_prob_vec)
+    
+    return list(missense_prob_vec)
+
+
+def get_miss_mut_prob_dict(mut_rate_dict, seq_df, mutability=False, mutability_config=None):
+    """
+    Given a dictionary of mut rate in 96 contexts (mut profile) and a 
+    dataframe including Uniprot ID, HUGO symbol and DNA sequences, 
+    get a dictionary with UniprotID-Fragment as keys and corresponding 
+    vectors of missense mutation probabilities as values.
+    """
+
+    miss_prob_dict = {}
+
+    if mutability:
+        # TODO if the execution time of this step is too long we could
+        # parallelize all these loops so that each gene is done in parallel
+
+        # Process any Protein/fragment in the sequence df
+        for _, row in seq_df.iterrows():
+            # Mutabilities
+            mutability_dict = Mutabilities(row.Uniprot_ID, row.Chr, row.Exons_coord, len(row.Seq_dna), row.Reverse_strand, mutability_config).mutabilities_by_pos
+            miss_prob_dict[f"{row.Uniprot_ID}-F{row.F}"] = get_miss_mut_prob(row.Seq_dna, row.Tri_context, mutability_dict, mutability=True)
+
+    else:
+        # Process any Protein/fragment in the sequence df
+        for _, row in seq_df.iterrows():
+            miss_prob_dict[f"{row.Uniprot_ID}-F{row.F}"] = get_miss_mut_prob(row.Seq_dna, row.Tri_context, mut_rate_dict)
+    
+    return miss_prob_dict

scripts/run/mutability.py

@@ -0,0 +1,289 @@
+"""
+This module contains the methods associated with the
+mutabilities that are assigned to the mutations.
+
+The mutabilities are read from a file.
+The file must be compressed using bgzip, and then indexed using tabix.
+$ bgzip ..../all_samples.mutability_per_site.tsv
+$ tabix -b 2 -e 2 ..../all_samples.mutability_per_site.tsv.gz
+"""
+
+import logging
+from collections import defaultdict, namedtuple
+from typing import List
+
+import tabix
+
+# TODO uncomment these lines to make sure that the logger and everything is working
+
+from scripts import __logger_name__
+# from oncodrivefml.reference import get_ref_triplet
+
+logger = logging.getLogger(__logger_name__ + ".run.mutability")
+
+transcribe = {"A":"T", "C":"G", "G":"C", "T":"A"}
+
+MutabilityValue = namedtuple('MutabilityValue', ['ref', 'alt', 'value'])
+"""
+Tuple that contains the reference, the alteration, the mutability value
+
+Parameters:
+    ref (str): reference base
+    alt (str): altered base
+    value (float): mutability value of that substitution
+"""
+
+mutabilities_reader = None
+
+
+class ReaderError(Exception):
+
+    def __init__(self, msg):
+        self.message = msg
+
+
+class ReaderGetError(ReaderError):
+    def __init__(self, chr, start, end):
+        self.message = 'Error reading chr: {} start: {} end: {}'.format(chr, start, end)
+
+class MutabilityTabixReader:
+
+    def __init__(self, conf):
+        self.file = conf['file']
+        self.conf_chr_prefix = conf['chr_prefix']
+        self.ref_pos = conf['ref']
+        self.alt_pos = conf['alt']
+        self.pos_pos = conf['pos']
+        self.mutability_pos = conf['mutab']
+        self.element_pos = None
+
+    def __enter__(self):
+        self.tb = tabix.open(self.file)
+        self.index_errors = 0
+        self.elements_errors = 0
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.index_errors > 0 or self.elements_errors > 0:
+            raise ReaderError('{} index errors and {} discrepancies between the expected and retrieved element'.format(self.index_errors, self.elements_errors))
+        return True
+
+    def _read_row(self, row):
+        mutability = float(row[self.mutability_pos])
+        ref = None if self.ref_pos is None else row[self.ref_pos]
+        alt = None if self.alt_pos is None else row[self.alt_pos]
+        pos = None if self.pos_pos is None else int(row[self.pos_pos])
+        element = None if self.element_pos is None else row[self.element_pos]
+        return (mutability, ref, alt, pos), element
+
+    def get(self, chromosome, start, stop, element=None):
+        try:
+            for row in self.tb.query("{}{}".format(self.conf_chr_prefix, chromosome), start, stop):
+                try:
+                    r = self._read_row(row)
+                except IndexError:
+                    self.index_errors += 1
+                    continue
+                else:
+                    if self.element_pos is not None and element is not None and r[1] != element:
+                        self.elements_errors += 1
+                        continue
+                    yield r[0]
+        except tabix.TabixError:
+            raise ReaderGetError(chromosome, start, stop)
+
+
+def init_mutabilities_module(conf):
+    global mutabilities_reader
+    mutabilities_reader = MutabilityTabixReader(conf)
+
+
+class Mutabilities(object):
+    """
+
+    Args:
+        element (str): element ID
+        segments (list): list of the segments associated to the element
+        config (dict): configuration
+
+    Attributes:
+        mutabilities_by_pos (dict): for each positions get all possible changes
+
+            .. code-block:: python
+
+                    { position:
+                        [
+                            MutabilityValue(
+                                ref,
+                                alt_1,
+                                value
+                            ),
+                            MutabilityValue(
+                                ref,
+                                alt_2,
+                                value
+                            ),
+                            MutabilityValue(
+                                ref,
+                                alt_3,
+                                value
+                            )
+                        ]
+                    }
+    """
+
+    def __init__(self, element: str, chromosome:str, segments: list, gene_len: int, gene_reverse_strand: bool, config: dict):
+
+        
+        self.element = element
+        self.chromosome = chromosome
+        self.segments = segments
+        self.gene_length = gene_len
+        self.reverse = True if float(gene_reverse_strand) == 1.0 else False
+
+        # mutability configuration
+        self.conf_file = config['file']
+        self.conf_chr = config['chr']
+        self.conf_chr_prefix = config['chr_prefix']
+        self.conf_ref = config['ref']
+        self.conf_alt = config['alt']
+        self.conf_pos = config['pos']
+        self.conf_element = config['element'] if 'element' in config.keys() else None
+        self.conf_extra = config['extra'] if 'extra' in config.keys() else None
+
+        # mutabilities to load
+        self.mutabilities_by_pos = defaultdict(dict)
+
+
+        # Initialize background mutabilities
+        self._load_mutabilities()
+
+    def get_mutability_by_position(self, position: int):
+        """
+        Get all MutabilityValue objects that are associated with that position
+
+        Args:
+            position (int): position
+
+        Returns:
+            :obj:`list` of :obj:`MutabilityValue`: list of all MutabilityValue related to that position
+
+        """
+        return self.mutabilities_by_pos.get(position, [])
+
+    def get_all_positions(self) -> List[int]:
+        """
+        Get all positions in the element
+
+        Returns:
+            :obj:`list` of :obj:`int`: list of positions
+
+        """
+        return self.mutabilities_by_pos.keys()
+
+    def _load_mutabilities(self):
+        """
+        For each position get all possible substitutions and for each
+        obtains the assigned mutability
+
+        Returns:
+            dict: for each positions get a list of MutabilityValue
+            (see :attr:`mutabilities_by_pos`)
+        """
+        cdna_pos = 0 
+        starting_cdna_pos = 0
+        start = 0 if not self.reverse else 1
+        end = 1 if not self.reverse else 0
+        update_pos = 1 if not self.reverse else -1
+        try:
+            with mutabilities_reader as reader:
+                for region in self.segments:
+                    # each region corresponds to an exon
+                    try:
+                        segment_len = region[end] - region[start] + 1
+                        cdna_pos = starting_cdna_pos if not self.reverse else starting_cdna_pos + segment_len
+                        starting_cdna_pos = int(cdna_pos)
+                        prev_pos = region[start] - 1
+                        # print(self.chromosome, region[start], region[end], self.element, segment_len, cdna_pos, prev_pos, starting_cdna_pos, update_pos)
+                        for row in reader.get(self.chromosome, region[start], region[end], self.element):
+                            # every row is a site
+
+                            mutability, ref, alt, pos = row
+                            # print(mutability, ref, alt, pos, sep = "\t")
+
+                            # TODO decide if we want to do this check or not,
+                            # I would say it is fine as it is now without the check
+
+                            # ref_triplet = get_ref_triplet(region['CHROMOSOME'], pos - 1)
+                            # ref = ref_triplet[1] if ref is None else ref
+                            # if ref_triplet[1] != ref:
+                            #     logger.warning("Background mismatch at position %d at '%s'", pos, self.element)
+
+
+                            # if the current position is different from the previous
+                            # update the cdna position accordingly to the strand
+                            # and also update the value of prev_pos
+                            if pos != prev_pos:
+                                cdna_pos += update_pos
+                                # print("changing position", pos, prev_pos, cdna_pos)
+                                
+                                # if it is not the first position of an exon and
+                                # the current position is not the one right after/before the previous position,
+                                # it means that the mutability for a given position(s) is missing
+                                # then
+                                # add a dictionary with all the alts and probability equals to 0,
+                                # if there are more mutabilities of the consecutive positions missing, keep adding 0s
+
+                                if pos != region[start]:
+                                    expected_previous_pos = pos - 1
+                                    # print(pos, prev_pos, expected_previous_pos)
+                                    while prev_pos != expected_previous_pos:
+                                        # print(pos, region[start], region[end], prev_pos, expected_previous_pos, cdna_pos)
+                                        for altt in "ACGT":
+                                            self.mutabilities_by_pos[cdna_pos][altt] = 0
+                                        cdna_pos += update_pos
+                                        expected_previous_pos -= 1
+
+                                prev_pos = pos
+
+                            # since at protein level we are looking at the nucleotide 
+                            # changes of the translated codons we store them as they will be queried later
+                            if self.reverse:
+                                alt = transcribe[alt]
+
+                            # add the mutability
+                            self.mutabilities_by_pos[cdna_pos][alt] = mutability
+                            # print(mutability, ref, alt, pos, cdna_pos, sep = "\t")
+
+                        # segment_starting_pos = cdna_pos if not self.reverse else cdna_pos + segment_len
+                        # print(self.chromosome, region[start], region[end], self.element, segment_len, cdna_pos, prev_pos, starting_cdna_pos, update_pos)
+
+                        ##
+                        #   IMPORTANT: filling step
+                        ##
+                        # check that all the positions at the end have been filled
+                        # otherwise add 0s to them
+                        if not self.reverse:
+                            while cdna_pos < (starting_cdna_pos + segment_len):
+                                for altt in "ACGT":
+                                    self.mutabilities_by_pos[cdna_pos][altt] = 0
+                                cdna_pos += 1
+                        else:
+                            while cdna_pos > (starting_cdna_pos - segment_len):
+                                for altt in "ACGT":
+                                    self.mutabilities_by_pos[cdna_pos][altt] = 0
+                                cdna_pos -= 1
+
+                        # print(self.chromosome, region[start], region[end], self.element, segment_len, cdna_pos, prev_pos, starting_cdna_pos, update_pos)
+                        # print("\n")
+
+                        # this is to get the cdna position pointer
+                        # back to the biggest cdna position annotated so far
+                        starting_cdna_pos = cdna_pos if not self.reverse else cdna_pos + segment_len
+
+
+                    except ReaderError as e:
+                        logger.warning(e.message)
+                        continue
+        except ReaderError as e:
+            logger.warning("Reader error: %s. Regions being analysed %s", e.message, self.segments)

scripts/run/pvalues.py

@@ -0,0 +1,91 @@
+"""
+Contains function to process the experimental p-values.
+"""
+
+import pandas as pd
+import numpy as np
+from statsmodels.stats.multitest import multipletests
+
+
+def fdr(p_vals, alpha=0.05):
+    """
+    Compute false discovery rate using Benjamini-Hochberg method.
+    """
+
+    return multipletests(p_vals, alpha=alpha, method='fdr_bh', is_sorted=True)[1]   
+
+
+def get_top_vol_info(gene_result_pos):
+    """
+    Get score, mutations count and other info of the top volume 
+    (most significant one) across the mutated ones in each gene. 
+    """
+    
+    if len(gene_result_pos) > 1:
+        lowest_pval = gene_result_pos[gene_result_pos['pval'] == gene_result_pos['pval'].min()]
+        if len(lowest_pval) > 1:
+            top_vol = lowest_pval[lowest_pval['Score_obs_sim'] == lowest_pval['Score_obs_sim'].max()].iloc[0]
+        else:
+            top_vol = lowest_pval.iloc[0]
+    else:
+        top_vol = gene_result_pos.iloc[0]
+    
+    pos_top_vol = top_vol.Pos
+    mut_in_top_vol = np.round(top_vol.Mut_in_vol, 2)
+    mut_in_top_cl_vol = np.round(top_vol.Mut_in_cl_vol, 2)
+    score_obs_sim_top_vol = top_vol.Score_obs_sim
+    pae_top_vol = np.round(top_vol.PAE_vol, 2)
+    plddt_top_vol = top_vol.pLDDT_vol
+    pLDDT_top_cl_vol = np.round(top_vol.pLDDT_cl_vol, 2)
+    
+    return pos_top_vol, mut_in_top_vol, mut_in_top_cl_vol, score_obs_sim_top_vol, pae_top_vol, plddt_top_vol, pLDDT_top_cl_vol
+
+
+def get_final_gene_result(result_pos, result_gene, alpha_gene=0.05, sample_info=False):
+    """
+    Output the final dataframe including gene global pval, qval,
+    significant positions, clumps, processing status, etc.
+    """
+
+    pos_hits = result_pos[result_pos["C"] == 1]
+
+    if len(pos_hits) > 0:
+        # Get significant positions and communities for each gene
+        clumps = pos_hits.groupby("Gene").apply(lambda x: (x["Pos"].values)).reset_index().rename(columns={0 : "C_pos"})
+        clumps["C_label"] = pos_hits.groupby("Gene").apply(lambda x: x["Clump"].values).reset_index(drop=True)
+        # Annotate each gene with significant hits
+        result_gene = clumps.merge(result_gene, on="Gene", how="outer")
+    else:
+        result_gene["C_pos"] = np.nan
+        result_gene["C_label"] = np.nan
+    
+    # Gene pval
+    gene_pvals = result_pos.groupby("Gene").apply(lambda x: min(x["pval"].values)).reset_index().rename(columns={0 : "pval"})
+ 
+    # Top volume info
+    gene_top_vol_info = result_pos.groupby("Gene").apply(lambda x: get_top_vol_info(x)).apply(pd.Series)
+    gene_top_vol_info.columns = ["Pos_top_vol",
+                                 "Mut_in_top_vol", 
+                                 "Mut_in_top_cl_vol", 
+                                 "Score_obs_sim_top_vol", 
+                                 "PAE_top_vol", 
+                                 "pLDDT_top_vol", 
+                                 "pLDDT_top_cl_vol"]
+    gene_top_vol_info = gene_top_vol_info.reset_index()
+    gene_pvals = gene_pvals.merge(gene_top_vol_info, on="Gene")
+
+    # Sort positions and get qval
+    gene_pvals = gene_pvals.sort_values(["pval", "Score_obs_sim_top_vol"], ascending=[True, False]).reset_index(drop=True)
+    not_processed_genes_count = sum(~result_gene.Status.str.contains("Processed", na=False))
+    gene_pvals["qval"] = fdr(np.concatenate((gene_pvals["pval"], np.repeat(1, not_processed_genes_count))))[:len(gene_pvals)]
+                     
+    # Combine gene-level clustering result, add label, sort genes, add fragment info
+    result_gene = gene_pvals.merge(result_gene, on="Gene", how="outer")
+    result_gene["C_gene"] = result_gene.apply(lambda x: 1 if x.qval < alpha_gene else 0, axis=1)
+    # result_gene = result_gene.sort_values(["pval", "Score_obs_sim_top_vol"], ascending=[True, False])
+    
+    # Convert C_pos and C_label to str
+    result_gene["C_pos"] = result_gene["C_pos"].apply(lambda x: str(x) if isinstance(x, list) else x)
+    result_gene["C_label"] = result_gene["C_label"].apply(lambda x: str(x) if isinstance(x, list) else x)
+
+    return result_gene