gwaslab 3.4.35__py3-none-any.whl → 3.4.37__py3-none-any.whl

gwaslab/util_ex_process_h5.py

@@ -3,31 +3,40 @@ import os
 import numpy as np
 from gwaslab.g_Log import Log
 
-def process_ref_vcf(vcf, directory=None, chr_dict=None, group_size=20000000,complevel=9,chunksize=20000000,log=Log()):
+def process_vcf_to_hfd5(vcf, 
+                    directory=None, 
+                    chr_dict=None, 
+                    group_size=20000000,
+                    complevel=9,
+                    chunksize=20000000,
+                    log=Log(),
+                    verbose=True):
+    
     #load vcf
-    log.write("Start processing VCF files:")
-    log.write(" -Reference VCF path:{}".format(vcf))
-    log.write(" -Output group size:{}".format(group_size))
-    log.write(" -Compression level:{}".format(complevel))
-    log.write(" -Loading chunksize:{}".format(chunksize))
+    log.write("Start to process VCF file to HDF5:", verbose=verbose)
+    log.write(" -Reference VCF path:{}".format(vcf), verbose=verbose)
+    log.write(" -Output group size:{}".format(group_size), verbose=verbose)
+    log.write(" -Compression level:{}".format(complevel), verbose=verbose)
+    log.write(" -Loading chunksize:{}".format(chunksize), verbose=verbose)
 
+    vcf_file_name = os.path.basename(vcf)
+    vcf_dir_path = os.path.dirname(vcf)
+    
     if directory is None:
-        directory="./"
-
+        directory = vcf_dir_path
     elif directory[-1] == "/":
         directory = directory.rstrip('/')
     
-    h5_path = "{}/rsID_CHR_POS_groups_{}.h5".format(directory,int(group_size))
-    log_path = "{}/rsID_CHR_POS_groups_{}.log".format(directory,int(group_size))
-    log.write(" -HDF5 Output path: {}".format(h5_path))
-    log.write(" -Log output path: {}".format(log_path))
+    h5_path = "{}/{}.rsID_CHR_POS_groups_{}.h5".format(directory,vcf_file_name,int(group_size))
+    log_path = "{}/{}.rsID_CHR_POS_groups_{}.log".format(directory,vcf_file_name, int(group_size))
+    log.write(" -HDF5 Output path: {}".format(h5_path), verbose=verbose)
+    log.write(" -Log output path: {}".format(log_path), verbose=verbose)
     df = pd.read_table(vcf,comment="#",usecols=[0,1,2],header=None,chunksize=chunksize)
 
-    
-    log.write(" -Processing chunk: ",end="")
+    log.write(" -Processing chunk: ",end="", verbose=verbose)
     
     for index,chunk in enumerate(df):
-        log.write(index,end=" ",show_time=False)
+        log.write(index,end=" ",show_time=False, verbose=verbose)
         chunk = chunk.rename(columns={0:"CHR",1:"POS",2:"rsn"})
         if chr_dict is not None:
             chunk["CHR"] = chunk["CHR"].map(chr_dict)
@@ -47,5 +56,5 @@ def process_ref_vcf(vcf, directory=None, chr_dict=None, group_size=20000000,comp
                                                                         dropna=True,
                                                                         format="table",
                                                                         complevel=complevel)
-    log.write("Processing finished!")
-    log.save(log_path, verbose=False)
+    log.write("Processing finished!", verbose=verbose)
+    log.save(log_path, verbose=verbose)

gwaslab/util_in_fill_data.py

@@ -4,13 +4,13 @@ import scipy.stats as ss
 from scipy import stats
 from gwaslab.g_Log import Log
 import gc
-from gwaslab.qc_fix_sumstats import sortcolumn
+#from gwaslab.qc_fix_sumstats import sortcolumn
 from gwaslab.g_version import _get_version
 from gwaslab.qc_check_datatype import check_datatype
 
 def filldata( 
     sumstats,
-    to_fill=[],
+    to_fill=None,
     df=None,
     overwrite=False,
     verbose=True,
@@ -38,7 +38,7 @@ def filldata(
         for i in skip_cols:
             to_fill.remove(i)
         if verbose: log.write("  -Skipping columns: ",skip_cols) 
-    if len(set(to_fill) & set(["OR","OR95L","OR95U","BETA","SE","P","Z","CHI2","MLOG10P"]))==0:
+    if len(set(to_fill) & set(["OR","OR_95L","OR_95U","BETA","SE","P","Z","CHISQ","MLOG10P","MAF"]))==0:
         log.write(" -No available columns to fill. Skipping.", verbose=verbose)
         log.write("Finished filling data using existing columns.", verbose=verbose)
         return sumstats
@@ -46,7 +46,7 @@ def filldata(
     fill_iteratively(sumstats,to_fill,log,only_sig,df,extreme,verbose,sig_level)
        
 # ###################################################################################
-    sumstats = sortcolumn(sumstats, verbose=verbose, log=log)
+    #sumstats = sortcolumn(sumstats, verbose=verbose, log=log)
     gc.collect()
     if verbose: log.write("Finished filling data using existing columns.")
     return sumstats
@@ -224,12 +224,12 @@ def fill_extreme_mlog10(sumstats, z):
     return sumstats
 
 ####################################################################################################################
-def fill_iteratively(sumstats,to_fill,log,only_sig,df,extreme,verbose,sig_level):
+def fill_iteratively(sumstats,raw_to_fill,log,only_sig,df,extreme,verbose,sig_level):
+    to_fill = raw_to_fill.copy()
     if verbose: log.write("  - Filling Columns iteratively...")
-    filled=[]
-    previous_count=0
+
     filled_count=0
-    for i in range(len(to_fill)):
+    for i in range(len(to_fill)+1):
     # beta to or ####################################################################################################     
         if "OR" in to_fill:
             status, filled_count = fill_or(sumstats,log,verbose=verbose,filled_count=filled_count)
@@ -269,9 +269,47 @@ def fill_iteratively(sumstats,to_fill,log,only_sig,df,extreme,verbose,sig_level)
             else:
                 status,filled_count = fill_mlog10p(sumstats,log,verbose=verbose)
             if status == 1 : to_fill.remove("MLOG10P")
-        
-        previous_count+=filled_count
-        if previous_count == filled_count:
+            
+        if filled_count == 0:
             break
          
-        
+###Base functions########################################################################################
+
+def _convert_betase_to_z(beta, se):
+    return beta/se 
+
+def _convert_betase_to_p(beta, se):
+    z = _convert_betase_to_z(beta, se)
+    p = _convert_z_to_p(z)
+    return p
+
+def _convert_betase_to_mlog10p(beta, se):
+    z = _convert_betase_to_z(beta, se)
+    mlog10p = _convert_z_to_mlog10p(z)
+    return mlog10p
+
+def _convert_p_to_chisq(p):
+    return ss.chi2.isf(p, 1)
+
+def _convert_z_to_chisq(z):
+    return (z)**2
+
+def _convert_z_to_p(z):
+    return ss.chi2.sf(z**2,1) 
+
+def _convert_z_to_mlog10p(z):
+    log_pvalue = np.log(2) + ss.norm.logsf(np.abs(z)) #two-sided
+    mlog10p = log_pvalue/np.log(10)
+    return -mlog10p 
+
+def _conver_chisq_to_p(chisq):
+    return ss.chi2.sf(chisq,1)
+
+def _convert_mlog10p_to_p(mlog10p):
+    return np.power(10, -mlog10p) 
+
+def _convert_or_to_beta(OR):
+    return np.log(OR)  
+
+def _convert_beta_to_or(beta):
+    return np.exp(beta)   

gwaslab/viz_aux_quickfix.py

@@ -159,63 +159,64 @@ def _quick_assign_i(sumstats, chrom="CHR",pos="POS"):
     return sumstats, chrom_df
 
 def _quick_assign_i_with_rank(sumstats, chrpad, use_rank=False, chrom="CHR",pos="POS",drop_chr_start=False,_posdiccul=None):
-        sumstats = sumstats.sort_values([chrom,pos])
-        if use_rank is True: 
-            sumstats["_POS_RANK"] = sumstats.groupby(chrom)[pos].rank("dense", ascending=True)
-            pos="_POS_RANK"
-        sumstats["_ID"]=range(len(sumstats))
-        sumstats=sumstats.set_index("_ID")
-
-        #create a df , groupby by chromosomes , and get the maximum position
-        if use_rank is True: 
-            posdic = sumstats.groupby(chrom)["_POS_RANK"].max()
-        else:
-            posdic = sumstats.groupby(chrom)[pos].max()
+    # align all variants on a single axis (i)
+    sumstats = sumstats.sort_values([chrom,pos])
+    if use_rank is True: 
+        sumstats["_POS_RANK"] = sumstats.groupby(chrom)[pos].rank("dense", ascending=True)
+        pos="_POS_RANK"
+    sumstats["_ID"]=range(len(sumstats))
+    sumstats=sumstats.set_index("_ID")
+
+    #create a df , groupby by chromosomes , and get the maximum position
+    if use_rank is True: 
+        posdic = sumstats.groupby(chrom)["_POS_RANK"].max()
+    else:
+        posdic = sumstats.groupby(chrom)[pos].max()
+    
+    if _posdiccul is None:
+        # convert to dictionary
+        posdiccul = dict(posdic)
         
-        if _posdiccul is None:
-            # convert to dictionary
-            posdiccul = dict(posdic)
-            
-            # fill empty chr with 0
+        # fill empty chr with 0
+        for i in range(0,sumstats[chrom].max()+1):
+            if i in posdiccul: 
+                continue
+            else: 
+                posdiccul[i]=0
+
+        # cumulative sum dictionary
+        for i in range(1,sumstats[chrom].max()+1):
+            posdiccul[i]= posdiccul[i-1] + posdiccul[i] + sumstats[pos].max()*chrpad
+    else:
+        posdiccul = _posdiccul
+
+    # convert base pair postion to x axis position using the cumulative sum dictionary
+    sumstats["_ADD"]=sumstats[chrom].apply(lambda x : posdiccul[int(x)-1])
+    
+    if drop_chr_start==True:
+            posdic_min =  sumstats.groupby(chrom)[pos].min()
+            posdiccul_min= dict(posdic_min)
             for i in range(0,sumstats[chrom].max()+1):
-                if i in posdiccul: 
+                if i in posdiccul_min: 
                     continue
                 else: 
-                    posdiccul[i]=0
-    
-            # cumulative sum dictionary
+                    posdiccul_min[i]=0
             for i in range(1,sumstats[chrom].max()+1):
-                posdiccul[i]= posdiccul[i-1] + posdiccul[i] + sumstats[pos].max()*chrpad
-        else:
-            posdiccul = _posdiccul
-
-        # convert base pair postion to x axis position using the cumulative sum dictionary
-        sumstats["_ADD"]=sumstats[chrom].apply(lambda x : posdiccul[int(x)-1])
-        
-        if drop_chr_start==True:
-                posdic_min =  sumstats.groupby(chrom)[pos].min()
-                posdiccul_min= dict(posdic_min)
-                for i in range(0,sumstats[chrom].max()+1):
-                    if i in posdiccul_min: 
-                        continue
-                    else: 
-                        posdiccul_min[i]=0
-                for i in range(1,sumstats[chrom].max()+1):
-                    posdiccul_min[i]= posdiccul_min[i-1] + posdiccul_min[i]
-                sumstats["_ADD"]=sumstats["_ADD"] - sumstats[chrom].apply(lambda x : posdiccul_min[int(x)])
-            
-        if use_rank is True: 
-            sumstats["i"]=sumstats["_POS_RANK"]+sumstats["_ADD"]
-        else:
-            sumstats["i"]=sumstats[pos]+sumstats["_ADD"]
+                posdiccul_min[i]= posdiccul_min[i-1] + posdiccul_min[i]
+            sumstats["_ADD"]=sumstats["_ADD"] - sumstats[chrom].apply(lambda x : posdiccul_min[int(x)])
         
+    if use_rank is True: 
+        sumstats["i"]=sumstats["_POS_RANK"]+sumstats["_ADD"]
+    else:
+        sumstats["i"]=sumstats[pos]+sumstats["_ADD"]
+    
 
-        #for plot, get the chr text tick position      
-        chrom_df=sumstats.groupby(chrom)['i'].agg(lambda x: (x.min()+x.max())/2)
-        #sumstats["i"] = sumstats["i"]+((sumstats[chrom].map(dict(chrom_df)).astype("int")))*0.02
-        #sumstats["i"] = sumstats["i"].astype("Int64")
-        sumstats["i"] = np.floor(pd.to_numeric(sumstats["i"], errors='coerce')).astype('Int64')
-        return sumstats, chrom_df
+    #for plot, get the chr text tick position      
+    chrom_df=sumstats.groupby(chrom)['i'].agg(lambda x: (x.min()+x.max())/2)
+    #sumstats["i"] = sumstats["i"]+((sumstats[chrom].map(dict(chrom_df)).astype("int")))*0.02
+    #sumstats["i"] = sumstats["i"].astype("Int64")
+    sumstats["i"] = np.floor(pd.to_numeric(sumstats["i"], errors='coerce')).astype('Int64')
+    return sumstats, chrom_df
 
 def _quick_assign_marker_relative_size(series, sig_level = 5e-8, suggestive_sig_level=5e-6, lower_level=5e-4):
     size_series = series.copy()
@@ -286,9 +287,9 @@ def _cut(series, mode,cutfactor,cut,skip, ylabels, cut_log, verbose,lines_to_plo
     maxy = series.max()
     series = series.copy()
     if "b" not in mode:
-        if verbose: log.write(" -Maximum -log10(P) values is "+str(maxy) +" .")
+        if verbose: log.write(" -Maximum -log10(P) value is "+str(maxy) +" .")
     elif "b" in mode:
-        if verbose: log.write(" -Maximum DENSITY values is "+str(maxy) +" .")
+        if verbose: log.write(" -Maximum DENSITY value is "+str(maxy) +" .")
     
     maxticker=int(np.round(series.max(skipna=True)))
     

gwaslab/viz_plot_compare_effect.py

@@ -4,7 +4,7 @@ import matplotlib.pyplot as plt
 import scipy.stats as ss
 import seaborn as sns
 import gc
-from statsmodels.stats.multitest import fdrcorrection
+import scipy.stats as ss
 from matplotlib.patches import Rectangle
 from adjustText import adjust_text
 from gwaslab.viz_aux_save_figure import save_figure
@@ -36,6 +36,7 @@ def compare_effect(path1,
                    wc_correction=False, 
                    null_beta=0,
                    is_q=False,
+                   is_q_mc = False,
                    include_all=True,
                    q_level=0.05,
                    sig_level=5e-8,
@@ -485,8 +486,10 @@ def compare_effect(path1,
             if verbose: log.write(" -No variants with EA not matching...")
     if fdr==True:
         if verbose: log.write(" -Using FDR...")
-        sig_list_merged["P_1"] = fdrcorrection(sig_list_merged["P_1"])[1]
-        sig_list_merged["P_2"] = fdrcorrection(sig_list_merged["P_2"])[1]
+        #sig_list_merged["P_1"] = fdrcorrection(sig_list_merged["P_1"])[1]
+        #sig_list_merged["P_2"] = fdrcorrection(sig_list_merged["P_2"])[1]
+        sig_list_merged["P_1"] =ss.false_discovery_control(sig_list_merged["P_1"])
+        sig_list_merged["P_2"] =ss.false_discovery_control(sig_list_merged["P_2"])
 
     ####################################################################################################################################
     ## winner's curse correction using aligned beta
@@ -528,9 +531,10 @@ def compare_effect(path1,
     if (is_q is True):
         if verbose: log.write(" -Calculating Cochran's Q statistics and peform chisq test...")
         if mode=="beta" or mode=="BETA" or mode=="Beta":
-            sig_list_merged = test_q(sig_list_merged,"EFFECT_1","SE_1","EFFECT_2_aligned","SE_2",q_level=q_level)
+            sig_list_merged = test_q(sig_list_merged,"EFFECT_1","SE_1","EFFECT_2_aligned","SE_2",q_level=q_level,is_q_mc=is_q_mc, log=log, verbose=verbose)
         else:
-            sig_list_merged = test_q(sig_list_merged,"BETA_1","SE_1","BETA_2_aligned","SE_2",q_level=q_level)
+            sig_list_merged = test_q(sig_list_merged,"BETA_1","SE_1","BETA_2_aligned","SE_2",q_level=q_level,is_q_mc=is_q_mc, log=log, verbose=verbose)
+
     ######################### save ###############################################################
     ## save the merged data
     save_path = label[0]+"_"+label[1]+"_beta_sig_list_merged.tsv"
@@ -804,8 +808,15 @@ def compare_effect(path1,
     if legend_mode == "full" and is_q==True :
         title_proxy = Rectangle((0,0), 0, 0, color='w',label=legend_title)
         title_proxy2 = Rectangle((0,0), 0, 0, color='w',label=legend_title2)
-        het_label_sig = r"$P_{het} < $" + "${}$".format(q_level)
-        het_label_sig2 = r"$P_{het} > $" + "${}$".format(q_level)
+        if is_q_mc=="fdr":
+            het_label_sig = r"$FDR_{het} < $" + "${}$".format(q_level)
+            het_label_sig2 = r"$FDR_{het} > $" + "${}$".format(q_level)
+        elif is_q_mc=="bon":
+            het_label_sig = r"$P_{het,bon} < $" + "${}$".format(q_level)
+            het_label_sig2 = r"$P_{het,bon} > $" + "${}$".format(q_level)
+        else:
+            het_label_sig = r"$P_{het} < $" + "${}$".format(q_level)
+            het_label_sig2 = r"$P_{het} > $" + "${}$".format(q_level)
         het_sig = Rectangle((0,0), 0, 0, facecolor='#cccccc',edgecolor="black", linewidth=1, label=het_label_sig)
         het_nonsig = Rectangle((0,0), 0, 0, facecolor='#cccccc',edgecolor="white",linewidth=1, label=het_label_sig2)
         
@@ -874,7 +885,7 @@ def reorderLegend(ax=None, order=None, add=None):
     new_handles = [info[l] for l in order]
     return new_handles, order
 
-def test_q(df,beta1,se1,beta2,se2,q_level=0.05):
+def test_q(df,beta1,se1,beta2,se2,q_level=0.05,is_q_mc=False, log=Log(), verbose=False):
     w1="Weight_1"
     w2="Weight_2"
     beta="BETA_FE"
@@ -889,6 +900,14 @@ def test_q(df,beta1,se1,beta2,se2,q_level=0.05):
     df[q] = df[w1]*(df[beta1]-df[beta])**2 + df[w2]*(df[beta2]-df[beta])**2
     df[pq] = ss.chi2.sf(df[q], 1)
     df["Edge_color"]="white"
+
+    if is_q_mc=="fdr":
+        if verbose: log.write(" -FDR correction applied...")
+        df[pq] = ss.false_discovery_control(df[pq])
+    elif is_q_mc=="bon":
+        if verbose: log.write(" -Bonferroni correction applied...")
+        df[pq] = df[pq] * len(df[pq])
+
     df.loc[df[pq]<q_level,"Edge_color"]="black"
     df.drop(columns=["Weight_1","Weight_2","BETA_FE"],inplace=True)
     # Huedo-Medina, T. B., Sánchez-Meca, J., Marín-Martínez, F., & Botella, J. (2006). Assessing heterogeneity in meta-analysis: Q statistic or I² index?. Psychological methods, 11(2), 193.

gwaslab/viz_plot_forestplot.py

@@ -2,7 +2,7 @@ import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt 
 from scipy import stats, optimize
-from statsmodels.stats.meta_analysis import combine_effects
+#from statsmodels.stats.meta_analysis import combine_effects
 from matplotlib.patches import Polygon
 from matplotlib.collections import PatchCollection
 # plot_forest