gwaslab 3.4.38__py3-none-any.whl → 3.4.39__py3-none-any.whl

gwaslab/util_in_fill_data.py

@@ -24,31 +24,31 @@ def filldata(
     if type(to_fill) is str:
         to_fill = [to_fill]
     sumstats = insumstats.copy()
-    if verbose: log.write("Start filling data using existing columns...{}".format(_get_version()))
+    log.write("Start filling data using existing columns...{}".format(_get_version()), verbose=verbose)
     
     check_datatype(sumstats,verbose=verbose,log=log)
     
 # check dupication ##############################################################################################
     skip_cols=[]
-    if verbose: log.write(" -Overwrite mode: ",overwrite)
+    log.write(" -Overwrite mode: ",overwrite, verbose=verbose)
     if overwrite is False:
         for i in to_fill:
             if i in sumstats.columns:
                 skip_cols.append(i)
         for i in skip_cols:
             to_fill.remove(i)
-        if verbose: log.write("  -Skipping columns: ",skip_cols) 
+        log.write("  -Skipping columns: ",skip_cols, verbose=verbose) 
     if len(set(to_fill) & set(["OR","OR_95L","OR_95U","BETA","SE","P","Z","CHISQ","MLOG10P","MAF"]))==0:
         log.write(" -No available columns to fill. Skipping.", verbose=verbose)
         log.write("Finished filling data using existing columns.", verbose=verbose)
         return sumstats
-    if verbose: log.write(" -Filling columns: ",to_fill)
+    log.write(" -Filling columns: ",to_fill, verbose=verbose)
     fill_iteratively(sumstats,to_fill,log,only_sig,df,extreme,verbose,sig_level)
        
 # ###################################################################################
     #sumstats = sortcolumn(sumstats, verbose=verbose, log=log)
     gc.collect()
-    if verbose: log.write("Finished filling data using existing columns.")
+    log.write("Finished filling data using existing columns.", verbose=verbose)
     return sumstats
     
 ##########################################################################################################################    
@@ -56,20 +56,20 @@ def filldata(
 def fill_p(sumstats,log,df=None,only_sig=False,sig_level=5e-8,overwrite=False,verbose=True,filled_count=0):
         # MLOG10P -> P
     if "MLOG10P" in sumstats.columns:    
-        if verbose: log.write("  - Filling P value using MLOG10P column...")
+        log.write("  - Filling P value using MLOG10P column...", verbose=verbose)
         sumstats["P"] = np.power(10,-sumstats["MLOG10P"])
         filled_count +=1
 
     # Z -> P
     elif "Z" in sumstats.columns:
-        if verbose: log.write("  - Filling P value using Z column...")
+        log.write("  - Filling P value using Z column...", verbose=verbose)
         stats.chisqprob = lambda chisq, degree_of_freedom: stats.chi2.sf(chisq, degree_of_freedom)
         sumstats["P"] = ss.chisqprob(sumstats["Z"]**2,1)
         filled_count +=1
 
     elif "CHISQ" in sumstats.columns:
     #CHISQ -> P
-        if verbose: log.write("  - Filling P value using CHISQ column...")
+        log.write("  - Filling P value using CHISQ column...", verbose=verbose)
         stats.chisqprob = lambda chisq, degree_of_freedom: stats.chi2.sf(chisq, degree_of_freedom)
         if df is None:
             if only_sig is True and overwrite is True:
@@ -80,11 +80,11 @@ def fill_p(sumstats,log,df=None,only_sig=False,sig_level=5e-8,overwrite=False,ve
                 filled_count +=1
         else:
             if only_sig is True and overwrite is True:
-                if verbose: log.write("  - Filling P value using CHISQ column for variants:" , sum(sumstats["P"]<sig_level))
+                log.write("  - Filling P value using CHISQ column for variants:" , sum(sumstats["P"]<sig_level), verbose=verbose)
                 sumstats.loc[sumstats["P"]<sig_level,"P"] = stats.chisqprob(sumstats.loc[sumstats["P"]<sig_level,"CHISQ"],sumstats.loc[sumstats["P"]<sig_level,df].astype("int"))
                 filled_count +=1
             else:
-                if verbose: log.write("  - Filling P value using CHISQ column for all valid variants:")
+                log.write("  - Filling P value using CHISQ column for all valid variants:", verbose=verbose)
                 sumstats["P"] = stats.chisqprob(sumstats["CHISQ"],sumstats[df].astype("int"))
                 filled_count +=1
     else:
@@ -94,7 +94,7 @@ def fill_p(sumstats,log,df=None,only_sig=False,sig_level=5e-8,overwrite=False,ve
 def fill_z(sumstats,log,verbose=True,filled_count=0):
     # BETA/SE -> Z
     if ("BETA" in sumstats.columns) and ("SE" in sumstats.columns):
-        if verbose: log.write("  - Filling Z using BETA/SE column...")
+        log.write("  - Filling Z using BETA/SE column...", verbose=verbose)
         sumstats["Z"] = sumstats["BETA"]/sumstats["SE"]
         filled_count +=1
     else:
@@ -104,12 +104,12 @@ def fill_z(sumstats,log,verbose=True,filled_count=0):
 def fill_chisq(sumstats,log,verbose=True,filled_count=0):
     # Z -> CHISQ
     if "Z" in sumstats.columns:
-        if verbose: log.write("  - Filling CHISQ using Z column...")
+        log.write("  - Filling CHISQ using Z column...", verbose=verbose)
         sumstats["CHISQ"] = (sumstats["Z"])**2
         filled_count +=1
     elif "P" in sumstats.columns:
     # P -> CHISQ
-        if verbose: log.write("  - Filling CHISQ using P column...")
+        log.write("  - Filling CHISQ using P column...", verbose=verbose)
         sumstats["CHISQ"] = ss.chi2.isf(sumstats["P"], 1)
         filled_count +=1
     else:
@@ -119,13 +119,13 @@ def fill_chisq(sumstats,log,verbose=True,filled_count=0):
 def fill_or(sumstats,log,verbose=True,filled_count=0):
     # BETA -> OR
     if "BETA" in sumstats.columns:
-        if verbose: log.write("  - Filling OR using BETA column...")
+        log.write("  - Filling OR using BETA column...", verbose=verbose)
         sumstats["OR"]   = np.exp(sumstats["BETA"])
         filled_count +=1
         # BETA/SE -> OR_95L / OR_95U
         # get confidence interval 95
         if ("BETA" in sumstats.columns) and ("SE" in sumstats.columns):
-            if verbose: log.write("  - Filling OR_95L/OR_95U using BETA/SE columns...")
+            log.write("  - Filling OR_95L/OR_95U using BETA/SE columns...", verbose=verbose)
             # beta - 1.96 x se , beta + 1.96 x se
             sumstats["OR_95L"] = np.exp(sumstats["BETA"]-ss.norm.ppf(0.975)*sumstats["SE"])
             sumstats["OR_95U"] = np.exp(sumstats["BETA"]+ss.norm.ppf(0.975)*sumstats["SE"])
@@ -136,7 +136,7 @@ def fill_or(sumstats,log,verbose=True,filled_count=0):
 def fill_or95(sumstats,log,verbose=True,filled_count=0):
     # get confidence interval 95
     if ("BETA" in sumstats.columns) and ("SE" in sumstats.columns):
-        if verbose: log.write("  - Filling OR_95L/OR_95U using BETA/SE columns...")
+        log.write("  - Filling OR_95L/OR_95U using BETA/SE columns...", verbose=verbose)
         # beta - 1.96 x se , beta + 1.96 x se
         sumstats["OR_95L"] = np.exp(sumstats["BETA"]-ss.norm.ppf(0.975)*sumstats["SE"])
         sumstats["OR_95U"] = np.exp(sumstats["BETA"]+ss.norm.ppf(0.975)*sumstats["SE"])
@@ -148,7 +148,7 @@ def fill_or95(sumstats,log,verbose=True,filled_count=0):
 def fill_beta(sumstats,log,verbose=True,filled_count=0):
     # OR -> beta
     if "OR" in sumstats.columns:
-        if verbose: log.write("  - Filling BETA value using OR column...")
+        log.write("  - Filling BETA value using OR column...", verbose=verbose)
         sumstats["BETA"]  = np.log(sumstats["OR"])    
         filled_count +=1
     else:
@@ -158,27 +158,27 @@ def fill_beta(sumstats,log,verbose=True,filled_count=0):
 def fill_se(sumstats,log,verbose=True,filled_count=0):
     # OR / OR_95L /OR_95U -> SE
     if ("P" in sumstats.columns) and ("BETA" in sumstats.columns):
-        if verbose: log.write("  - Filling SE value using BETA and P column...")
+        log.write("  - Filling SE value using BETA and P column...", verbose=verbose)
         sumstats["SE"]= np.abs(sumstats["BETA"]/ ss.norm.ppf(1-sumstats["P"]/2))
         filled_count +=1
     elif ("OR" in sumstats.columns) and ("OR_95U" in sumstats.columns): 
-        if verbose: log.write("  - Filling SE value using OR/OR_95U column...")
+        log.write("  - Filling SE value using OR/OR_95U column...", verbose=verbose)
         # 
         sumstats["SE"]=(np.log(sumstats["OR_95U"]) - np.log(sumstats["OR"]))/ss.norm.ppf(0.975)
         filled_count +=1
     elif ("OR" in sumstats.columns) and ("OR_95L" in sumstats.columns):
-        if verbose: log.write("  - Filling SE value using OR/OR_95L column...")
+        log.write("  - Filling SE value using OR/OR_95L column...", verbose=verbose)
         sumstats["SE"]=(np.log(sumstats["OR"]) - np.log(sumstats["OR_95L"]))/ss.norm.ppf(0.975)
         filled_count +=1
     else:
-        if verbose: log.write("  - Not enough information to fill SE...")
+        log.write("  - Not enough information to fill SE...", verbose=verbose)
         return 0,filled_count
     return 1,filled_count
 
 def fill_mlog10p(sumstats,log,verbose=True,filled_count=0):
     if "P" in sumstats.columns:
         # P -> MLOG10P
-        if verbose: log.write("  - Filling MLOG10P using P column...")
+        log.write("  - Filling MLOG10P using P column...", verbose=verbose)
         sumstats["MLOG10P"] = -np.log10(sumstats["P"])
         filled_count +=1
     else:
@@ -188,14 +188,14 @@ def fill_extreme_mlog10p(sumstats,log,verbose=True,filled_count=0):
     # ref: https://stackoverflow.com/questions/46416027/how-to-compute-p-values-from-z-scores-in-r-when-the-z-score-is-large-pvalue-muc/46416222#46416222
     if "Z" in sumstats.columns:
         # P -> MLOG10P
-        if verbose: log.write("  - Filling MLOG10P using Z column...")
+        log.write("  - Filling MLOG10P using Z column...", verbose=verbose)
         sumstats = fill_extreme_mlog10(sumstats, "Z")
         filled_count +=1
     elif "BETA" in sumstats.columns and "SE" in sumstats.columns:
-        if verbose: log.write("  - Z column not available...")
-        if verbose: log.write("  - Filling Z using BETA/SE column...")
+        log.write("  - Z column not available...", verbose=verbose)
+        log.write("  - Filling Z using BETA/SE column...", verbose=verbose)
         sumstats["Z"] = sumstats["BETA"]/sumstats["SE"]
-        if verbose: log.write("  - Filling MLOG10P using Z column...")
+        log.write("  - Filling MLOG10P using Z column...", verbose=verbose)
         sumstats = fill_extreme_mlog10(sumstats, "Z")
         filled_count +=1
     else:
@@ -205,7 +205,7 @@ def fill_extreme_mlog10p(sumstats,log,verbose=True,filled_count=0):
 def fill_maf(sumstats,log,verbose=True,filled_count=0):
     if "EAF" in sumstats.columns:
         # EAF -> MAF
-        if verbose: log.write("  - Filling MAF using EAF column...")
+        log.write("  - Filling MAF using EAF column...", verbose=verbose)
         sumstats["MAF"] =  sumstats["EAF"].apply(lambda x: min(x,1-x) if pd.notnull(x) else np.nan)
         filled_count +=1
     else:
@@ -226,7 +226,7 @@ def fill_extreme_mlog10(sumstats, z):
 ####################################################################################################################
 def fill_iteratively(sumstats,raw_to_fill,log,only_sig,df,extreme,verbose,sig_level):
     to_fill = raw_to_fill.copy()
-    if verbose: log.write("  - Filling Columns iteratively...")
+    log.write("  - Filling Columns iteratively...", verbose=verbose)
 
     filled_count=0
     for i in range(len(to_fill)+1):

gwaslab/util_in_filter_value.py

@@ -10,65 +10,66 @@ from gwaslab.g_vchange_status import vchange_status
 from gwaslab.qc_fix_sumstats import sortcoordinate
 from gwaslab.qc_fix_sumstats import start_to
 from gwaslab.qc_fix_sumstats import finished
+from gwaslab.hm_harmonize_sumstats import is_palindromic
 
 import gc
 def filtervalues(sumstats,expr,remove=False,verbose=True,log=Log()):
-    if verbose: log.write("Start filtering values by condition:",expr)
+    log.write("Start filtering values by condition:",expr, verbose=verbose)
     prenum = len(sumstats)
     sumstats = sumstats.query(expr,engine='python').copy()
     afternum = len(sumstats)
-    if verbose: log.write(" -Removing "+ str(prenum-afternum) +" variants not meeting the conditions:",expr)
-    if verbose: log.write("Finished filtering values.")
+    log.write(" -Removing "+ str(prenum-afternum) +" variants not meeting the conditions:",expr, verbose=verbose)
+    log.write("Finished filtering values.", verbose=verbose)
     gc.collect()
     return sumstats
 
 def filterout(sumstats,interval={},lt={},gt={},eq={},remove=False,verbose=True,log=Log()):
-    if verbose: log.write("Start filtering values:")
+    log.write("Start filtering values:", verbose=verbose)
     for key,threshold in gt.items():
         num = len(sumstats.loc[sumstats[key]>threshold,:])
-        if verbose:log.write(" -Removing "+ str(num) +" variants with "+key+" > "+ str(threshold)+" ...")
+        log.write(" -Removing "+ str(num) +" variants with "+key+" > "+ str(threshold)+" ...", verbose=verbose)
         sumstats = sumstats.loc[sumstats[key]<threshold,:]
     for key,threshold in lt.items():
         num = len(sumstats.loc[sumstats[key]<threshold,:])
-        if verbose:log.write(" -Removing "+ str(num) +" variants with "+key+" < "+ str(threshold)+" ...")
+        log.write(" -Removing "+ str(num) +" variants with "+key+" < "+ str(threshold)+" ...", verbose=verbose)
         sumstats = sumstats.loc[sumstats[key]>threshold,:]
     for key,threshold in eq.items():
         num = len(sumstats.loc[sumstats[key]==threshold,:])
-        if verbose:log.write(" -Removing "+ str(num) +" variants with "+key+" = "+ str(threshold)+" ...")
+        log.write(" -Removing "+ str(num) +" variants with "+key+" = "+ str(threshold)+" ...", verbose=verbose)
         sumstats = sumstats.loc[sumstats[key]!=threshold,:]
-    if verbose: log.write("Finished filtering values.")
+    log.write("Finished filtering values.", verbose=verbose)
     gc.collect()
     return sumstats.copy()
 
 def filterin(sumstats,lt={},gt={},eq={},remove=False,verbose=True,log=Log()):
-    if verbose: log.write("Start filtering values:")
+    log.write("Start filtering values:", verbose=verbose)
     for key,threshold in gt.items():
         num = len(sumstats.loc[sumstats[key]>threshold,:])
-        if verbose:log.write(" -Keeping "+ str(num) +" variants with "+key+" > "+ str(threshold)+" ...")
+        log.write(" -Keeping "+ str(num) +" variants with "+key+" > "+ str(threshold)+" ...", verbose=verbose)
         sumstats = sumstats.loc[sumstats[key]>threshold,:]
     for key,threshold in lt.items():
         num = len(sumstats.loc[sumstats[key]<threshold,:])
-        if verbose:log.write(" -Keeping "+ str(num) +" variants with "+key+" < "+ str(threshold)+" ...")
+        log.write(" -Keeping "+ str(num) +" variants with "+key+" < "+ str(threshold)+" ...", verbose=verbose)
         sumstats = sumstats.loc[sumstats[key]<threshold,:]
     for key,threshold in eq.items():
         num = len(sumstats.loc[sumstats[key]==threshold,:])
-        if verbose:log.write(" -Keeping "+ str(num) +" variants with "+key+" = "+ str(threshold)+" ...")
+        log.write(" -Keeping "+ str(num) +" variants with "+key+" = "+ str(threshold)+" ...", verbose=verbose)
         sumstats = sumstats.loc[sumstats[key]==threshold,:]
-    if verbose: log.write("Finished filtering values.")
+    log.write("Finished filtering values.", verbose=verbose)
     gc.collect()
     return sumstats.copy()
 
 def filterregionin(sumstats,path=None, chrom="CHR",pos="POS", high_ld=False, build="19", verbose=True,log=Log()):
     sumstats = sortcoordinate(sumstats,verbose=verbose)
-    if verbose: log.write("Start to filter in variants if in intervals defined in bed files:")
-    if verbose: log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns))
+    log.write("Start to filter in variants if in intervals defined in bed files:", verbose=verbose)
+    log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns), verbose=verbose)
     
     if high_ld is True:
         path = get_high_ld(build=build)
-        if verbose: log.write(" -Loading bed format file for hg"+build)
+        log.write(" -Loading bed format file for hg"+build, verbose=verbose)
 
     else:
-        if verbose: log.write(" -Loading bed format file: " , path)
+        log.write(" -Loading bed format file: " , path, verbose=verbose)
     bed = pd.read_csv(path,sep="\s+",header=None,dtype={0:"string",1:"Int64",2:"Int64"})
     
     bed["tuple"] = bed.apply(lambda x: (x[1],x[2]),axis=1)
@@ -80,7 +81,7 @@ def filterregionin(sumstats,path=None, chrom="CHR",pos="POS", high_ld=False, bui
     sumstats = sumstats.sort_values(["CHR","POS"])
     
     if len(bed)<100:
-        if verbose: log.write(" -Bed file < 100 lines: using pd IntervalIndex... ")
+        log.write(" -Bed file < 100 lines: using pd IntervalIndex... ", verbose=verbose)
         for i in sumstats[chrom].unique():
             if sum(bed[0]==i)>0:
                 interval = pd.IntervalIndex.from_tuples(bed.loc[bed[0]==i,"tuple"])
@@ -88,7 +89,7 @@ def filterregionin(sumstats,path=None, chrom="CHR",pos="POS", high_ld=False, bui
             else:
                 continue
     else:
-        if verbose: log.write(" -Bed file > 100 lines: using two pointers, please make files are all sorted... ")
+        log.write(" -Bed file > 100 lines: using two pointers, please make files are all sorted... ", verbose=verbose)
         bed_num  =0
         bed_chr   =bed.iloc[bed_num,0]
         bed_left  =bed.iloc[bed_num,1]
@@ -136,23 +137,23 @@ def filterregionin(sumstats,path=None, chrom="CHR",pos="POS", high_ld=False, bui
     ## in
     
     sumstats = sumstats.loc[sumstats["bed_indicator"],:]
-    if verbose: log.write(" -Number of variants in the specified regions to keep:",sum(sumstats["bed_indicator"]))
-    if verbose: log.write(" -Number of variants removed:",sum(~sumstats["bed_indicator"]))
+    log.write(" -Number of variants in the specified regions to keep:",sum(sumstats["bed_indicator"]), verbose=verbose)
+    log.write(" -Number of variants removed:",sum(~sumstats["bed_indicator"]), verbose=verbose)
     sumstats = sumstats.drop(columns="bed_indicator")
-    if verbose: log.write("Finished filtering in variants.")
+    log.write("Finished filtering in variants.", verbose=verbose)
     gc.collect()
     return sumstats
 
 def filterregionout(sumstats, path=None, chrom="CHR",pos="POS", high_ld=False, build="19", verbose=True,log=Log()):
     sumstats = sortcoordinate(sumstats,verbose=verbose)
-    if verbose: log.write("Start to filter out variants if in intervals defined in bed files:")
-    if verbose: log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns))
+    log.write("Start to filter out variants if in intervals defined in bed files:", verbose=verbose)
+    log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns), verbose=verbose)
     if high_ld is True:
         path = get_high_ld(build=build)
-        if verbose: log.write(" -Loading bed format file for hg"+build)
+        log.write(" -Loading bed format file for hg"+build, verbose=verbose)
 
     else:
-        if verbose: log.write(" -Loading bed format file: " , path)
+        log.write(" -Loading bed format file: " , path, verbose=verbose)
             
     bed = pd.read_csv(path,sep="\s+",header=None,dtype={0:"string",1:"Int64",2:"Int64"})
     bed["tuple"] = bed.apply(lambda x: (x[1],x[2]),axis=1)
@@ -164,7 +165,7 @@ def filterregionout(sumstats, path=None, chrom="CHR",pos="POS", high_ld=False, b
     bed[0]=bed[0].astype("Int64")
     
     if len(bed)<100:
-        if verbose: log.write(" -Bed file < 100 lines: using pd IntervalIndex... ")
+        log.write(" -Bed file < 100 lines: using pd IntervalIndex... ", verbose=verbose)
         for i in sumstats[chrom].unique():
             if sum(bed[0]==i)>0:
                 interval = pd.IntervalIndex.from_tuples(bed.loc[bed[0]==i,"tuple"])
@@ -172,7 +173,7 @@ def filterregionout(sumstats, path=None, chrom="CHR",pos="POS", high_ld=False, b
             else:
                 continue
     else:
-        if verbose: log.write(" -Bed file > 100 lines: using two pointers, please make files are all sorted... ")
+        log.write(" -Bed file > 100 lines: using two pointers, please make files are all sorted... ", verbose=verbose)
         bed_num  =0
         bed_chr  =bed.iloc[bed_num,0]
         bed_left  =bed.iloc[bed_num,1]
@@ -208,10 +209,10 @@ def filterregionout(sumstats, path=None, chrom="CHR",pos="POS", high_ld=False, b
     ## out
     
     sumstats = sumstats.loc[~sumstats["bed_indicator"],:]
-    if verbose: log.write(" -Number of variants in the specified regions to exclude:",sum(sumstats["bed_indicator"]))
-    if verbose: log.write(" -Number of variants left:",len(sumstats))
+    log.write(" -Number of variants in the specified regions to exclude:",sum(sumstats["bed_indicator"]), verbose=verbose)
+    log.write(" -Number of variants left:",len(sumstats), verbose=verbose)
     sumstats = sumstats.drop(columns="bed_indicator")
-    if verbose: log.write("Finished filtering out variants.")
+    log.write("Finished filtering out variants.", verbose=verbose)
     gc.collect()
     return sumstats
 
@@ -235,14 +236,14 @@ def inferbuild(sumstats,status="STATUS",chrom="CHR", pos="POS", ea="EA", nea="NE
     ############################################################################################
 
     inferred_build="Unknown"
-    if verbose:log.write("Start to infer genome build version using hapmap3 SNPs...")    
+    log.write("Start to infer genome build version using hapmap3 SNPs...", verbose=verbose)    
     data_path_19 =  path.dirname(__file__) + '/data/hapmap3_SNPs/hapmap3_db150_hg19.snplist.gz'    
     data_path_38 =  path.dirname(__file__) + '/data/hapmap3_SNPs/hapmap3_db151_hg38.snplist.gz'    
-    if verbose:log.write(" -Loading Hapmap3 variants data...")        
+    log.write(" -Loading Hapmap3 variants data...", verbose=verbose)        
     hapmap3_ref_19 = pd.read_csv(data_path_19,sep="\s+",usecols=["#CHROM","POS"],dtype={"#CHROM":"string","POS":"string"})
     hapmap3_ref_38 = pd.read_csv(data_path_38,sep="\s+",usecols=["#CHROM","POS"],dtype={"#CHROM":"string","POS":"string"})
     
-    if verbose: log.write(" -CHR:POS will be used for matching...")
+    log.write(" -CHR:POS will be used for matching...", verbose=verbose)
     raw_chrpos = sumstats[chrom].astype("string")+":"+sumstats[pos].astype("string")
     
     hapmap3_ref_19["chr:pos"] = hapmap3_ref_19["#CHROM"]+":"+hapmap3_ref_19["POS"]
@@ -251,50 +252,50 @@ def inferbuild(sumstats,status="STATUS",chrom="CHR", pos="POS", ea="EA", nea="NE
     match_count_for_19 = sum(raw_chrpos.isin(hapmap3_ref_19["chr:pos"].values))
     match_count_for_38 = sum(raw_chrpos.isin(hapmap3_ref_38["chr:pos"].values))
     
-    if verbose:log.write(" -Matching variants for hg19: num_hg19 = ",match_count_for_19)        
-    if verbose:log.write(" -Matching variants for hg38: num_hg38 = ",match_count_for_38) 
+    log.write(" -Matching variants for hg19: num_hg19 = ",match_count_for_19, verbose=verbose)        
+    log.write(" -Matching variants for hg38: num_hg38 = ",match_count_for_38, verbose=verbose) 
     
     if max(match_count_for_19, match_count_for_38)<10000:
-        if verbose:log.write(" -Warning: please be cautious due to the limited number of variants.") 
+        log.warning("Please be cautious due to the limited number of variants.", verbose=verbose) 
     
     if match_count_for_19 > match_count_for_38:
-        if verbose:log.write(" -Since num_hg19 >> num_hg38, assigning genome build hg19...") 
+        log.write(" -Since num_hg19 >> num_hg38, assigning genome build hg19...", verbose=verbose) 
         sumstats[status] = vchange_status(sumstats[status],1,"9","1")
         sumstats[status] = vchange_status(sumstats[status],2,"9","9")
         inferred_build="19"
     elif match_count_for_19 < match_count_for_38:
-        if verbose:log.write(" -Since num_hg19 << num_hg38, assigning genome build hg38...") 
+        log.write(" -Since num_hg19 << num_hg38, assigning genome build hg38...", verbose=verbose) 
         sumstats[status] = vchange_status(sumstats[status],1,"9","3")
         sumstats[status] = vchange_status(sumstats[status],2,"9","8")
         inferred_build="38"
     else:
-        if verbose:log.write(" -Since num_hg19 = num_hg38, unable to infer...") 
+        log.write(" -Since num_hg19 = num_hg38, unable to infer...", verbose=verbose) 
         
     finished(log,verbose,_end_line)
     return sumstats, inferred_build
 
 def sampling(sumstats,n=1, p=None, verbose=True,log=Log(),**args):
 
-    if verbose:log.write("Start to randomly select variants from the sumstats...") 
+    log.write("Start to randomly select variants from the sumstats...", verbose=verbose) 
     if p is None:
-        if verbose:log.write(" -Number of variants selected from the sumstats:",n)
+        log.write(" -Number of variants selected from the sumstats:",n, verbose=verbose)
         if n > len(sumstats):
             raise ValueError("Please input a number < {}".format(len(sumstats)))
     else:
         if p>-0.00000001 and p<1.00000001:
-            if verbose:log.write(" -Percentage of variants selected from the sumstats: ",p)
+            log.write(" -Percentage of variants selected from the sumstats: ",p, verbose=verbose)
             n = int(len(sumstats)*p)
-            if verbose:log.write(" -Number of variants selected from the sumstats:",n)
+            log.write(" -Number of variants selected from the sumstats:",n, verbose=verbose)
         else:
             raise ValueError("Please input a number in (0,1)")
     
     if "random_state" in args.keys():
-        if verbose:log.write(" -Random state (seed): {}".format(args["random_state"]))
+        log.write(" -Random state (seed): {}".format(args["random_state"]), verbose=verbose)
     else:
         args["random_state"] = np.random.randint(0,4294967295)
-        if verbose:log.write(" -Random state (seed): {}".format(args["random_state"]))
+        log.write(" -Random state (seed): {}".format(args["random_state"]), verbose=verbose)
     sampled = sumstats.sample(n=n,**args)
-    if verbose:log.write("Finished sampling...")
+    log.write("Finished sampling...", verbose=verbose)
     gc.collect()
     return sampled
 
@@ -322,8 +323,8 @@ def _get_flanking(sumstats, snpid, windowsizekb=500, verbose=True,log=Log(),**ar
 def _get_flanking_by_id(sumstats, snpid, windowsizekb=500, verbose=True,log=Log(),**args):
     
     log.write("Start to extract variants in the flanking regions using rsID or SNPID...",verbose=verbose)
-    log.write(" - Central variants: {}".format(snpid))
-    log.write(" - Flanking windowsize in kb: {}".format(windowsizekb))
+    log.write(" - Central variants: {}".format(snpid), verbose=verbose)
+    log.write(" - Flanking windowsize in kb: {}".format(windowsizekb), verbose=verbose)
 
     if type(snpid) == str:
         snpid = [snpid]
@@ -361,8 +362,8 @@ def _get_flanking_by_id(sumstats, snpid, windowsizekb=500, verbose=True,log=Log(
 def _get_flanking_by_chrpos(sumstats, chrpos, windowsizekb=500, verbose=True,log=Log(),**args):
     
     log.write("Start to extract variants in the flanking regions using CHR and POS...",verbose=verbose)
-    log.write(" - Central positions: {}".format(chrpos))
-    log.write(" - Flanking windowsize in kb: {}".format(windowsizekb))
+    log.write(" - Central positions: {}".format(chrpos), verbose=verbose)
+    log.write(" - Flanking windowsize in kb: {}".format(windowsizekb), verbose=verbose)
 
     if type(chrpos) == tuple:
         chrpos_to_check = [chrpos]
@@ -389,4 +390,42 @@ def _get_flanking_by_chrpos(sumstats, chrpos, windowsizekb=500, verbose=True,log
     log.write(" - Extracted {} variants in the regions.".format(len(flanking)),verbose=verbose)
     log.write("Finished extracting variants in the flanking regions.",verbose=verbose)
 
-    return flanking
+    return flanking
+
+def _filter_palindromic(sumstats, mode="in", ea="EA",nea="NEA", log=Log(),verbose=True):
+    log.write("Start to filter palindromic variants...",verbose=verbose)
+    is_palindromic_snp = is_palindromic(sumstats[[nea,ea]],a1=nea,a2=ea)   
+    
+    log.write(" -Identified palindromic variants: {}".format(sum(is_palindromic_snp)),verbose=verbose)
+    
+    if mode=="in":
+        palindromic = sumstats.loc[is_palindromic_snp,:]
+    else:
+        palindromic = sumstats.loc[~is_palindromic_snp,:]
+
+    log.write("Finished filtering palindromic variants.",verbose=verbose)
+    return palindromic
+
+def _filter_indel(sumstats, mode="in", ea="EA",nea="NEA", log=Log(),verbose=True):
+    log.write("Start to filter indels...",verbose=verbose)
+    is_indel = (sumstats[ea].str.len()!=sumstats[nea].str.len()) 
+    
+    log.write(" -Identified indels: {}".format(sum(is_indel)),verbose=verbose)
+    if mode=="in":
+        indel = sumstats.loc[is_indel,:]
+    else:
+        indel = sumstats.loc[~is_indel,:]
+    log.write("Finished filtering indels.",verbose=verbose)
+    return indel
+
+def _filter_snp(sumstats, mode="in", ea="EA",nea="NEA", log=Log(),verbose=True):
+    log.write("Start to filter SNPs...",verbose=verbose)
+    is_snp = (sumstats[ea].str.len()==1) &(sumstats[nea].str.len()==1)
+    
+    log.write(" -Identified SNPs: {}".format(sum(is_snp)),verbose=verbose)
+    if mode=="in":
+        snp = sumstats.loc[is_snp,:]
+    else:
+        snp = sumstats.loc[~is_snp,:]
+    log.write("Finished filtering SNPs.",verbose=verbose)
+    return snp

gwaslab/util_in_get_density.py

@@ -5,9 +5,9 @@ from gwaslab.g_Log import Log
 import gc
 
 def getsignaldensity(insumstats, id="SNPID", chrom="CHR",pos="POS", bwindowsizekb=100,log=Log(),verbose=True):    
-    if verbose:log.write("Start to calculate signal DENSITY...")
+    log.write("Start to calculate signal DENSITY..." ,verbose=verbose)
     sumstats = insumstats[[id,chrom,pos]].copy()
-    if verbose:log.write(" -Calculating DENSITY with windowsize of ",bwindowsizekb ," kb")
+    log.write(" -Calculating DENSITY with windowsize of ",bwindowsizekb ," kb",verbose=verbose)
     #stack=[]
 
     large_number = 1000000000
@@ -58,13 +58,13 @@ def getsignaldensity(insumstats, id="SNPID", chrom="CHR",pos="POS", bwindowsizek
     bmax = sumstats["DENSITY"].max()
     bmaxid = sumstats["DENSITY"].idxmax()
 
-    if verbose:log.write(" -Mean : {} signals per {} kb".format(bmean,bwindowsizekb))
-    if verbose:log.write(" -SD : {}".format(bsd))
-    if verbose:log.write(" -Median : {} signals per {} kb".format(bmedian,bwindowsizekb))
-    if verbose:log.write(" -Max : {} signals per {} kb at variant(s) {}".format(bmax,bwindowsizekb,sumstats.loc[bmaxid,id]))
+    log.write(" -Mean : {} signals per {} kb".format(bmean,bwindowsizekb),verbose=verbose)
+    log.write(" -SD : {}".format(bsd),verbose=verbose)
+    log.write(" -Median : {} signals per {} kb".format(bmedian,bwindowsizekb),verbose=verbose)
+    log.write(" -Max : {} signals per {} kb at variant(s) {}".format(bmax,bwindowsizekb,sumstats.loc[bmaxid,id]),verbose=verbose)
     
     sumstats = sumstats.drop("TCHR+POS",axis=1)
-    if verbose:log.write("Finished calculating signal DENSITY successfully!")
+    log.write("Finished calculating signal DENSITY successfully!",verbose=verbose)
     return sumstats["DENSITY"]
 
 def assigndensity(insumstats,
@@ -92,7 +92,7 @@ def assigndensity(insumstats,
         to_add =(sumstats["TCHR+POS"]>=(row["TCHR+POS"]- 1000*bwindowsizekb)) & (sumstats["TCHR+POS"]<=(row["TCHR+POS"]+ 1000*bwindowsizekb))
         sumstats.loc[to_add,"DENSITY"] += 1
         if counter%1000==0:
-            if verbose:log.write(" -Processed {} signals".format(counter//1000))
+            log.write(" -Processed {} signals".format(counter//1000),verbose=verbose)
             gc.collect()
 	
     return sumstats["DENSITY"]