gwaslab 3.5.5__py3-none-any.whl → 3.5.6__py3-none-any.whl

gwaslab/hm_harmonize_sumstats.py

@@ -1490,10 +1490,100 @@ def infer_af(chr,start,end,ref,alt,vcf_reader,alt_freq,chr_dict=None):
             elif record.ref==alt and (ref in record.alts):
                 return 1 - record.info[alt_freq][0]
     return np.nan
+##############################################################################################################################################################################################
 
+################################################################################################################
 
+def _paralleleinferafwithmaf(sumstats,ref_infer,ref_alt_freq=None,n_cores=1, chr="CHR",pos="POS",ref="NEA",alt="EA",
+                            eaf="EAF",maf="MAF",ref_eaf="_REF_EAF",status="STATUS",chr_dict=None,force=False, verbose=True,log=Log()):
+    ##start function with col checking##########################################################
+    _start_line = "infer sumstats EAF from sumstats MAF using reference VCF ALT frequency"
+    _end_line = "inferring sumstats EAF from sumstats MAF using reference VCF ALT frequency"
+    _start_cols = [chr,pos,ref,alt,status]
+    _start_function = ".infer_af()"
+    _must_args ={"ref_alt_freq":ref_alt_freq}
 
-################################################################################################################
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            n_cores=n_cores,
+                            ref_vcf=ref_infer,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+    chr_dict = auto_check_vcf_chr_dict(ref_infer, chr_dict, verbose, log)
+    
+    if eaf not in sumstats.columns:
+        sumstats[eaf]=np.nan
+    if ref_eaf not in sumstats.columns:
+        sumstats[ref_eaf]=np.nan
+
+    prenumber = sum(sumstats[eaf].isna())
+
+    # ref_alt_freq INFO in vcf was provided
+    if ref_alt_freq is not None:
+        log.write(" -Field for alternative allele frequency in VCF INFO: {}".format(ref_alt_freq), verbose=verbose)   
+        if not force:
+            good_chrpos =  sumstats[status].str.match(r'\w\w\w[0]\w\w\w', case=False, flags=0, na=False)  
+        log.write(" -Checking variants:", sum(good_chrpos),verbose=verbose) 
+    
+        ########################  
+        #extract ref af
+        if sum(sumstats[eaf].isna())<10000: 
+            n_cores=1       
+        #df_split = np.array_split(sumstats.loc[good_chrpos,[chr,pos,ref,alt]], n_cores)
+        df_split = _df_split(sumstats.loc[good_chrpos,[chr,pos,ref,alt]], n_cores)
+        pool = Pool(n_cores)
+        map_func = partial(inferaf,chr=chr,pos=pos,ref=ref,alt=alt,eaf=ref_eaf,ref_infer=ref_infer,ref_alt_freq=ref_alt_freq,chr_dict=chr_dict) 
+        sumstats.loc[good_chrpos,[ref_eaf]] = pd.concat(pool.map(map_func,df_split))
+        pool.close()
+        pool.join()
+        
+        ###########################
+        # infer sumstats EAF 
+        # based on sumstats MAF and reference EAF
+        is_filpped = ((sumstats[ref_eaf]>=0.5)&(sumstats[maf]<=0.5)) |((sumstats[ref_eaf]<0.5)&(sumstats[maf]>0.5)) 
+        sumstats[eaf] = sumstats[maf]
+        log.write(" -Flipping MAF to obtain EAF for {} variants".format(sum(is_filpped)),verbose=verbose) 
+        sumstats.loc[is_filpped,eaf] = 1 - sumstats.loc[is_filpped,maf]
+
+        ###########################    
+        afternumber = sum(sumstats[eaf].isna())
+        log.write(" -Inferred EAF for {} variants.".format(prenumber - afternumber),verbose=verbose) 
+        log.write(" -EAF is still missing for {} variants.".format(afternumber),verbose=verbose) 
+        sumstats = sumstats.drop(columns=[ref_eaf])
+    
+    finished(log,verbose,_end_line)
+    return sumstats
+
+def inferaf(sumstats,ref_infer,ref_alt_freq=None,chr="CHR",pos="POS",ref="NEA",alt="EA",eaf="EAF",chr_dict=None):
+    #vcf_reader = vcf.Reader(open(ref_infer, 'rb'))
+    vcf_reader = VariantFile(ref_infer)
+    def afapply(x,vcf,alt_freq,chr_dict):
+            return infer_af(x.iloc[0],x.iloc[1]-1,x.iloc[1],x.iloc[2],x.iloc[3],vcf_reader,ref_alt_freq,chr_dict)
+    map_func = partial(afapply,vcf=vcf_reader,alt_freq=ref_alt_freq,chr_dict=chr_dict)
+    status_inferred = sumstats.apply(map_func,axis=1)
+    sumstats[eaf] = status_inferred.values
+    sumstats[eaf]=sumstats[eaf].astype("float") 
+    return sumstats
+
+def infer_af(chr,start,end,ref,alt,vcf_reader,alt_freq,chr_dict=None):
+    if chr_dict is not None: chr=chr_dict[chr]
+    chr_seq = vcf_reader.fetch(chr,start,end)
+    
+    for record in chr_seq:
+        if record.pos==end:
+            if record.ref==ref and (alt in record.alts):
+                return record.info[alt_freq][0]
+            elif record.ref==alt and (ref in record.alts):
+                return 1 - record.info[alt_freq][0]
+    return np.nan
+
+##############################################################################################################################################################################################
 def auto_check_vcf_chr_dict(vcf_path, vcf_chr_dict, verbose, log):    
     if vcf_path is not None:
         if vcf_chr_dict is None:

gwaslab/qc_fix_sumstats.py

@@ -1178,7 +1178,7 @@ def sanitycheckstats(sumstats,
                      t=(-99999,99999),
                      f=(0,float("Inf")),
                      p=(0,1),
-                     mlog10p=(0,9999),
+                     mlog10p=(0,99999),
                      beta=(-100,100),
                      se=(0,float("Inf")),
                      OR=(-100,100),

gwaslab/util_ex_ldproxyfinder.py

@@ -37,7 +37,7 @@ from gwaslab.hm_harmonize_sumstats import auto_check_vcf_chr_dict
             #check if in outcome and exposure snp list
             #replace
 
-def _extract_with_ld_proxy(  snplist=None,
+def _extract_with_ld_proxy( snplist=None,
                             common_sumstats=None,
                             sumstats1=None,
                             vcf_path=None, 
@@ -58,6 +58,7 @@ def _extract_with_ld_proxy(  snplist=None,
     is_needed=[]
     no_need  =[]
     
+    print(common_sumstats.head())
     for i in snplist:
         if i in common_sumstats["SNPID"].values:
             no_need.append(i)
@@ -72,7 +73,7 @@ def _extract_with_ld_proxy(  snplist=None,
     if len(in_sumstats)==0:
         log.write(" -No available variants for LD proxy checking...Skipping... ", verbose=verbose)
     else:
-        log.write(" -{}} available variants for LD proxy checking... ".format(len(in_sumstats)), verbose=verbose)
+        log.write(" -{} available variants for LD proxy checking... ".format(len(in_sumstats)), verbose=verbose)
 
     for index,row in in_sumstats.iterrows():
         # determine SNP and select region
@@ -93,6 +94,16 @@ def _extract_with_ld_proxy(  snplist=None,
         if len(flanking_sumstats)==0:
             log.write("  -No availble variants in the region...Skipping!", verbose=verbose)
             continue
+        
+        _get_rsq_single(in_sumstats.loc[index,["POS","NEA_1","EA_1"]], 
+                        row_pos=row["POS"], 
+                        vcf_path=vcf_path, 
+                        region=region,
+                        log=log, 
+                        verbose=verbose, 
+                        vcf_chr_dict=vcf_chr_dict, 
+                        tabix=tabix)
+
 
         flanking_sumstats = _get_rsq(row =in_sumstats.loc[index,["POS","NEA_1","EA_1"]],
                                      sumstats = flanking_sumstats, 
@@ -126,6 +137,81 @@ def _extract_with_ld_proxy(  snplist=None,
     return extracted_sumstats
 
 
+def _extract_ld_proxy(  snplist=None,
+                        common_sumstats=None,
+                        vcf_path=None, 
+                        vcf_chr_dict=None,
+                        tabix=None,
+                        log=Log(), 
+                        verbose=True, 
+                        windowsizekb=100,
+                        ld_threshold=0.8
+                            ):
+    ### Load vcf#######################################################################################
+    log.write("Start to load reference genotype...", verbose=verbose)
+    log.write(" -reference vcf path : "+ vcf_path, verbose=verbose)
+    if tabix is None:
+        tabix = which("tabix")
+    vcf_chr_dict = auto_check_vcf_chr_dict(vcf_path=vcf_path, vcf_chr_dict=vcf_chr_dict, verbose=verbose, log=log)
+    
+    ld_proxies  = pd.DataFrame()
+    in_sumstats = common_sumstats.loc[common_sumstats["SNPID"].isin(snplist),:]
+    
+    if len(in_sumstats)==0:
+        log.write(" -No available variants for LD proxy checking...Skipping... ", verbose=verbose)
+    else:
+        log.write(" -{} available variants for LD proxy checking... ".format(len(in_sumstats)), verbose=verbose)
+
+    for index,row in in_sumstats.iterrows():
+        # determine SNP and select region
+        snpid = row["SNPID"]
+        chrom= int(row["CHR"])
+        start= int(row["POS"]-windowsizekb*1000)
+        end=   int(row["POS"]+windowsizekb*1000)
+
+        region = (chrom, start, end)
+        
+        ###  #######################################################################################
+        #is_flanking = common_sumstats["CHR"] == chrom & common_sumstats["CHR"]>start & common_sumstats["CHR"]<end
+        #flanking_sumstats = common_sumstats.loc[is_flanking,:]
+        flanking_sumstats = common_sumstats.query('CHR == @chrom and @start < POS < @end',engine='python').copy()
+        
+        log.write(" -Extract {} variants in flanking region of {} for checking: {}:{}-{}".format(len(flanking_sumstats), snpid, chrom, start, end), verbose=verbose)
+
+        if len(flanking_sumstats)==0:
+            log.write("  -No availble variants in the region...Skipping!", verbose=verbose)
+            continue
+
+        flanking_sumstats = _get_rsq(row =in_sumstats.loc[index,["POS","NEA","EA"]],
+                                     sumstats = flanking_sumstats, 
+                                     row_pos=row["POS"], 
+                                     vcf_path=vcf_path, 
+                                     region=region,
+                                     log=log, 
+                                     verbose=verbose, 
+                                     vcf_chr_dict=vcf_chr_dict, 
+                                     tabix=tabix)
+        if flanking_sumstats is None:
+            log.write("  -{} is not found in the vcf...Skipping!".format(snpid))
+            continue
+        flanking_sumstats = flanking_sumstats.loc[flanking_sumstats["RSQ"]>ld_threshold,:]
+        
+        log.write("  -Variants in LD with {} (RSQ > {}): {}".format(snpid, ld_threshold,len(flanking_sumstats)), verbose=verbose)
+        
+        if len(flanking_sumstats)>0:
+            flanking_sumstats["LD_REF_VARIANT"]= snpid
+            for i,row_with_rsq in flanking_sumstats.iterrows():
+                if row_with_rsq["SNPID"] in common_sumstats["SNPID"].values:
+                    log.write("  -Top Proxy for {} is found: {} (LD RSQ= {})".format(snpid, row_with_rsq["SNPID"], row_with_rsq["RSQ"]))
+                    break
+            #row_with_rsq = pd.DataFrame(row_with_rsq)
+            ld_proxies = pd.concat([ld_proxies, flanking_sumstats], ignore_index=True)
+                    
+
+    log.write("Finished loading reference genotype successfully!", verbose=verbose)
+    return ld_proxies.sort_values(by="RSQ",ascending=False)
+
+
 def _get_rsq(    row,
                  sumstats,
                  row_pos,
@@ -205,4 +291,77 @@ def _get_rsq(    row,
         return sumstats
 
 def _check_if_in_sumstats2(row, sumstast):
-    pass
+    pass
+
+def _get_rsq_single(  row,
+                 row_pos,
+                 vcf_path, 
+                 region,
+                 log, 
+                 verbose, 
+                 vcf_chr_dict,
+                 tabix):
+    #load genotype data of the targeted region
+    ref_genotype = read_vcf(vcf_path,region=vcf_chr_dict[region[0]]+":"+str(region[1])+"-"+str(region[2]),tabix=tabix)
+    
+    if ref_genotype is None:
+        log.warning("No data was retrieved. Skipping ...", verbose=verbose)
+        ref_genotype=dict()
+        ref_genotype["variants/POS"]=np.array([],dtype="int64")
+        return None
+    
+    log.write("  -Retrieving index...", verbose=verbose)
+    log.write("  -Ref variants in the region: {}".format(len(ref_genotype["variants/POS"])), verbose=verbose)
+    #  match sumstats pos and ref pos: 
+    # get ref index for its first appearance of sumstats pos
+    #######################################################################################
+    def match_varaint(x):
+        # x: "POS,NEA,EA"
+        if np.any(ref_genotype["variants/POS"] == x.iloc[0]):
+            if len(np.where(ref_genotype["variants/POS"] == x.iloc[0] )[0])>1:  
+            # multiple position matches
+                for j in np.where(ref_genotype["variants/POS"] == x.iloc[0])[0]:
+                # for each possible match, compare ref and alt
+                    if x.iloc[1] == ref_genotype["variants/REF"][j]:
+                        if x.iloc[2] in ref_genotype["variants/ALT"][j]:
+                            return j
+                    elif x.iloc[1] in ref_genotype["variants/ALT"][j]:
+                        if x.iloc[2] == ref_genotype["variants/REF"][j]:
+                            return j
+                    else:
+                        return None
+            else: 
+                # single match
+                return np.where(ref_genotype["variants/POS"] == x.iloc[0] )[0][0] 
+        else:
+            # no position match
+            return None
+
+    #############################################################################################
+    lead_pos = row_pos
+
+    # if lead pos is available: 
+    if lead_pos in ref_genotype["variants/POS"]:
+        
+        # get ref index for lead snp
+        lead_snp_ref_index = match_varaint(row)
+        #lead_snp_ref_index = np.where(ref_genotype["variants/POS"] == lead_pos)[0][0]
+
+        # non-na other snp index
+        other_snps_ref_index = list(range(len(ref_genotype["calldata/GT"])))
+        other_snps_ref_index.remove(lead_snp_ref_index)
+
+        # get genotype 
+        lead_snp_genotype = GenotypeArray([ref_genotype["calldata/GT"][lead_snp_ref_index]]).to_n_alt()
+        other_snp_genotype = GenotypeArray(ref_genotype["calldata/GT"][other_snps_ref_index]).to_n_alt()
+        
+        log.write("  -Calculating Rsq...", verbose=verbose)
+        
+        if len(other_snp_genotype)>1:
+            valid_r2= np.power(rogers_huff_r_between(lead_snp_genotype,other_snp_genotype)[0],2)
+        else:
+            valid_r2= np.power(rogers_huff_r_between(lead_snp_genotype,other_snp_genotype),2)
+
+        ld_proxy = pd.DataFrame( {"SNPID":ref_genotype["variants/ID"][other_snps_ref_index],"RSQ":valid_r2 })
+
+    return  ld_proxy.sort_values(by="RSQ",ascending=False)

gwaslab/util_in_fill_data.py

@@ -40,7 +40,7 @@ def filldata(
         for i in skip_cols:
             to_fill.remove(i)
         log.write("  -Skipping columns: ",skip_cols, verbose=verbose) 
-    if len(set(to_fill) & set(["OR","OR_95L","OR_95U","BETA","SE","P","Z","CHISQ","MLOG10P","MAF"]))==0:
+    if len(set(to_fill) & set(["OR","OR_95L","OR_95U","BETA","SE","P","Z","CHISQ","MLOG10P","MAF","SIG"]))==0:
         log.write(" -No available columns to fill. Skipping.", verbose=verbose)
         log.write("Finished filling data using existing columns.", verbose=verbose)
         return sumstats
@@ -219,6 +219,20 @@ def fill_maf(sumstats,log,verbose=True,filled_count=0):
         return 0,filled_count
     return 1,filled_count
 
+def fill_sig(sumstats,log,sig_level=5e-8, verbose=True,filled_count=0):
+    if "P" in sumstats.columns or "MLOG10P" in sumstats.columns:
+        log.write("  - Determining significant using P and MLOG10P with threshold:{}".format(sig_level), verbose=verbose)
+        if "P" in sumstats.columns:
+            is_sig = sumstats["P"]<sig_level 
+        elif "MLOG10P" in sumstats.columns:
+            is_sig = sumstats["MLOG10P"]>np.log10(sig_level) 
+        sumstats["SIGNIFICANT"] = False
+        sumstats.loc[is_sig, "SIGNIFICANT"] = True
+        filled_count +=1
+    else:
+        return 0,filled_count
+    return 1,filled_count
+
 ####################################################################################################################
 def fill_extreme_mlog10(sumstats, z):
     log_pvalue = np.log(2) + ss.norm.logsf(np.abs(sumstats[z])) #two-sided
@@ -289,7 +303,10 @@ def fill_iteratively(sumstats,raw_to_fill,log,only_sig,df,extreme,verbose,sig_le
             else:
                 status,filled_count = fill_mlog10p(sumstats,log,verbose=verbose)
             if status == 1 : to_fill.remove("MLOG10P")
-            
+
+        if "SIG" in to_fill:
+            status,filled_count = fill_sig(sumstats,sig_level=sig_level ,log=log,verbose=verbose,filled_count=filled_count)
+            if status == 1 : to_fill.remove("SIG")
         if filled_count == 0:
             break
          

gwaslab/util_in_filter_value.py

@@ -527,4 +527,55 @@ def _filter_region(sumstats, region, chrom="CHR",pos="POS",log=Log(),verbose=Tru
         
         log.write(" -Extract SNPs in specified regions: "+str(sum(in_region_snp)),verbose=verbose)
         sumstats = sumstats.loc[in_region_snp,:]
-        return sumstats.copy()
+        return sumstats.copy()
+    
+def _search_variants( sumstats, snplist=None, 
+                     snpid="SNPID" ,rsid="rsID",
+                     chrom="CHR",pos="POS",ea="EA",nea="NEA",
+                     log=Log(),verbose=True):
+    log.write("Start to search for variants...", verbose=verbose)
+    # create a boolean col with FALSE 
+    if snpid in sumstats.columns:
+        is_extract = sumstats[snpid]!=sumstats[snpid]
+    else:
+        is_extract = sumstats[rsid]!=sumstats[rsid]
+    
+    # search each variant
+    for variant in snplist:
+        
+        if pd.api.types.is_list_like(variant):
+            # (1:1234)
+            single_chrom=variant[0]
+            single_pos=variant[1]
+            is_extract = is_extract | ((sumstats[pos] == single_pos ) &(sumstats[chrom] == single_chrom))
+        
+        elif pd.api.types.is_string_dtype(type(variant)):
+            # rs123
+            if "rsID" in sumstats.columns:
+                is_extract = is_extract | (sumstats["rsID"] == variant)
+
+            # 1:123:A:D
+            if "SNPID" in sumstats.columns:
+                is_extract = is_extract | (sumstats["SNPID"] == variant)
+
+            # 1:123:A:D -> (1:1234)
+            a= re.match(r'^(chr|Chr|CHR)?(\d+)[:_-](\d+)([:_-]([ATCG]+)[:_-]([ATCG]+))?$', variant, flags=0)
+            
+            if a is not None:
+                if a[4] is None:
+                    single_chrom=int(a[2])
+                    single_pos=int(a[3])
+                    is_extract = is_extract | ((sumstats[pos] == single_pos ) &(sumstats[chrom] == single_chrom))
+                else:
+                    single_chrom = int(a[2])
+                    single_pos = int(a[3])
+                    single_ea = a[5]
+                    single_nea = a[6]
+                    a_match = ((sumstats[nea] == single_nea) & (sumstats[ea] == single_ea)) | ((sumstats[nea] == single_ea) & (sumstats[ea] == single_nea))
+                    is_extract = is_extract | ((sumstats[pos] == single_pos ) &(sumstats[chrom] == single_chrom)  & a_match)
+                        
+    to_search =  sumstats.loc[is_extract,:].copy()
+    log.write(" -Found {} variants...".format(len(to_search)),verbose=verbose)
+
+    log.write("Finished searching variants.", verbose=verbose)
+    return to_search

gwaslab/util_in_merge.py

@@ -0,0 +1,51 @@
+import pandas as pd
+from gwaslab.g_Log import Log
+import re
+
+def _extract_variant(variant_set, sumstats_dic, log=Log(), verbose=True):
+    
+    combined = pd.DataFrame()
+    log.write("Start to initialize gl.SumstatsSet...", verbose=verbose)
+    for key, sumstats_gls in sumstats_dic.items():
+        log.write(" -{} : {}".format(key, sumstats_gls), verbose=verbose)
+
+    for key, sumstats_gls in sumstats_dic.items():
+        
+        sumstats_single = sumstats_gls.data
+        
+        # create a boolean col with FALSE 
+        is_extract = sumstats_single["SNPID"]!=sumstats_single["SNPID"]
+        
+        for variant in variant_set:
+            
+            if pd.api.types.is_list_like(variant):
+
+                chrom=variant[0]
+                pos=variant[1]
+
+                is_extract = is_extract | ((sumstats_single["POS"] == pos ) &(sumstats_single["CHR"] == chrom))
+            elif pd.api.types.is_string_dtype(type(variant)):
+                
+                is_extract = is_extract | (sumstats_single["SNPID"] == variant)
+
+                a= re.search(r'^(chr|Chr|CHR)?(\d+)[:_-](\d+)[:_-][ATCG]+[:_-][ATCG]+$', variant, flags=0)
+                if a is not None:
+                    chrom=int(a[2])
+                    pos=int(a[3])
+                    is_extract = is_extract | ((sumstats_single["POS"] == pos ) &(sumstats_single["CHR"] == chrom))
+
+        to_extract =  sumstats_single.loc[is_extract,:].copy()
+        log.write(" -Extracted {} variants from {}".format(len(to_extract), key),verbose=verbose)
+        to_extract["STUDY"] = key
+        
+        to_extract_cols=["STUDY"]
+
+        default_cols=["SNPID","EA","NEA","CHR","POS","BETA","SE","P","MLOG10P","EAF","MAF","STATUS"]
+        
+        for i in default_cols:
+            if i in sumstats_single.columns:
+                to_extract_cols.append(i)
+
+        combined = pd.concat([combined, to_extract[to_extract_cols]], ignore_index=True)
+    log.write("Finished initializing gl.SumstatsSet.", verbose=verbose)
+    return combined

gwaslab/viz_aux_save_figure.py

@@ -52,7 +52,8 @@ def get_default_path(keyword,fmt="png"):
                         "esc":"effect_size_comparision",
                         "afc":"allele_frequency_comparision",
                         "gwheatmap":"genome_wide_heatmap",
-                        "scatter":"scatter"
+                        "scatter":"scatter",
+                        "forest":"forest"
                         }
     prefix = path_dictionary[keyword]
     count = 1