gwaslab 3.5.7__py3-none-any.whl → 3.6.0__py3-none-any.whl

gwaslab/hm_casting.py

@@ -11,20 +11,38 @@ from gwaslab.util_in_fill_data import filldata
 from Bio import SeqIO
 from itertools import combinations
 
-def _merge_mold_with_sumstats_by_chrpos(mold, sumstats, ref_path=None, windowsizeb=10, log=Log(),suffixes=("_MOLD",""),verbose=True,return_not_matched_mold =False):
+def _merge_mold_with_sumstats_by_chrpos(mold, sumstats, ref_path=None,add_raw_index=False, stats_cols1=None, stats_cols2=None,
+                                        windowsizeb=10, 
+                                        log=Log(),
+                                        suffixes=("_MOLD",""),
+                                        merge_mode="inner",
+                                        verbose=True,
+                                        return_not_matched_mold =False):
     
-
+    log.write("Start to merge sumstats...", verbose=verbose)
+    if merge_mode=="outer":
+        sumstats = sumstats.rename(columns={
+                                            "SNPID":"_SNPID_RIGHT",
+                                            "rsID":"_rsID_RIGHT"
+                                            })
+    
+    # drop old ids
     cols_to_drop = []
     for i in sumstats.columns:
         if i in ["SNPID","rsID"]:
-            cols_to_drop.append(i)
-
-    log.write("Start to merge sumstats...", verbose=verbose)
-    
+            cols_to_drop.append(i)    
     if len(cols_to_drop)>0:
         log.write(" -Dropping old IDs:{}".format(cols_to_drop), verbose=verbose)
         sumstats = sumstats.drop(columns=cols_to_drop)
-    
+
+
+    if add_raw_index==True:
+        index1= "_INDEX" + suffixes[0]
+        index2= "_INDEX" + suffixes[1]
+        mold[index1] = mold.index
+        sumstats[index2] =  sumstats.index
+        
+
     if ref_path is not None :
         # index for checking removed variants
         index1= "_INDEX" + suffixes[0]
@@ -32,11 +50,35 @@ def _merge_mold_with_sumstats_by_chrpos(mold, sumstats, ref_path=None, windowsiz
         mold[index1] = range(len(mold))
         sumstats[index2] = range(len(sumstats))
     
-    if return_not_matched_mold:
-        mold["_IDENTIFIER_FOR_VARIANT"] = range(len(mold))
+    #if return_not_matched_mold:
+    #   mold["_IDENTIFIER_FOR_VARIANT"] = range(len(mold))
+    #   sumstats["_IDENTIFIER_FOR_VARIANT2"] = range(len(sumstats))
 
     # mold sumffix + mold 
-    mold_sumstats = pd.merge(mold, sumstats, on=["CHR","POS"], how="inner",suffixes=suffixes)
+    mold_sumstats = pd.merge(mold, sumstats, on=["CHR","POS"], how=merge_mode,suffixes=suffixes)
+
+    if merge_mode=="outer":
+        is_temp_na = mold_sumstats["EA_1"].isna()
+        log.write(" -Detected {} variants not in the template...".format(sum(is_temp_na)), verbose=verbose)
+        
+        mold_sumstats["EA_1"] = mold_sumstats["EA_1"].astype("string")
+        mold_sumstats["NEA_1"] = mold_sumstats["NEA_1"].astype("string")
+        mold_sumstats["EA"] = mold_sumstats["EA"].astype("string")
+        mold_sumstats["NEA"] = mold_sumstats["NEA"].astype("string")
+
+        # for variants not in template, copy snp info
+        mold_sumstats.loc[is_temp_na, ["SNPID","EA_1","NEA_1","STATUS_1"]] = mold_sumstats.loc[is_temp_na, ["_SNPID_RIGHT","EA","NEA","STATUS"]].values
+        
+        # 
+        if "_rsID_RIGHT" in mold_sumstats.columns:
+            mold_sumstats.loc[is_temp_na, "rsID"] = mold_sumstats.loc[is_temp_na, "_rsID_RIGHT"].values
+        
+        
+        # for variants not in right sumstats, copy snp info
+        is_temp_na_2 = mold_sumstats["EA"].isna()
+        mold_sumstats.loc[is_temp_na_2, ["EA","NEA"]] = mold_sumstats.loc[is_temp_na_2, ["EA_1","NEA_1"]].values
+        mold_sumstats = mold_sumstats.drop(columns=["_SNPID_RIGHT"])
+
     log.write(" -After merging by CHR and POS:{}".format(len(mold_sumstats)), verbose=verbose)
     
     mold_sumstats = _keep_variants_with_same_allele_set(mold_sumstats,suffixes=suffixes)
@@ -51,10 +93,17 @@ def _merge_mold_with_sumstats_by_chrpos(mold, sumstats, ref_path=None, windowsiz
     #    mold_sumstats.drop(columns=["_INDEX",""])
     
     if return_not_matched_mold == True:
-        sumstats1 = mold.loc[~mold["_IDENTIFIER_FOR_VARIANT"].isin(mold_sumstats["_IDENTIFIER_FOR_VARIANT"]),:]
-        sumstats1= sumstats1.drop(columns=["_IDENTIFIER_FOR_VARIANT"])
-        mold_sumstats= mold_sumstats.drop(columns=["_IDENTIFIER_FOR_VARIANT"])
-        return mold_sumstats, sumstats1
+
+        sumstats1 = mold.loc[~mold["_RAW_INDEX_1"].isin(mold_sumstats["_RAW_INDEX_1"]),:]
+        sumstats1 = sumstats1.drop(columns=["_RAW_INDEX_1"])
+        sumstats1 = _renaming_cols_r(sumstats1, stats_cols1 +["EA","NEA"],suffix="_1", verbose=False)
+        
+        sumstats2 = sumstats.loc[~sumstats["_RAW_INDEX_2"].isin(mold_sumstats["_RAW_INDEX_2"]),:]
+        sumstats2 = sumstats2.drop(columns=["_RAW_INDEX_2"])
+
+        mold_sumstats= mold_sumstats.drop(columns=["_RAW_INDEX_1","_RAW_INDEX_2"])
+        
+        return mold_sumstats, sumstats1, sumstats2
     
     return mold_sumstats
 
@@ -117,6 +166,16 @@ def _renaming_cols(sumstats, columns, log=Log(),verbose=True, suffixes=("_1","_2
     log.write(" -Renaming sumstats2 columns by adding suffix {}".format(suffixes[1]),verbose=verbose)
     return sumstats
 
+def _renaming_cols_r(sumstats, columns, log=Log(),verbose=True, suffix=""):
+    # columns: name without suffix
+    to_rename =[]
+    for col in columns:
+        if col + suffix in sumstats.columns:
+            to_rename.append(col)
+    sumstats = sumstats.rename(columns={i + suffix:i for i in to_rename})
+    log.write(" -Renaming sumstats columns by removing suffix {}".format(suffix),verbose=verbose)
+    return sumstats
+
 def _sort_pair_cols(molded_sumstats, verbose=True, log=Log(), order=None, stats_order=None,suffixes=("_1","_2")):
     if stats_order is None:
         order = ["SNPID","rsID", "CHR", "POS", "EA", "NEA"]
@@ -158,99 +217,99 @@ def _assign_warning_code(sumstats, threshold=0.2, log=Log(),verbose=True):
     return sumstats
 
 
-def _match_two_sumstats(mold,sumstats,ref_path,windowsizeb=25,verbose=True,log=Log()):
-    
-    records = SeqIO.parse(ref_path, "fasta")
-
-    chromlist = list(set(mold["CHR"].values) & set(sumstats["CHR"].values))
-    
-    for record in records:
-        if len(chromlist) ==0:
-            break
-
-        if record is not None:
-            ##############################################################################
-            record_chr = int(str(record.id).strip("chrCHR").upper())
-            
-            if record_chr in chromlist:
-                log.write(record_chr," ", end="",show_time=False,verbose=verbose) 
-                chromlist.remove(record_chr)
-            else:
-                continue
-            ###############################################################################
-            mold_chr = mold.loc[mold["CHR"]==record_chr,:]
-            sumstats_chr = sumstats.loc[sumstats["CHR"]==record_chr,:]
-
-            for index, row in sumstats_chr.iterrows():
-                if len(row["EA"])>1 or len(row["NEA"])>1:
-                    is_in_variants_lista = (mold_chr["POS"] > row["POS"] - windowsizeb) & (mold_chr["POS"]< row["POS"] + windowsizeb) 
-
-                    is_in_variants_listb = (sumstats_chr["POS"] > row["POS"] - windowsizeb) & (sumstats_chr["POS"]< row["POS"] + windowsizeb) 
-                    
-                    if sum(is_in_variants_lista)>0 and sum(is_in_variants_listb)>0 and (sum(is_in_variants_lista) + sum(is_in_variants_listb) >2):
-                        variants_lista = mold.loc[is_in_variants_lista,:]
-                        variants_listb = sumstats.loc[is_in_variants_listb,:]
-                        
-                        refseq = record[row["POS"]-1 - windowsizeb: row["POS"] + windowsizeb].seq.upper()
-                        _match_single_variant(refseq, variants_lista, variants_listb, left_offset=row["POS"] - windowsizeb, windowsizeb=windowsizeb)
-
-def _match_single_variant(refseq,  variants_lista, variants_listb, left_offset,windowsizeb):
-    
-    
-    seta=set()
-    setb=set()
-
-    seta_pumutations=[]
-    for i in range(1, len(variants_lista)+1):
-        seta_pumutations+=combinations(variants_lista.index, i)
-
-    for i in seta_pumutations:
-        if _is_ref_overlap(variants_lista.loc[i,:],suffix="_MOLD"):
-            continue
-        else:
-            seta = _form_haplotype(refseq, variants_lista.loc[i,:], seta, left_offset,suffix="_MOLD")
-    
-    setb_pumutations=[]
-    for i in range(1,len(variants_listb)+1):
-        setb_pumutations+=combinations(variants_listb.index, i)
-    for i in setb_pumutations:
-        if _is_ref_overlap(variants_listb.loc[i,:],suffix=""):
-            continue
-        else:
-            setb = _form_haplotype(refseq, variants_listb.loc[i,:], setb, left_offset,suffix="")
-        
-    if len(seta & setb)>0:
-        print("-Topmed--------------------------------")  
-        print(variants_lista[["CHR","POS","NEA_MOLD","EA_MOLD","EAF_MOLD"]])
-        print("-Finngen--------------------------------")  
-        print(variants_listb[["CHR","POS","NEA","EA","EAF"]])
-        print(refseq,left_offset)
-        print("-set a--------------------------------")  
-        print(seta)
-        print("-set b---------------------------------")    
-        print(setb)   
-        print("------------------------------------")    
-        print("maybe equivalent ########################################################################")
-        a = seta & setb
-        for i in a:
-            print(i)
-
-def _is_ref_overlap(variants_list,suffix="_MOLD"):
-    previous_end = 0
-    for index, row in variants_list.iterrows():
-        if row["POS"] <= previous_end:
-            return True    
-        if row["POS"] + len(row["NEA"+suffix]) -1 > previous_end:
-            previous_end = row["POS"] + len(row["NEA"+suffix]) -1
-    return False
-
-def _form_haplotype(refseq, variants_list, haplotype_set, left_offset,suffix="_MOLD"):       
-        new_haplotype = ""
-        lastpos = 0
-        for index, row in variants_list.iterrows():
-            new_haplotype += refseq[lastpos:row["POS"] - left_offset]
-            new_haplotype += row["EA"+suffix]
-            lastpos = row["POS"] + len(row["NEA"+suffix])- left_offset
-        new_haplotype  += refseq[lastpos:]
-        haplotype_set.add(new_haplotype)
-        return haplotype_set
+#def _match_two_sumstats(mold,sumstats,ref_path,windowsizeb=25,verbose=True,log=Log()):
+#    
+#    records = SeqIO.parse(ref_path, "fasta")
+#
+#    chromlist = list(set(mold["CHR"].values) & set(sumstats["CHR"].values))
+#    
+#    for record in records:
+#        if len(chromlist) ==0:
+#            break
+#
+#        if record is not None:
+#            ##############################################################################
+#            record_chr = int(str(record.id).strip("chrCHR").upper())
+#            
+#            if record_chr in chromlist:
+#                log.write(record_chr," ", end="",show_time=False,verbose=verbose) 
+#                chromlist.remove(record_chr)
+#            else:
+#                continue
+#            ###############################################################################
+#            mold_chr = mold.loc[mold["CHR"]==record_chr,:]
+#            sumstats_chr = sumstats.loc[sumstats["CHR"]==record_chr,:]
+#
+#            for index, row in sumstats_chr.iterrows():
+#                if len(row["EA"])>1 or len(row["NEA"])>1:
+#                    is_in_variants_lista = (mold_chr["POS"] > row["POS"] - windowsizeb) & (mold_chr["POS"]< row["POS"] + windowsizeb) 
+#
+#                    is_in_variants_listb = (sumstats_chr["POS"] > row["POS"] - windowsizeb) & (sumstats_chr["POS"]< row["POS"] + windowsizeb) 
+#                    
+#                    if sum(is_in_variants_lista)>0 and sum(is_in_variants_listb)>0 and (sum(is_in_variants_lista) + sum(is_in_variants_listb) >2):
+#                        variants_lista = mold.loc[is_in_variants_lista,:]
+#                        variants_listb = sumstats.loc[is_in_variants_listb,:]
+#                        
+#                        refseq = record[row["POS"]-1 - windowsizeb: row["POS"] + windowsizeb].seq.upper()
+#                        _match_single_variant(refseq, variants_lista, variants_listb, left_offset=row["POS"] - windowsizeb, windowsizeb=windowsizeb)
+#
+#def _match_single_variant(refseq,  variants_lista, variants_listb, left_offset,windowsizeb):
+#    
+#    
+#    seta=set()
+#    setb=set()
+#
+#    seta_pumutations=[]
+#    for i in range(1, len(variants_lista)+1):
+#        seta_pumutations+=combinations(variants_lista.index, i)
+#
+#    for i in seta_pumutations:
+#        if _is_ref_overlap(variants_lista.loc[i,:],suffix="_MOLD"):
+#            continue
+#        else:
+#            seta = _form_haplotype(refseq, variants_lista.loc[i,:], seta, left_offset,suffix="_MOLD")
+#    
+#    setb_pumutations=[]
+#    for i in range(1,len(variants_listb)+1):
+#        setb_pumutations+=combinations(variants_listb.index, i)
+#    for i in setb_pumutations:
+#        if _is_ref_overlap(variants_listb.loc[i,:],suffix=""):
+#            continue
+#        else:
+#            setb = _form_haplotype(refseq, variants_listb.loc[i,:], setb, left_offset,suffix="")
+#        
+#    if len(seta & setb)>0:
+#        print("-Topmed--------------------------------")  
+#        print(variants_lista[["CHR","POS","NEA_MOLD","EA_MOLD","EAF_MOLD"]])
+#        print("-Finngen--------------------------------")  
+#        print(variants_listb[["CHR","POS","NEA","EA","EAF"]])
+#        print(refseq,left_offset)
+#        print("-set a--------------------------------")  
+#        print(seta)
+#        print("-set b---------------------------------")    
+#        print(setb)   
+#        print("------------------------------------")    
+#        print("maybe equivalent ########################################################################")
+#        a = seta & setb
+#        for i in a:
+#            print(i)
+#
+#def _is_ref_overlap(variants_list,suffix="_MOLD"):
+#    previous_end = 0
+#    for index, row in variants_list.iterrows():
+#        if row["POS"] <= previous_end:
+#            return True    
+#        if row["POS"] + len(row["NEA"+suffix]) -1 > previous_end:
+#            previous_end = row["POS"] + len(row["NEA"+suffix]) -1
+#    return False
+#
+#def _form_haplotype(refseq, variants_list, haplotype_set, left_offset,suffix="_MOLD"):       
+#        new_haplotype = ""
+#        lastpos = 0
+#        for index, row in variants_list.iterrows():
+#            new_haplotype += refseq[lastpos:row["POS"] - left_offset]
+#            new_haplotype += row["EA"+suffix]
+#            lastpos = row["POS"] + len(row["NEA"+suffix])- left_offset
+#        new_haplotype  += refseq[lastpos:]
+#        haplotype_set.add(new_haplotype)
+#        return haplotype_set

gwaslab/hm_casting_polars.py

@@ -0,0 +1,202 @@
+import pandas as pd
+import numpy as np
+from gwaslab.g_Log import Log
+from pandas.api.types import CategoricalDtype
+from gwaslab.g_vchange_status import copy_status
+from gwaslab.g_vchange_status_polars import vchange_statusp
+from gwaslab.g_vchange_status_polars import copy_statusp
+from gwaslab.qc_fix_sumstats import flipallelestats
+from gwaslab.qc_check_datatype import check_datatype
+from gwaslab.qc_fix_sumstats import start_to
+from gwaslab.util_in_fill_data import filldata
+from Bio import SeqIO
+from itertools import combinations
+import polars as pl
+
+def _merge_mold_with_sumstats_by_chrposp(mold, sumstats, ref_path=None,add_raw_index=False, stats_cols1=None, stats_cols2=None,
+                                        windowsizeb=10, 
+                                        log=Log(),
+                                        suffixes=("_MOLD",""),
+                                        merge_mode="full",
+                                        verbose=True,
+                                        return_not_matched_mold =False):
+    
+    log.write("Start to merge sumstats...", verbose=verbose)
+    if merge_mode=="full":
+        sumstats = sumstats.rename({
+                                            "SNPID":"_SNPID_RIGHT",
+                                            "rsID":"_rsID_RIGHT"
+                                            }, strict=False) #,
+    
+    # drop old ids
+    cols_to_drop = []
+    for i in sumstats.columns:
+        if i in ["SNPID","rsID"]:
+            cols_to_drop.append(i)    
+    if len(cols_to_drop)>0:
+        log.write(" -Dropping old IDs:{}".format(cols_to_drop), verbose=verbose)
+        sumstats = sumstats.drop(columns=cols_to_drop)
+
+    ##################################################################################################################
+
+    # mold sumffix + mold 
+
+    mold_sumstats = mold.join(sumstats, on=["CHR","POS"], how=merge_mode, suffix="_", coalesce=True)
+
+    if merge_mode=="full":
+        is_temp_na = mold_sumstats["EA_1"].is_null()
+        log.write(" -Detected {} variants not in the template...".format(sum(is_temp_na)), verbose=verbose)
+
+        for i in ["EA_1","NEA_1","EA","NEA"]:
+            mold_sumstats = mold_sumstats.with_columns(pl.col(i).cast(pl.String).alias(i))
+
+        # for variants not in template, copy snp info
+        mold_sumstats = mold_sumstats.with_columns(
+            pl.when( is_temp_na )  
+                .then(   pl.col("_SNPID_RIGHT")  )  
+                .otherwise( pl.col("SNPID") )
+                .alias("SNPID")
+        ).with_columns(
+            pl.when( is_temp_na )  
+                .then( pl.col("EA")  )  
+                .otherwise( pl.col("EA_1") )
+                .alias("EA_1")
+        ).with_columns(
+            pl.when( is_temp_na )  
+                .then( pl.col("NEA")  )  
+                .otherwise( pl.col("NEA_1") )
+                .alias("NEA_1")
+        ).with_columns(
+            pl.when( is_temp_na )  
+                .then( pl.col("STATUS")  )  
+                .otherwise( pl.col("STATUS_1") )
+                .alias("STATUS_1")
+        )
+        # 
+        if "_rsID_RIGHT" in mold_sumstats.columns:
+            mold_sumstats = mold_sumstats.with_columns(
+                pl.when( is_temp_na )  
+                .then(   pl.col("_rsID_RIGHT")  )  
+                .otherwise( pl.col("rsID") )
+                .alias("rsID")
+                )
+        
+        
+        # for variants not in right sumstats, copy snp info
+        is_temp_na_2 = mold_sumstats["EA"].is_null()
+        
+        mold_sumstats = mold_sumstats.with_columns(
+                pl.when( is_temp_na_2 )  
+                .then(   pl.col("EA_1")  )  
+                .otherwise( pl.col("EA") )
+                .alias("EA")
+                ).with_columns(
+                pl.when( is_temp_na_2 )  
+                .then(   pl.col("NEA_1")  )  
+                .otherwise( pl.col("NEA") )
+                .alias("NEA")
+                )
+        
+        mold_sumstats = mold_sumstats.drop(["_SNPID_RIGHT"])
+
+    log.write(" -After merging by CHR and POS:{}".format(len(mold_sumstats)), verbose=verbose)
+    
+    mold_sumstats = _keep_variants_with_same_allele_setp(mold_sumstats,suffixes=suffixes)
+
+    log.write(" -Matched variants:{}".format(len(mold_sumstats)), verbose=verbose)
+    
+    return mold_sumstats
+
+def _keep_variants_with_same_allele_setp(sumstats, log=Log(),verbose=True,suffixes=("_MOLD","")):
+
+    ea1="EA"+suffixes[0]
+    nea1="NEA"+suffixes[0]
+    ea2="EA"+suffixes[1]
+    nea2="NEA"+suffixes[1]
+    
+    is_perfect_match = (sumstats[ea2] == sumstats[ea1]) & (sumstats[nea2] == sumstats[nea1])
+    is_flipped_match = (sumstats[ea2] == sumstats[nea1]) & (sumstats[nea2] == sumstats[ea1])
+    is_allele_set_match = is_flipped_match | is_perfect_match
+    
+    log.write(" -Matching alleles and keeping only variants with same allele set: ", verbose=verbose)
+    log.write("  -Perfect match: {}".format(sum(is_perfect_match)), verbose=verbose)
+    log.write("  -Flipped match: {}".format(sum(is_flipped_match)), verbose=verbose)
+    log.write("  -Unmatched : {}".format(sum(~is_allele_set_match)), verbose=verbose)
+    sumstats = sumstats.filter(is_allele_set_match)
+    return sumstats
+
+def _align_with_moldp(sumstats, log=Log(),verbose=True, suffixes=("_MOLD","")):
+    
+    ea1="EA"+suffixes[0]
+    nea1="NEA"+suffixes[0]
+    ea2="EA"+suffixes[1]
+    nea2="NEA"+suffixes[1]
+    status1="STATUS"+suffixes[0]
+    status2="STATUS"+suffixes[1]
+
+    is_perfect_match = (sumstats[ea2] == sumstats[ea1]) & (sumstats[nea2] == sumstats[nea1])
+    is_flipped_match = (sumstats[ea2] == sumstats[nea1]) & (sumstats[nea2] == sumstats[ea1])
+    
+    log.write(" -Aligning alleles with reference: ", verbose=verbose)
+    log.write("  -Perfect match: {}".format(sum(is_perfect_match)), verbose=verbose)
+    log.write("  -Flipped match: {}".format(sum(is_flipped_match)), verbose=verbose)
+    
+    log.write("  -For perfect match: copy STATUS from reference...", verbose=verbose)
+
+    sumstats  = copy_statusp(sumstats, is_perfect_match, status1, status2, 6)
+
+    log.write("  -For Flipped match: convert STATUS xxxxx[456789]x to xxxxx3x...", verbose=verbose)
+    
+    sumstats  = vchange_statusp(sumstats, is_flipped_match, status2, 6,"456789","333333")
+    
+    return sumstats
+
+def _fill_missing_columnsp(sumstats, columns, log=Log(),verbose=True):
+    sumstats = filldata(sumstats, to_fill=columns)
+    return sumstats
+
+def _renaming_colsp(sumstats, columns, log=Log(),verbose=True, suffixes=("_1","_2")):
+    to_rename =["STATUS"]
+    for col in columns:
+        if col in sumstats.columns:
+            to_rename.append(col)
+    sumstats = sumstats.rename({i:i + suffixes[1] for i in to_rename})
+    log.write(" -Renaming sumstats2 columns by adding suffix {}".format(suffixes[1]),verbose=verbose)
+    return sumstats
+
+def _renaming_cols_rp(sumstats, columns, log=Log(),verbose=True, suffix=""):
+    # columns: name without suffix
+    to_rename =[]
+    for col in columns:
+        if col + suffix in sumstats.columns:
+            to_rename.append(col)
+    sumstats = sumstats.rename({i + suffix:i for i in to_rename})
+    log.write(" -Renaming sumstats columns by removing suffix {}".format(suffix),verbose=verbose)
+    return sumstats
+
+def _sort_pair_colsp(molded_sumstats, verbose=True, log=Log(), order=None, stats_order=None,suffixes=("_1","_2")):
+    if stats_order is None:
+        order = ["SNPID","rsID", "CHR", "POS", "EA", "NEA"]
+        stats_order = ["EAF", "MAF", "BETA", "SE","BETA_95L","BETA_95U", "Z",
+        "CHISQ", "P", "MLOG10P", "OR", "OR_95L", "OR_95U","HR", "HR_95L", "HR_95U","INFO", "N","N_CASE","N_CONTROL","DIRECTION","I2","P_HET","DOF","SNPR2","STATUS"]
+        
+    for suffix in suffixes:
+        for i in stats_order:
+            order.append(i+suffix)
+    
+    log.write("Start to reorder the columns...",verbose=verbose)
+    
+    output_columns = []
+    
+    for i in order:
+        if i in molded_sumstats.columns: 
+            output_columns.append(i)
+    for i in molded_sumstats.columns:
+        if i not in order: 
+            output_columns.append(i)
+    
+    log.write(" -Reordering columns to    :", ",".join(output_columns), verbose=verbose)
+    molded_sumstats = molded_sumstats[ output_columns]
+    log.write("Finished sorting columns successfully!", verbose=verbose)
+    
+    return molded_sumstats

gwaslab/hm_harmonize_sumstats.py

@@ -17,6 +17,7 @@ from gwaslab.qc_fix_sumstats import check_col
 from gwaslab.qc_fix_sumstats import start_to
 from gwaslab.qc_fix_sumstats import finished
 from gwaslab.qc_fix_sumstats import skipped
+from gwaslab.qc_fix_sumstats import sortcoordinate
 from gwaslab.qc_check_datatype import check_dataframe_shape
 from gwaslab.bd_common_data import get_number_to_chr
 from gwaslab.bd_common_data import get_chr_list
@@ -397,7 +398,6 @@ def oldcheckref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status=
 def _fast_check_status(x: pd.DataFrame, record: np.array, starting_positions: np.array, records_len: np.array):
     # starting_positions and records_len must be 1D arrays containing data only for the chromosomes contained in x,
     # and these arrays must be ordered in the same way as the chromosomes in np.unique(x['CHR'].values).
-
     # status 
     #0 /  ----->  match
     #1 /  ----->  Flipped Fixed
@@ -435,12 +435,13 @@ def _fast_check_status(x: pd.DataFrame, record: np.array, starting_positions: np
     _chrom = _chrom.values
     unique_values, _ = np.unique(_chrom, return_inverse=True) # Get the sorted unique values and their indices
     chrom = np.searchsorted(unique_values, _chrom) # Replace each value in '_chrom' with its corresponding index in the sorted unique values
-
+ 
     max_len_nea = _nea.str.len().max()
     max_len_ea = _ea.str.len().max()
 
     ########################################## mask for variants with out of range POS
     mask_outlier = pos > records_len[chrom]
+
     #########################################
 
     # Let's apply the same magic used for the fasta records (check build_fasta_records() for details) to convert the NEA and EA to
@@ -538,6 +539,7 @@ def _fast_check_status(x: pd.DataFrame, record: np.array, starting_positions: np
     #  -> nea == output_nea: [[True, True], [True, False]], mask: [[True, True], [True, False]]
     #  -> nea == output_nea + ~mask: [[True, True], [True, True]]
     #  -> np.all(nea == output_nea + ~mask, 1): [True, True]
+
     nea_eq_ref = np.all((nea == output_nea) + ~mask_nea, 1)
     rev_nea_eq_ref = np.all((rev_nea == output_nea) + ~mask_nea, 1)
 
@@ -550,6 +552,7 @@ def _fast_check_status(x: pd.DataFrame, record: np.array, starting_positions: np
     output_ea[mask_outlier] = PADDING_VALUE
     ##################################################################
 
+
     ea_eq_ref = np.all((ea == output_ea) + ~mask_ea, 1)
     rev_ea_eq_ref = np.all((rev_ea == output_ea) + ~mask_ea, 1)
 
@@ -617,6 +620,7 @@ def check_status(sumstats: pd.DataFrame, fasta_records_dict, log=Log(), verbose=
     unique_chrom_cond = sumstats_cond[chrom].unique()
     starting_pos_cond = np.array([starting_positions_dict[k] for k in unique_chrom_cond])
     records_len_cond = np.array([records_len_dict[k] for k in unique_chrom_cond])
+
     sumstats.loc[condition, status] = _fast_check_status(sumstats_cond, record=record, starting_positions=starting_pos_cond, records_len=records_len_cond)
 
     log.write(f"   -Checking records for ( len(NEA) > {max_len} or len(EA) > {max_len} )", verbose=verbose)
@@ -651,6 +655,8 @@ def checkref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="ST
     log.write(" -Loading fasta records:",end="", verbose=verbose)
     chromlist = get_chr_list(add_number=True)
     records = SeqIO.parse(ref_seq, "fasta")
+    
+    sumstats = sortcoordinate(sumstats,verbose=False)
 
     all_records_dict = {}
     chroms_in_sumstats = sumstats[chrom].unique() # load records from Fasta file only for the chromosomes present in the sumstats
@@ -729,17 +735,21 @@ def build_fasta_records(fasta_records_dict, pos_as_dict=True, log=Log(), verbose
         r = r.seq._data.translate(TRANSLATE_TABLE)
         r = np.array([r], dtype=f'<U{len(r)}').view('<u4').astype(np.uint8)
         all_r.append(r)
-
+    
     # We've just created a list of numpy arrays, so we can concatenate them to obtain a single numpy array
     # Then we keep track of the starting position of each record in the concatenated array. This will be useful later
     # to index the record array depending on the position of the variant and the chromosome
     records_len = np.array([len(r) for r in all_r])
+
     starting_positions = np.cumsum(records_len) - records_len
+
+    
     if pos_as_dict:
         starting_positions = {k: v for k, v in zip(fasta_records_dict.keys(), starting_positions)}
         records_len_dict =  {k: v for k, v in zip(fasta_records_dict.keys(), records_len)}
     record = np.concatenate(all_r)
     del all_r # free memory
+    
 
     return record, starting_positions,records_len_dict
 
@@ -1335,8 +1345,8 @@ def parallelinferstrand(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.40,
 ################################################################################################################
 def parallelecheckaf(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.4,column_name="DAF",suffix="",n_cores=1, chr="CHR",pos="POS",ref="NEA",alt="EA",eaf="EAF",status="STATUS",chr_dict=None,force=False, verbose=True,log=Log()):
     ##start function with col checking##########################################################
-    _start_line = "check the difference between EAF and reference VCF ALT frequency"
-    _end_line = "checking the difference between EAF and reference VCF ALT frequency"
+    _start_line = "check the difference between EAF (sumstats) and ALT frequency (reference VCF)"
+    _end_line = "checking the difference between EAF (sumstats) and ALT frequency (reference VCF)"
     _start_cols = [chr,pos,ref,alt,eaf,status]
     _start_function = ".check_daf()"
     _must_args ={"ref_alt_freq":ref_alt_freq}
@@ -1381,7 +1391,8 @@ def parallelecheckaf(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.4,colu
         pool.join()
     ###########################
         #status_inferred = sumstats.loc[good_chrpos,[chr,pos,ref,alt,eaf]].apply(lambda x:check_daf(x[0],x[1]-1,x[1],x[2],x[3],x[4],vcf_reader,ref_alt_freq,chr_dict),axis=1)
-        
+        log.write(" -Difference in allele frequency (DAF) = EAF (sumstats) - ALT_AF (reference VCF)", verbose=verbose)  
+        log.write(" -Note: this DAF is not the derived allele frequency.", verbose=verbose)  
         #sumstats.loc[good_chrpos,"DAF"] = status_inferred.values
         #sumstats["DAF"]=sumstats["DAF"].astype("float")     
         log.write(" - {} max:".format(column_name), np.nanmax(sumstats[column_name]),verbose=verbose)
@@ -1417,8 +1428,8 @@ def check_daf(chr,start,end,ref,alt,eaf,vcf_reader,alt_freq,chr_dict=None):
 
 def paralleleinferaf(sumstats,ref_infer,ref_alt_freq=None,n_cores=1, chr="CHR",pos="POS",ref="NEA",alt="EA",eaf="EAF",status="STATUS",chr_dict=None,force=False, verbose=True,log=Log()):
     ##start function with col checking##########################################################
-    _start_line = "infer EAF using reference VCF ALT frequency"
-    _end_line = "inferring EAF using reference VCF ALT frequency"
+    _start_line = "infer sumstats EAF using reference VCF ALT frequency"
+    _end_line = "inferring sumstats EAF using reference VCF ALT frequency"
     _start_cols = [chr,pos,ref,alt,status]
     _start_function = ".infer_af()"
     _must_args ={"ref_alt_freq":ref_alt_freq}