gwaslab 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

gwaslab/hm_harmonize_sumstats.py

@@ -27,7 +27,7 @@ from gwaslab.bd_common_data import _maketrans
 from gwaslab.g_vchange_status import vchange_status
 from gwaslab.g_version import _get_version
 from gwaslab.cache_manager import CacheManager, PALINDROMIC_INDEL, NON_PALINDROMIC
-
+from gwaslab.g_vchange_status import STATUS_CATEGORIES
 #rsidtochrpos
 #checkref
 #parallelizeassignrsid
@@ -357,10 +357,7 @@ def oldcheckref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status=
     
     log.write("\n",end="",show_time=False,verbose=verbose) 
         
-    CATEGORIES = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
-    sumstats[status] = pd.Categorical(sumstats[status],categories=CATEGORIES)
-    #sumstats[status] = sumstats[status].astype("string")
-
+    sumstats[status] = pd.Categorical(sumstats[status],categories=STATUS_CATEGORIES)
 
     available_to_check =sum( (~sumstats[pos].isna()) & (~sumstats[nea].isna()) & (~sumstats[ea].isna()))
     status_0=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[0]\w", case=False, flags=0, na=False))
@@ -681,9 +678,7 @@ def checkref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="ST
         sumstats.loc[to_check_ref,status] = check_status(sumstats_to_check, all_records_dict, log=log, verbose=verbose)
         log.write(" -Finished checking records", verbose=verbose) 
     
-    CATEGORIES = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
-    sumstats[status] = pd.Categorical(sumstats[status],categories=CATEGORIES)
-    #sumstats[status] = sumstats[status].astype("string")
+    sumstats[status] = pd.Categorical(sumstats[status],categories=STATUS_CATEGORIES)
 
     available_to_check =sum( (~sumstats[pos].isna()) & (~sumstats[nea].isna()) & (~sumstats[ea].isna()))
     status_0=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[0]\w", case=False, flags=0, na=False))

gwaslab/{io_load_ld.py → io/io_load_ld.py}

@@ -1,8 +1,6 @@
-
 import scipy.sparse as sparse
 import numpy as np
 import pandas as pd
-from gwaslab.hm_casting import _merge_mold_with_sumstats_by_chrpos
 import subprocess
 import os
 import re
@@ -10,16 +8,22 @@ import gc
 import pandas as pd
 import numpy as np
 from gwaslab.g_Log import Log
-from gwaslab.qc_fix_sumstats import start_to
-from gwaslab.qc_fix_sumstats import finished
-from gwaslab.util_in_get_sig import getsig
-from gwaslab.util_ex_process_ref import _process_plink_input_files
 from gwaslab.g_version import _checking_plink_version
-from gwaslab.util_in_filter_value import _exclude_hla
-from gwaslab.util_ex_calculate_ldmatrix import _extract_variants_in_locus
-from gwaslab.util_ex_calculate_ldmatrix import _export_snplist_and_locus_sumstats
-from gwaslab.viz_plot_regional2 import _get_lead_id
-from gwaslab.util_ex_calculate_ldmatrix import _extract_variants_in_locus
+
+from gwaslab.hm.hm_casting import _merge_mold_with_sumstats_by_chrpos
+
+from gwaslab.qc.qc_fix_sumstats import start_to
+from gwaslab.qc.qc_fix_sumstats import finished
+
+from gwaslab.util.util_in_get_sig import getsig
+from gwaslab.util.util_ex_process_ref import _process_plink_input_files
+from gwaslab.util.util_in_filter_value import _exclude_hla
+from gwaslab.util.util_ex_calculate_ldmatrix import _extract_variants_in_locus
+from gwaslab.util.util_ex_calculate_ldmatrix import _export_snplist_and_locus_sumstats
+from gwaslab.util.util_ex_calculate_ldmatrix import _extract_variants_in_locus
+
+from gwaslab.viz.viz_plot_regional2 import _get_lead_id
+
 
 def tofinemapping_using_ld(sumstats, 
                   study=None, 
@@ -72,7 +76,7 @@ def tofinemapping_using_ld(sumstats,
         sig_df = getsig(sumstats,id="SNPID",chrom="CHR",pos="POS",p="P"+suffixes[0],**getlead_args)
     else:
         sig_df = sumstats.loc[sumstats["SNPID"].isin(loci),:]
-
+    log.write(" -Number of loci: {}...".format(len(sig_df)),verbose=verbose)
     # Drop duplicate!!!!
     log.write(" -Dropping duplicated SNPIDs...",verbose=verbose)
     sumstats = sumstats.drop_duplicates(subset=["SNPID"]).copy()
@@ -447,7 +451,6 @@ def _merge_ld_map_with_sumstats(row,
     # matching by SNPID
     # preserve bim keys (use intersection of keys from both frames, similar to a SQL inner join; preserve the order of the left keys.)
     combined_df = pd.merge(ld_map, locus_sumstats, on=["CHR","POS"],how="inner")
-    
     # match allele
     perfect_match =  ((combined_df["EA"] == combined_df["EA_bim"]) & (combined_df["NEA"] == combined_df["NEA_bim"]) ) 
     log.write("   -Variants with perfect matched alleles:{}".format(sum(perfect_match)))

gwaslab/{io_preformat_input.py → io/io_preformat_input.py}

@@ -3,14 +3,16 @@ import numpy as np
 import scipy.stats as ss
 import gzip
 import os
+import re
 import gc
-from gwaslab.bd_common_data import get_format_dict
-from gwaslab.qc_fix_sumstats import sortcolumn
-from gwaslab.qc_fix_sumstats import _process_build
-from gwaslab.qc_check_datatype import check_datatype
-from gwaslab.qc_check_datatype import quick_convert_datatype
-from gwaslab.qc_check_datatype import check_dataframe_memory_usage
+from gwaslab.bd.bd_common_data import get_format_dict
+from gwaslab.qc.qc_fix_sumstats import sortcolumn
+from gwaslab.qc.qc_fix_sumstats import _process_build
+from gwaslab.qc.qc_check_datatype import check_datatype
+from gwaslab.qc.qc_check_datatype import quick_convert_datatype
+from gwaslab.qc.qc_check_datatype import check_dataframe_memory_usage
 from gwaslab.g_headers import _check_overlap_with_reserved_keys
+from gwaslab.g_vchange_status import STATUS_CATEGORIES
 #20221030
 def preformat(sumstats,
           fmt=None,
@@ -122,62 +124,22 @@ def preformat(sumstats,
         
         if "sep" not in readargs.keys():
             readargs["sep"] = "\t"
+    else:
+        meta_data = None
         
 #########################################################################################################################################################      
     
-    # check chr-separated path / vcf / then print header.            
-    try:
-        if type(sumstats) is str:
-            ## loading data from path #################################################
-            inpath = sumstats
-            ###load sumstats by each chromosome #################################################  
-            if "@" in inpath:
-                log.write(" -Detected @ in path: load sumstats by each chromosome...",verbose=verbose)
-                inpath_chr_list=[]
-                inpath_chr_num_list=[]
-                for chromosome in list(range(1,26))+["x","y","X","Y","MT","mt","m","M"]:
-                    inpath_chr = inpath.replace("@",str(chromosome))  
-                    if isfile_casesensitive(inpath_chr):
-                        inpath_chr_num_list.append(str(chromosome))
-                        inpath_chr_list.append(inpath_chr)        
-                log.write(" -Chromosomes detected:",",".join(inpath_chr_num_list),verbose=verbose)
-                readargs_header = get_readargs_header(inpath = inpath_chr_list[0], readargs = readargs)
-                row_one = pd.read_table(inpath_chr_list[0],**readargs_header)
-                # columns in the sumstats
-                raw_cols = row_one.columns
-            else:
-            ##### loading data from tabular file#################################################
-                readargs_header = get_readargs_header(inpath = inpath, readargs = readargs)
-                row_one = pd.read_table(inpath,**readargs_header)
-                raw_cols = row_one.columns
-            
-            if fmt=="vcf":
-                # expanded
-                format_cols = list(row_one["FORMAT"].str.split(":"))[0]
-                # fixed + study1 + expanded
-                raw_cols = meta_data["format_fixed"] + [raw_cols[9]] + format_cols
+    # check chr-separated path / vcf / then print header.     
+    inpath, inpath_chr_list, inpath_chr_num_list, format_cols, raw_cols, usecols, dtype_dictionary  = check_path_and_header(sumstats, 
+                                                                                                                    fmt, 
+                                                                                                                    meta_data, 
+                                                                                                                    readargs, 
+                                                                                                                    usecols, 
+                                                                                                                    dtype_dictionary, 
+                                                                                                                    rename_dictionary, 
+                                                                                                                    log, 
+                                                                                                                    verbose)     
 
-            ###################################################################################### 
-        elif type(sumstats) is pd.DataFrame:
-            ## loading data from dataframe
-            raw_cols = sumstats.columns
-        
-        ################################################
-        for key,value in rename_dictionary.items():
-            # check avaiable keys  key->raw header
-            # usecols : a list of raw headers to load from file/DataFrame 
-            if key in raw_cols:
-                usecols.append(key)
-            if value in ["EA","NEA"]:
-                dtype_dictionary[key]="category"
-            if value in ["STATUS"]:
-                dtype_dictionary[key]="string"     
-            if value in ["CHR"]:
-                dtype_dictionary[key]="string"  
-    
-    except ValueError:
-        raise ValueError("Please input a path or a pd.DataFrame, and make sure the separator is correct and the columns you specified are in the file.")
-        
     ###################################################################################################################################################
     ## check columns/datatype to use 
     if snpid:
@@ -359,7 +321,7 @@ def preformat(sumstats,
     try:
         if type(sumstats) is str:
             ## loading data from path
-            inpath = sumstats
+            #inpath = sumstats
             if "@" in inpath:
                 log.write("Start to initialize gl.Sumstats from files with pattern :" + inpath,verbose=verbose)
                 sumstats_chr_list=[]
@@ -445,14 +407,14 @@ def preformat(sumstats,
         sumstats["N_CONTROL"] = ncontrol
     
     ### status ######################################################################################################
-    if status is None:
-        sumstats = process_status(sumstats=sumstats,build=build,log=log,verbose=verbose)
+    
+    sumstats = process_status(sumstats=sumstats,build=build,status=status,log=log,verbose=verbose)
     
     ## ea/nea, ref/alt ##############################################################################################
     sumstats = process_allele(sumstats=sumstats,log=log,verbose=verbose)
         
     ## NEAF to EAF ###########################################################################################################
-    if neaf is not None :
+    if neaf is not None or ("NEAF" in sumstats.columns and "EAF" not in sumstats.columns):
         sumstats = process_neaf(sumstats=sumstats,log=log,verbose=verbose)
 
     ## reodering ###################################################################################################  
@@ -562,9 +524,15 @@ def process_neaf(sumstats,log,verbose):
     log.write(" -NEAF is specified...",verbose=verbose) 
     pre_number=len(sumstats)
     log.write(" -Checking if 0<= NEAF <=1 ...",verbose=verbose) 
-    sumstats["EAF"] = pd.to_numeric(sumstats["EAF"], errors='coerce')
-    sumstats = sumstats.loc[(sumstats["EAF"]>=0) & (sumstats["EAF"]<=1),:]
-    sumstats["EAF"] = 1- sumstats["EAF"]
+    if "NEAF" in sumstats.columns:
+        sumstats["NEAF"] = pd.to_numeric(sumstats["NEAF"], errors='coerce')
+        sumstats = sumstats.loc[(sumstats["NEAF"]>=0) & (sumstats["NEAF"]<=1),:]
+        sumstats["EAF"] = 1- sumstats["NEAF"]
+        sumstats.drop(columns=["NEAF"], inplace=True)
+    else:
+        sumstats["EAF"] = pd.to_numeric(sumstats["EAF"], errors='coerce')
+        sumstats = sumstats.loc[(sumstats["EAF"]>=0) & (sumstats["EAF"]<=1),:]
+        sumstats["EAF"] = 1- sumstats["EAF"]
     log.write(" -Converted NEAF to EAF.",verbose=verbose) 
     after_number=len(sumstats)
     log.write(" -Removed "+str(pre_number - after_number)+" variants with bad NEAF.",verbose=verbose) 
@@ -599,13 +567,14 @@ def process_allele(sumstats,log,verbose):
         sumstats["NEA"]=sumstats["NEA"].astype("category")  
     return sumstats
 
-def process_status(sumstats,build,log,verbose):
-    log.write(" -Initiating a status column: STATUS ...",verbose=verbose)
-    #sumstats["STATUS"] = int(build)*(10**5) +99999
-    build = _process_build(build,log,verbose)
-    sumstats["STATUS"] = build +"99999"
-    categories = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
-    sumstats["STATUS"] = pd.Categorical(sumstats["STATUS"],categories=categories)
+def process_status(sumstats,build,status, log,verbose):
+    if status is None:
+        log.write(" -Initiating a status column: STATUS ...",verbose=verbose)
+        #sumstats["STATUS"] = int(build)*(10**5) +99999
+        build = _process_build(build,log,verbose)
+        sumstats["STATUS"] = build +"99999"
+
+    sumstats["STATUS"] = pd.Categorical(sumstats["STATUS"],categories=STATUS_CATEGORIES)
     return sumstats
 
 
@@ -649,4 +618,114 @@ def _load_variants_with_pattern(inpath,usecols,dtype_dictionary,readargs,rename_
     log.write(" -Loading only variants with pattern :  {} ...".format(snpid_pat),verbose=verbose)
     sumstats_filtered = pd.concat([chunk[chunk[chunk_snpid].str.match(snpid_pat, case=False,na=False) ] for chunk in sumstats_iter])
     log.write(" -Loaded {} variants with pattern : {} ...".format(len(sumstats_filtered), snpid_pat),verbose=verbose)
-    return sumstats_filtered
+    return sumstats_filtered
+
+
+def check_path_and_header(sumstats=None, 
+                          fmt=None, 
+                          meta_data=None, 
+                          readargs=None, 
+                          usecols=None, 
+                          dtype_dictionary=None, 
+                          rename_dictionary=None, 
+                          log=None, 
+                          verbose=None):
+    
+
+    if type(sumstats) is str:
+        ## loading data from path #################################################
+        inpath = sumstats
+        
+        try:
+            format_cols, raw_cols, inpath_chr_list, inpath_chr_num_list = process_inpath_and_load_header(inpath, fmt, meta_data,  readargs, log, verbose)
+        
+        except (FileNotFoundError, IndexError):
+            log.warning("Loading {} failed...Tesing if compressed/uncompressed...".format(inpath),verbose=verbose)
+            try:
+                if inpath[-3:]==".gz":
+                    inpath = inpath[:-3]
+                    log.write(" -Trying to load {}...".format(inpath),verbose=verbose)
+                    format_cols, raw_cols, inpath_chr_list, inpath_chr_num_list =process_inpath_and_load_header(inpath, fmt, meta_data,  readargs, log, verbose)
+                else:
+                    inpath = inpath+".gz"
+                    log.write(" -Trying to load {}...".format(inpath),verbose=verbose)
+                    format_cols, raw_cols, inpath_chr_list, inpath_chr_num_list = process_inpath_and_load_header(inpath, fmt, meta_data,  readargs, log, verbose)
+            except:
+                raise ValueError("Please input a valid path, and make sure the separator is correct and the columns you specified are in the file.") 
+
+        ###################################################################################### 
+    elif type(sumstats) is pd.DataFrame:
+        inpath = None
+        format_cols = None
+        inpath_chr_list = None
+        inpath_chr_num_list = None
+        ## loading data from dataframe
+        raw_cols = sumstats.columns
+
+    ################################################
+    for key,value in rename_dictionary.items():
+        # check avaiable keys  key->raw header
+        # usecols : a list of raw headers to load from file/DataFrame 
+        if key in raw_cols:
+            usecols.append(key)
+        if value in ["EA","NEA"]:
+            dtype_dictionary[key]="category"
+        if value in ["STATUS"]:
+            dtype_dictionary[key]="string"     
+        if value in ["CHR"]:
+            dtype_dictionary[key]="string"  
+
+    return inpath, inpath_chr_list, inpath_chr_num_list, format_cols, raw_cols, usecols, dtype_dictionary
+
+def process_inpath_and_load_header(inpath, fmt, meta_data,  readargs, log, verbose):
+    
+    format_cols = None
+    inpath_chr_list = None
+    inpath_chr_num_list = None
+
+    if "@" in inpath:
+        log.write(" -Detected @ in path: load sumstats by each chromosome...",verbose=verbose)
+        inpath_chr_list=[]
+        inpath_chr_num_list=[]
+        
+        # create a regex pattern for matching
+        pat = os.path.basename(inpath).replace("@","(\w+)")
+        
+        # get dir
+        dirname = os.path.dirname(inpath)
+        
+        # all files in the directory
+        files = os.listdir(dirname)
+        
+        files.sort()
+
+        for file in files:
+            # match
+            result = re.match(pat, file)
+            if result:
+                # get chr
+                chr_matched = str(result.group(1))
+                inpath_chr_num_list.append(chr_matched)
+                inpath_chr_list.append(inpath.replace("@",str(chr_matched))  )
+        
+        log.write(" -Chromosomes detected:",",".join(inpath_chr_num_list),verbose=verbose)
+
+        #if inpath_chr_list is empty-> IndexError
+        readargs_header = get_readargs_header(inpath = inpath_chr_list[0], readargs = readargs)
+        row_one = pd.read_table(inpath_chr_list[0],**readargs_header)
+        # columns in the sumstats
+        raw_cols = row_one.columns
+    else:
+    ##### loading data from tabular file#################################################
+    #if file not found, FileNotFoundError
+        readargs_header = get_readargs_header(inpath = inpath, readargs = readargs)
+        row_one = pd.read_table(inpath,**readargs_header)
+        raw_cols = row_one.columns
+
+    if fmt=="vcf":
+        # expanded
+        format_cols = list(row_one["FORMAT"].str.split(":"))[0]
+        # fixed + study1 + expanded
+        raw_cols = meta_data["format_fixed"] + [raw_cols[9]] + format_cols
+
+    return format_cols, raw_cols, inpath_chr_list, inpath_chr_num_list

gwaslab/{io_preformat_input_polars.py → io/io_preformat_input_polars.py}

@@ -5,12 +5,12 @@ import scipy.stats as ss
 import gzip
 import os
 import gc
-from gwaslab.bd_common_data import get_format_dict
-from gwaslab.qc_fix_sumstats import sortcolumn
-from gwaslab.qc_fix_sumstats import _process_build
-from gwaslab.qc_check_datatype_polars import check_datatype
-from gwaslab.qc_check_datatype_polars import quick_convert_datatype
-from gwaslab.qc_check_datatype_polars import check_dataframe_memory_usage
+from gwaslab.bd.bd_common_data import get_format_dict
+from gwaslab.qc.qc_fix_sumstats import sortcolumn
+from gwaslab.qc.qc_fix_sumstats import _process_build
+from gwaslab.qc.qc_check_datatype_polars import check_datatype_polars
+from gwaslab.qc.qc_check_datatype_polars import quick_convert_datatype
+from gwaslab.qc.qc_check_datatype_polars import check_dataframe_memory_usage
 from gwaslab.g_headers import _check_overlap_with_reserved_keys
 #20221030
 def preformatp(sumstats,
@@ -433,7 +433,7 @@ def preformatp(sumstats,
     #sumstats = sortcolumn(sumstats=sumstats,log=log,verbose=verbose)    
     sumstats = quick_convert_datatype(sumstats,log=log,verbose=verbose)
     
-    check_datatype(sumstats,log=log,verbose=verbose)
+    check_datatype_polars(sumstats,log=log,verbose=verbose)
     #gc.collect()
     check_dataframe_memory_usage(sumstats,log=log,verbose=verbose)
     

gwaslab/{io_read_pipcs.py → io/io_read_pipcs.py}

@@ -1,7 +1,7 @@
 import pandas as pd
 from gwaslab.g_Log import Log
-from gwaslab.qc_check_datatype import check_datatype
-from gwaslab.qc_check_datatype import check_dataframe_memory_usage
+from gwaslab.qc.qc_check_datatype import check_datatype
+from gwaslab.qc.qc_check_datatype import check_dataframe_memory_usage
 import re
 import os
 

gwaslab/{io_read_tabular.py → io/io_read_tabular.py}

@@ -1,7 +1,7 @@
 import pandas as pd
-from gwaslab.bd_common_data import get_formats_list
+from gwaslab.bd.bd_common_data import get_formats_list
 from gwaslab.g_Log import Log
-from gwaslab.bd_common_data import get_format_dict
+from gwaslab.bd.bd_common_data import get_format_dict
 
 def _read_tabular(path, fmt, **kwargs):
     

gwaslab/{io_to_formats.py → io/io_to_formats.py}

@@ -7,16 +7,19 @@ from pysam import tabix_compress
 from pysam import tabix_index
 from datetime import datetime
 from datetime import date
-from gwaslab.io_preformat_input import print_format_info
-from gwaslab.bd_common_data import get_formats_list
 from gwaslab.g_Log import Log
-from gwaslab.bd_common_data import get_format_dict
-from gwaslab.bd_common_data import get_number_to_chr
 from gwaslab.g_version import gwaslab_info
-from gwaslab.bd_get_hapmap3 import gethapmap3
-from gwaslab.util_in_filter_value import _exclude_hla
-from gwaslab.util_in_filter_value import _exclude
-from gwaslab.util_in_filter_value import _extract
+
+from gwaslab.io.io_preformat_input import print_format_info
+
+from gwaslab.bd.bd_common_data import get_format_dict
+from gwaslab.bd.bd_common_data import get_number_to_chr
+from gwaslab.bd.bd_common_data import get_formats_list
+from gwaslab.bd.bd_get_hapmap3 import gethapmap3
+
+from gwaslab.util.util_in_filter_value import _exclude_hla
+from gwaslab.util.util_in_filter_value import _exclude
+from gwaslab.util.util_in_filter_value import _extract
 # to vcf
 # to fmt
     ## vcf

gwaslab/{io_to_pickle.py → io/io_to_pickle.py}

@@ -44,4 +44,19 @@ def load_data_from_pickle(path,usecols=None):
                 existing_cols.append(i)
         data = data.loc[:,existing_cols]
         gc.collect()
-    return data
+    return data
+
+def _offload(df,path,log):
+    with open(path, 'wb') as file:
+        pickle.dump(df, file)
+        log.write("Dumpping dataframe to : ", path)
+
+def _reload(path,log):
+    with open(path, 'rb') as file:
+        df =  pickle.load(file)
+        log.write("Loaded dataframe back from : ", path)
+    try:
+        os.remove(path)
+    except:
+        pass
+    return df

gwaslab/{qc_check_datatype_polars.py → qc/qc_check_datatype_polars.py}

@@ -56,7 +56,7 @@ dtype_dict ={
     'P_RANDOM' :[pl.Float64()]
     }
 
-def check_datatype(sumstats, verbose=True, log=Log()):
+def check_datatype_polars(sumstats, verbose=True, log=Log()):
     
     #try:
     headers = []
@@ -112,7 +112,7 @@ def quick_convert_datatype(sumstats, log, verbose):
                     pass
     return sumstats
 
-def check_dataframe_shape(sumstats, log, verbose):
+def check_dataframe_shape_polars(sumstats, log, verbose):
     memory_in_mb = sumstats.estimated_size(unit="mb") 
     try:
         log.write(" -Current Dataframe shape : {} x {} ; Memory usage: {:.2f} MB".format(len(sumstats),len(sumstats.columns),memory_in_mb), verbose=verbose)

gwaslab/{qc_fix_sumstats.py → qc/qc_fix_sumstats.py}

@@ -7,22 +7,29 @@ from multiprocessing import  Pool
 from liftover import get_lifter
 from liftover import ChainFile
 from functools import partial
+
 from gwaslab.g_vchange_status import vchange_status
 from gwaslab.g_vchange_status import status_match
 from gwaslab.g_vchange_status import change_status
 from gwaslab.g_Log import Log
-from gwaslab.bd_common_data import get_chr_to_number
-from gwaslab.bd_common_data import get_number_to_chr
-from gwaslab.bd_common_data import get_chr_list
-from gwaslab.qc_check_datatype import check_datatype
-from gwaslab.qc_check_datatype import check_dataframe_shape
-from gwaslab.qc_build import _process_build
-from gwaslab.qc_build import _set_build
 from gwaslab.g_version import _get_version
-from gwaslab.util_in_fill_data import _convert_betase_to_mlog10p
-from gwaslab.util_in_fill_data import _convert_betase_to_p
-from gwaslab.util_in_fill_data import _convert_mlog10p_to_p
-from gwaslab.bd_common_data import get_chain
+from gwaslab.g_vchange_status import STATUS_CATEGORIES
+
+from gwaslab.bd.bd_common_data import get_chr_to_number
+from gwaslab.bd.bd_common_data import get_number_to_chr
+from gwaslab.bd.bd_common_data import get_chr_list
+from gwaslab.bd.bd_common_data import get_chain
+from gwaslab.bd.bd_common_data import NA_STRINGS
+
+from gwaslab.qc.qc_check_datatype import check_datatype
+from gwaslab.qc.qc_check_datatype import check_dataframe_shape
+from gwaslab.qc.qc_build import _process_build
+from gwaslab.qc.qc_build import _set_build
+
+from gwaslab.util.util_in_fill_data import _convert_betase_to_mlog10p
+from gwaslab.util.util_in_fill_data import _convert_betase_to_p
+from gwaslab.util.util_in_fill_data import _convert_mlog10p_to_p
+
 #process build
 #setbuild
 #fixID
@@ -69,7 +76,7 @@ from gwaslab.bd_common_data import get_chain
 
 def fixID(sumstats,
        snpid="SNPID",rsid="rsID",chrom="CHR",pos="POS",nea="NEA",ea="EA",status="STATUS",fixprefix=False,
-       fixchrpos=False,fixid=False,fixeanea=False,fixeanea_flip=False,fixsep=False,
+       fixchrpos=False,fixid=False,fixeanea=False,fixeanea_flip=False,fixsep=False, reversea=False,
        overwrite=False,verbose=True,forcefixid=False,log=Log()):  
     '''
     1. fx SNPid
@@ -120,7 +127,21 @@ def fixID(sumstats,
         except:
             log.write(" -Force converting SNPID to pd.string data type...",verbose=verbose)
             sumstats[snpid] = sumstats[snpid].astype("string")
+    ############################  checking string NA ###################################################
+    log.write(" -Checking NA strings :{}".format(",".join(NA_STRINGS)),verbose=verbose)  
+    if snpid in sumstats.columns:  
+        log.write(" -Checking if SNPID contains NA strings...",verbose=verbose)
+        is_snpid_string_na = sumstats[snpid].isin(NA_STRINGS)
+        if sum(is_snpid_string_na) >0:
+            log.write("  -Converting {} NA strings in SNPID to pd.NA...".format(sum(is_snpid_string_na)),verbose=verbose)
+            sumstats.loc[is_snpid_string_na ,snpid] = pd.NA
 
+    if rsid in sumstats.columns: 
+        log.write(" -Checking if rsID contains NA strings...",verbose=verbose)
+        is_rsid_string_na = sumstats[rsid].isin(NA_STRINGS)
+        if sum(is_rsid_string_na) >0:
+            log.write("  -Converting {} NA strings in rsID to pd.NA...".format(sum(is_rsid_string_na)),verbose=verbose)
+            sumstats.loc[is_rsid_string_na ,rsid] = pd.NA
     ############################  checking ###################################################  
     if snpid in sumstats.columns:  
         log.write(" -Checking if SNPID is CHR:POS:NEA:EA...(separator: - ,: , _)",verbose=verbose)
@@ -148,7 +169,15 @@ def fixID(sumstats,
         log.write(" -A look at the unrecognized rsID :",set(sumstats.loc[(~is_rsid)&(~is_rs_chrpos),rsid].head()),"...", verbose=verbose) 
       
     ############################  fixing chr pos###################################################  
-    
+    if reversea == True:
+        if snpid in sumstats.columns: 
+            log.write(" -Reversing Alleles in SNPID...", verbose=verbose)
+            to_fix = is_chrposrefalt
+            to_fix_num = sum(to_fix)
+            if to_fix_num>0 and verbose: log.write(" -Number of variants could be reversed: "+str(to_fix_num)+" ...")
+            extracted = sumstats.loc[to_fix, snpid].str.extract(r'(^\w+[:_-]\d+[:_-])([ATCG]+)([:_-])([ATCG]+$)', flags=re.IGNORECASE)
+            sumstats.loc[to_fix, snpid] = extracted[0] + extracted[3] + extracted[2] + extracted[1]
+            
     if fixchrpos == True:
     # from snpid or rsid, extract CHR:POS to fix CHR and POS    
         if snpid in sumstats.columns: 
@@ -537,24 +566,24 @@ def removedup(sumstats,mode="dm",chrom="CHR",pos="POS",snpid="SNPID",ea="EA",nea
         pre_number =len(sumstats) 
         specified_columns = []
         if "d" in mode:
-            specified_columns.append(rsid)
-            specified_columns.append(snpid)
-            specified_columns.append(chrom)
-            specified_columns.append(pos)
-            specified_columns.append(ea)
-            specified_columns.append(nea)
+            if rsid in sumstats.columns: specified_columns.append(rsid)
+            if snpid in sumstats.columns: specified_columns.append(snpid)
+            if chrom in sumstats.columns: specified_columns.append(chrom)
+            if pos in sumstats.columns: specified_columns.append(pos)
+            if ea in sumstats.columns: specified_columns.append(ea)
+            if nea in sumstats.columns: specified_columns.append(nea)
         if "r" in mode:
-            specified_columns.append(rsid)
+            if rsid in sumstats.columns:specified_columns.append(rsid)
         if "s" in mode:
-            specified_columns.append(snpid)
+            if snpid in sumstats.columns:specified_columns.append(snpid)
         if "m" in mode:
-            specified_columns.append(chrom)
-            specified_columns.append(pos)
+            if chrom in sumstats.columns:specified_columns.append(chrom)
+            if pos in sumstats.columns:specified_columns.append(pos)
         if "c" in mode:
-            specified_columns.append(chrom)
-            specified_columns.append(pos)
-            specified_columns.append(ea)
-            specified_columns.append(nea)
+            if chrom in sumstats.columns:specified_columns.append(chrom)
+            if pos in sumstats.columns:specified_columns.append(pos)
+            if ea in sumstats.columns:specified_columns.append(ea)
+            if nea in sumstats.columns:specified_columns.append(nea)
         sumstats = sumstats.loc[~sumstats[specified_columns].isna().any(axis=1),:]
         after_number=len(sumstats) 
         log.write(" -Removed ",pre_number -after_number," variants with NA values in {} .".format(set(specified_columns)), verbose=verbose)  
@@ -1123,19 +1152,17 @@ def check_range(sumstats, var_range, header, coltocheck, cols_to_check, log, ver
         cols_to_check.append(header)
         if header=="STATUS": 
             log.write(" -Checking STATUS and converting STATUS to categories....", verbose=verbose) 
-            categories = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
-            sumstats[header] = pd.Categorical(sumstats[header],categories=categories)
+            sumstats[header] = pd.Categorical(sumstats[header],categories=STATUS_CATEGORIES)
             return sumstats
         
         if dtype in ["Int64","Int32","int","int32","in64"]:
             log.write(" -Checking if {} <= {} <= {} ...".format( var_range[0] ,header, var_range[1]), verbose=verbose) 
             sumstats[header] = np.floor(pd.to_numeric(sumstats[header], errors='coerce')).astype(dtype)
-        
+            is_valid = (sumstats[header]>=var_range[0]) & (sumstats[header]<=var_range[1])
         elif dtype in ["Float64","Float32","float","float64","float32"]:
             log.write(" -Checking if {} < {} < {} ...".format( var_range[0] ,header, var_range[1]),verbose=verbose) 
             sumstats[header] = pd.to_numeric(sumstats[header], errors='coerce').astype(dtype)
-        
-        is_valid = (sumstats[header]>=var_range[0]) & (sumstats[header]<=var_range[1])
+            is_valid = (sumstats[header]>var_range[0]) & (sumstats[header]<var_range[1])
         is_valid = is_valid.fillna(False)
 
         if header=="P":
@@ -1835,4 +1862,4 @@ def check_col(df_col_names, verbose=True, log=Log(), cols=None, function=None):
 ###############################################################################################################
 def _df_split(dataframe, n):
     k, m = divmod(len(dataframe), n)
-    return [dataframe.iloc[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n)]
+    return [dataframe.iloc[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n)]