gwaslab 3.4.37__py3-none-any.whl → 3.4.38__py3-none-any.whl

gwaslab/qc_fix_sumstats.py

@@ -47,14 +47,14 @@ def _process_build(build,log,verbose):
         log.write(" -Genomic coordinates are based on GRCh38/hg38...", verbose=verbose)
         final_build = "38"
     else:
-        log.write(" -WARNING! Version of genomic coordinates is unknown...", verbose=verbose)
+        log.warning("Version of genomic coordinates is unknown...", verbose=verbose)
         final_build = "99"
     return final_build
 
 def _set_build(sumstats, build="99", status="STATUS",verbose=True,log=Log()):
     build = _process_build(build,log=log,verbose=verbose)
-    sumstats.loc[:,status] = vchange_status(sumstats.loc[:,status], 1, "139",build[0]*3)
-    sumstats.loc[:,status] = vchange_status(sumstats.loc[:,status], 2, "89",build[1]*3)
+    sumstats[status] = vchange_status(sumstats[status], 1, "139",build[0]*3)
+    sumstats[status] = vchange_status(sumstats[status], 2, "89",build[1]*3)
     return sumstats, build
 
 def fixID(sumstats,
@@ -66,35 +66,49 @@ def fixID(sumstats,
     2. fix chr and pos using snpid
     3. checking rsid and chr:pos:nea:ea
     '''
-    if verbose: log.write("Start to check IDs...{}".format(_get_version()))
-    check_dataframe_shape(sumstats, log, verbose)
-    check_col(sumstats,[snpid,rsid],status)
+    ##start function with col checking##########################################################
+    _start_line = "check SNPID/rsID"
+    _end_line = "checking SNPID/rsID"
+    _start_cols =[]
+    _start_function = ".fix_id()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
 
     ############################  checking datatype ###################################################  
     if rsid in sumstats.columns:
         # convert to string datatype
         try:
             log.write(" -Checking rsID data type...",verbose=verbose)
-            if sumstats.loc[:,rsid].dtype == "string":
+            if sumstats[rsid].dtype == "string":
                 pass
             else:
                 log.write(" -Converting rsID to pd.string data type...",verbose=verbose)
-                sumstats.loc[:,rsid] = sumstats.loc[:,rsid].astype("string")
+                sumstats[rsid] = sumstats[rsid].astype("string")
         except:
             log.write(" -Force converting rsID to pd.string data type...",verbose=verbose)
-            sumstats.loc[:,rsid] = sumstats.loc[:,rsid].astype("string")
+            sumstats[rsid] = sumstats[rsid].astype("string")
     if snpid in sumstats.columns: 
         # convert to string datatype
         try:
             log.write(" -Checking SNPID data type...",verbose=verbose)
-            if sumstats.loc[:,snpid].dtype == "string":
+            if sumstats[snpid].dtype == "string":
                 pass
             else:
                 log.write(" -Converting SNPID to pd.string data type...",verbose=verbose)
-                sumstats.loc[:,snpid] = sumstats.loc[:,snpid].astype("string")
+                sumstats[snpid] = sumstats[snpid].astype("string")
         except:
             log.write(" -Force converting SNPID to pd.string data type...",verbose=verbose)
-            sumstats.loc[:,snpid] = sumstats.loc[:,snpid].astype("string")
+            sumstats[snpid] = sumstats[snpid].astype("string")
 
     ############################  checking ###################################################  
     if snpid in sumstats.columns:  
@@ -141,7 +155,7 @@ def fixID(sumstats,
             
             elif (chrom not in sumstats.columns) and (pos in sumstats.columns):
                 if verbose: log.write(" -Initiating CHR columns...")
-                sumstats.loc[:,chrom]=pd.Series(dtype="string")   
+                sumstats[chrom]=pd.Series(dtype="string")   
                 to_fix = is_chrposrefalt & sumstats[chrom].isna() & sumstats[pos].isna()
                 to_fix_num = sum(to_fix)
                 if to_fix_num>0 and verbose: log.write(" -Number of variants could be fixed: "+str(to_fix_num)+" ...")
@@ -149,7 +163,7 @@ def fixID(sumstats,
             
             elif (chrom in sumstats.columns) and (pos not in sumstats.columns):
                 if verbose: log.write(" -Initiating CHR and POS column...")
-                sumstats.loc[:,pos]=pd.Series(dtype="Int64") 
+                sumstats[pos]=pd.Series(dtype="Int64") 
                 to_fix = is_chrposrefalt & sumstats[chrom].isna() & sumstats[pos].isna() 
                 to_fix_num = sum(to_fix)
                 if to_fix_num>0 and verbose: log.write(" -Number of variants could be fixed: "+str(to_fix_num)+" ...")
@@ -157,8 +171,8 @@ def fixID(sumstats,
                 
             else:
                 if verbose: log.write(" -Initiating CHR and POS columns...")
-                sumstats.loc[:,chrom]=pd.Series(dtype="string")   
-                sumstats.loc[:,pos]=pd.Series(dtype="Int64") 
+                sumstats[chrom]=pd.Series(dtype="string")   
+                sumstats[pos]=pd.Series(dtype="Int64") 
                 to_fix = is_chrposrefalt
                 to_fix_num = sum(to_fix)
                 if to_fix_num>0 and verbose: log.write(" -Number of variants could be fixed: "+str(to_fix_num)+" ...")
@@ -187,20 +201,20 @@ def fixID(sumstats,
                 elif verbose: log.write(" -No fixable variants ...")
             elif (chrom not in sumstats.columns) and (pos in sumstats.columns):
                 if verbose: log.write(" -Initiating CHR columns...")
-                sumstats.loc[:,chrom]=pd.Series(dtype="string")   
+                sumstats[chrom]=pd.Series(dtype="string")   
                 to_fix = is_rs_chrpos & sumstats[chrom].isna() & sumstats[pos].isna()
                 if sum(to_fix)>0 and verbose: log.write(" -Number of variants could be fixed: "+str(sum(to_fix))+" ...")
                 elif verbose: log.write(" -No fixable variants ...")
             elif (chrom in sumstats.columns) and (pos not in sumstats.columns):
                 if verbose: log.write(" -Initiating CHR and POS column...")
-                sumstats.loc[:,pos]=pd.Series(dtype="Int64") 
+                sumstats[pos]=pd.Series(dtype="Int64") 
                 to_fix = is_rs_chrpos & sumstats[chrom].isna() & sumstats[pos].isna() 
                 if sum(to_fix)>0 and verbose: log.write(" -Number of variants could be fixed: "+str(sum(to_fix))+" ...")
                 elif verbose: log.write(" -No fixable variants ...")
             else:
                 if verbose: log.write(" -Initiating CHR and POS columns...")
-                sumstats.loc[:,chrom]=pd.Series(dtype="string")   
-                sumstats.loc[:,pos]=pd.Series(dtype="Int64") 
+                sumstats[chrom]=pd.Series(dtype="string")   
+                sumstats[pos]=pd.Series(dtype="Int64") 
                 to_fix = is_rs_chrpos
                 if sum(to_fix)>0 and verbose: log.write(" -Number of variants could be fixed: "+str(sum(to_fix))+" ...")
                 elif verbose: log.write(" -No fixable variants ...")   
@@ -214,7 +228,7 @@ def fixID(sumstats,
                 
     ############################  fixing chr pos###################################################   
     if fixeanea == True:
-        if verbose: log.write(" -WARNING! gwaslab assumes SNPID is in the format of CHR:POS:NEA:EA / CHR:POS:REF:ALT")
+        if verbose: log.warning("gwaslab assumes SNPID is in the format of CHR:POS:NEA:EA / CHR:POS:REF:ALT")
         if overwrite is True:
             if verbose: log.write(" -Overwrite mode is applied...")
             to_fix = is_chrposrefalt
@@ -223,12 +237,12 @@ def fixID(sumstats,
             if sum(to_fix)>0 and verbose: log.write(" -Number of variants could be fixed: "+str(sum(to_fix))+" ...")
         elif (nea in sumstats.columns) and (ea not in sumstats.columns):
             if verbose: log.write(" -Initiating EA columns...")
-            sumstats.loc[:,ea]=pd.Series(dtype="string")
+            sumstats[ea]=pd.Series(dtype="string")
             to_fix = is_chrposrefalt&(sumstats[nea].isna()|sumstats[ea].isna())
             if sum(to_fix)>0 and verbose: log.write(" -Number of variants could be fixed: "+str(sum(to_fix))+" ...")
         elif (nea not in sumstats.columns) and (ea in sumstats.columns):
             if verbose: log.write(" -Initiating NEA columns...")
-            sumstats.loc[:,nea]=pd.Series(dtype="string")
+            sumstats[nea]=pd.Series(dtype="string")
             to_fix = is_chrposrefalt&(sumstats[nea].isna()|sumstats[ea].isna())
             if sum(to_fix)>0 and verbose: log.write(" -Number of variants could be fixed: "+str(sum(to_fix))+" ...")
         else:
@@ -260,21 +274,21 @@ def fixID(sumstats,
     if fixsep == True:
         if snpid in sumstats.columns: 
             if verbose: log.write(' -Replacing [_-] in SNPID with ":" ...')
-            sumstats.loc[:,snpid] = sumstats.loc[:,snpid].str.replace(r"[_-]",":",regex=True)
+            sumstats[snpid] = sumstats[snpid].str.replace(r"[_-]",":",regex=True)
     
     if fixprefix == True:
         if snpid in sumstats.columns: 
             if verbose: log.write(' -Removing /^chr/ in SNPID ...')
-            prefix_removed = sumstats.loc[:,snpid].str.extract(r'^(chr)?(\w+[:_-]\d+[:_-][ATCG]+[:_-][ATCG]+)$',flags=re.IGNORECASE|re.ASCII)[1]
+            prefix_removed = sumstats[snpid].str.extract(r'^(chr)?(\w+[:_-]\d+[:_-][ATCG]+[:_-][ATCG]+)$',flags=re.IGNORECASE|re.ASCII)[1]
             sumstats.loc[~prefix_removed.isna(),snpid] = prefix_removed[~prefix_removed.isna()]
 
     if fixid == True:
         if snpid not in sumstats.columns: 
         # initiate a SNPID column
-            sumstats.loc[:,snpid]=pd.Series(dtype="string")
+            sumstats[snpid]=pd.Series(dtype="string")
         
         if (rsid in sumstats.columns) and (sum(is_rs_chrpos)>0) :
-            sumstats.loc[:,snpid]= sumstats.loc[is_rs_chrpos,rsid]
+            sumstats[snpid]= sumstats.loc[is_rs_chrpos,rsid]
             
         if (chrom in sumstats.columns) and (pos in sumstats.columns):
             #only fix when CHR and POS is available
@@ -329,7 +343,8 @@ def fixID(sumstats,
             after_number=sum(sumstats[snpid].isna())
             if verbose: log.write(" -Fixed "+ str(pre_number - after_number) +" variants ID...")
         elif verbose: log.write(" -ID unfixable: no CHR and POS columns or no SNPID. ")
-    if verbose: log.write("Finished checking IDs successfully!")
+
+    finished(log,verbose,_end_line)
     return sumstats
 
 ""
@@ -344,8 +359,25 @@ def removedup(sumstats,mode="dm",chrom="CHR",pos="POS",snpid="SNPID",ea="EA",nea
     remove duplicate SNPs based on 3. rsID
     remove multiallelic SNPs based on 4. CHR, POS
     '''
-    
-    if verbose: log.write("Start to remove duplicated/multiallelic variants...{}".format(_get_version()))
+
+    ##start function with col checking##########################################################
+    _start_line = "remove duplicated/multiallelic variants"
+    _end_line = "removing duplicated/multiallelic variants"
+    _start_cols =[]
+    _start_function = ".remove_dup()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+
     if verbose: log.write(" -Removing mode:{}".format(mode))
     # sort the variants using the specified column before removing
     if keep_col is not None : 
@@ -397,7 +429,7 @@ def removedup(sumstats,mode="dm",chrom="CHR",pos="POS",snpid="SNPID",ea="EA",nea
         if verbose: log.write("Start to remove multiallelic variants based on chr:pos...")    
         check_dataframe_shape(sumstats, log, verbose)
         if verbose: log.write(" -Which variant to keep: ",  keep ) 
-        sumstats = sumstats.loc[(~sumstats.loc[:,[chrom,pos]].all(axis=1)) | (~sumstats.duplicated(subset=[chrom,pos], keep=keep)),:]
+        sumstats = sumstats.loc[(~sumstats[[chrom,pos]].all(axis=1)) | (~sumstats.duplicated(subset=[chrom,pos], keep=keep)),:]
         after_number=len(sumstats)  
         if verbose:  log.write(" -Removed ",pre_number -after_number," multiallelic variants...")   
     after_number=len(sumstats)   
@@ -435,300 +467,346 @@ def removedup(sumstats,mode="dm",chrom="CHR",pos="POS",snpid="SNPID",ea="EA",nea
         sumstats = sumstats.loc[~sumstats[specified_columns].isna().any(axis=1),:]
         after_number=len(sumstats) 
         if verbose:  log.write(" -Removed ",pre_number -after_number," variants with NA values in {} .".format(set(specified_columns)))  
-    if verbose: log.write("Finished removing duplicated/multiallelic variants successfully!")
+
+    finished(log,verbose,_end_line)
     return sumstats
 
 ###############################################################################################################
 # 20230128
 def fixchr(sumstats,chrom="CHR",status="STATUS",add_prefix="",x=("X",23),y=("Y",24),mt=("MT",25), remove=False, verbose=True, chrom_list = None, minchr=1,log=Log()):
-        #chrom_list = get_chr_list() #bottom 
-        if chrom_list is None:
-            chrom_list = get_chr_list()
-        if check_col(sumstats,chrom,status) is not True:
-            if verbose: log.write(".fix_chr: Specified not detected..skipping...")
-            return sumstats
-        if verbose: log.write("Start to fix chromosome notation...{}".format(_get_version()))
-        check_dataframe_shape(sumstats, log, verbose)      
-        
-        # convert to string datatype
-        try:
-            if verbose: log.write(" -Checking CHR data type...")
-            if sumstats.loc[:,chrom].dtype == "string":
-                pass
-            else:
-                sumstats.loc[:,chrom] = sumstats.loc[:,chrom].astype("string")
-        except:
-            if verbose: log.write(" -Force converting to pd string data type...")
-            sumstats.loc[:,chrom] = sumstats.loc[:,chrom].astype("string")
-        
-        # check if CHR is numeric
-        is_chr_fixed = sumstats[chrom].str.isnumeric()
-        # fill NAs with False
-        is_chr_fixed = is_chr_fixed.fillna(False)
-        if verbose: log.write(" -Variants with standardized chromosome notation:",sum(is_chr_fixed))  
-        
-        # if there are variants whose CHR need to be fixed
-        if sum(is_chr_fixed)<len(sumstats):
-            
-            #extract the CHR number or X Y M MT
-            chr_extracted = sumstats.loc[~is_chr_fixed,chrom].str.extract(r'^(chr)?(\d{1,3}|[XYM]|MT)$',flags=re.IGNORECASE|re.ASCII)[1]
+    ##start function with col checking##########################################################
+    _start_line = "fix chromosome notation (CHR)"
+    _end_line = "fixing chromosome notation (CHR)"
+    _start_cols =[chrom,status]
+    _start_function = ".fix_chr()"
+    _must_args ={}
 
-            is_chr_fixable = ~chr_extracted.isna()
-            if verbose: log.write(" -Variants with fixable chromosome notations:",sum(is_chr_fixable))  
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
 
-            # For not fixed variants, check if na
-            is_chr_na  = sumstats.loc[~is_chr_fixed, chrom].isna()
-            if sum(is_chr_na)>0 and verbose: 
-                log.write(" -Variants with NA chromosome notations:",sum(is_chr_na))  
-            
-            # Check variants with CHR being not NA and not fixable
-            is_chr_invalid = (~is_chr_fixable)&(~is_chr_na)
-            if sum(is_chr_invalid)>0 and verbose: 
-                log.write(" -Variants with invalid chromosome notations:",sum(is_chr_invalid)) 
-                try:
-                    log.write(" -A look at invalid chromosome notations:" , set(sumstats.loc[~is_chr_fixed,chrom][is_chr_invalid].head()))
-                except:
-                    pass
-            elif verbose:
-                log.write(" -No unrecognized chromosome notations...")
-            
-            # Assign good chr back to sumstats
-            sumstats.loc[is_chr_fixable.index,chrom] = chr_extracted[is_chr_fixable.index]
+    #chrom_list = get_chr_list() #bottom 
+    if chrom_list is None:
+        chrom_list = get_chr_list()
+    #if check_col(sumstats,chrom,status) is not True:
+    #    if verbose: log.write(".fix_chr: Specified not detected..skipping...")
+    #    return sumstats
 
-            # X, Y, MT to 23,24,25
-            xymt_list = [x[0].lower(),y[0].lower(),mt[0].lower(),x[0].upper(),y[0].upper(),mt[0].upper()]
-            
-            # check if sumstats contain sex CHR
-            sex_chr = sumstats[chrom].isin(xymt_list)
-            
-            # if sumstats contain sex CHR
-            if sum(sex_chr)>0:
-                if verbose: log.write(" -Identifying non-autosomal chromosomes : {}, {}, and {} ...".format(x[0],y[0],mt[0]))
-                if verbose: log.write(" -Identified ",str(sum(sex_chr))," variants on sex chromosomes...")
-                
-                # convert "X, Y, MT" to numbers
-                convert_num_to_xymt={}
-                if x[0].lower() in sumstats[chrom].values or x[0].upper() in sumstats[chrom].values:
-                    convert_num_to_xymt[x[0].lower()] = str(x[1])
-                    convert_num_to_xymt[x[0].upper()] = str(x[1])
-                    if verbose: log.write(" -Standardizing sex chromosome notations: {} to {}...".format(x[0], x[1]))
-                if y[0].lower() in sumstats[chrom].values or y[0].upper() in sumstats[chrom].values:
-                    convert_num_to_xymt[y[0].lower()] = str(y[1])
-                    convert_num_to_xymt[y[0].upper()] = str(y[1])
-                    if verbose: log.write(" -Standardizing sex chromosome notations: {} to {}...".format(y[0], y[1]))
-                if mt[0].lower() in sumstats[chrom].values or mt[0].upper() in sumstats[chrom].values:
-                    convert_num_to_xymt[mt[0].lower()] = str(mt[1])
-                    convert_num_to_xymt[mt[0].upper()] = str(mt[1])
-                    if verbose: log.write(" -Standardizing sex chromosome notations: {} to {}...".format(mt[0], mt[1]))
-                sumstats.loc[sex_chr,chrom] =sumstats.loc[sex_chr,chrom].map(convert_num_to_xymt)
-            
-            # change status code
-            sumstats.loc[is_chr_fixed,status] = vchange_status(sumstats.loc[is_chr_fixed,status],4,"986","520")
-            if len(is_chr_fixable.index)>0:
-                sumstats.loc[is_chr_fixable.index,status] = vchange_status(sumstats.loc[is_chr_fixable.index,status],4,"986","520")
-            if len(is_chr_fixable.index)>0:
-                sumstats.loc[is_chr_invalid.index,status] = vchange_status(sumstats.loc[is_chr_invalid.index,status],4,"986","743")
-            
-            # check variants with unrecognized CHR
-            unrecognized_num = sum(~sumstats[chrom].isin(chrom_list))
-            if (remove is True) and unrecognized_num>0:
-                # remove variants with unrecognized CHR 
-                try:
-                    if verbose: log.write(" -Valid CHR list: {} - {}".format(min([int(x) for x in chrom_list if x.isnumeric()]),max([int(x) for x in chrom_list if x.isnumeric()])))
-                except:
-                    pass
-                if verbose: log.write(" -Removed "+ str(unrecognized_num)+ " variants with chromosome notations not in CHR list.") 
-                try:
-                    log.write(" -A look at chromosome notations not in CHR list:" , set(sumstats.loc[~sumstats[chrom].isin(chrom_list),chrom].head()))
-                except:
-                    pass
-                #sumstats = sumstats.loc[sumstats.index[sumstats[chrom].isin(chrom_list)],:]
-                good_chr = sumstats[chrom].isin(chrom_list)
-                sumstats = sumstats.loc[good_chr, :].copy()
+    
+    # convert to string datatype
+    try:
+        if verbose: log.write(" -Checking CHR data type...")
+        if sumstats[chrom].dtype == "string":
+            pass
         else:
-            if verbose: log.write(" -All CHR are already fixed...") 
-            sumstats.loc[is_chr_fixed,status] = vchange_status(sumstats.loc[is_chr_fixed,status],4,"986","520")
-        
-        # Convert string to int
-        try:
-            sumstats.loc[:,chrom] = sumstats.loc[:,chrom].astype('Int64')
-        except:
-            # force convert
-            sumstats.loc[:,chrom] = np.floor(pd.to_numeric(sumstats.loc[:,chrom], errors='coerce')).astype('Int64')
-        
-        # filter out variants with CHR <=0
-        out_of_range_chr = sumstats[chrom] < minchr
-        out_of_range_chr = out_of_range_chr.fillna(False)
-        if sum(out_of_range_chr)>0:
-            if verbose: log.write(" -Sanity check for CHR...") 
-            if verbose:log.write(" -Removed {} variants with CHR < {}...".format(sum(out_of_range_chr),minchr))
-            sumstats = sumstats.loc[~out_of_range_chr,:]
-
-        if verbose: log.write("Finished fixing chromosome notation successfully!")
+            sumstats[chrom] = sumstats[chrom].astype("string")
+    except:
+        if verbose: log.write(" -Force converting to pd string data type...")
+        sumstats[chrom] = sumstats[chrom].astype("string")
+    
+    # check if CHR is numeric
+    is_chr_fixed = sumstats[chrom].str.isnumeric()
+    # fill NAs with False
+    is_chr_fixed = is_chr_fixed.fillna(False)
+    if verbose: log.write(" -Variants with standardized chromosome notation:",sum(is_chr_fixed))  
+    
+    # if there are variants whose CHR need to be fixed
+    if sum(is_chr_fixed)<len(sumstats):
         
-        return sumstats
+        #extract the CHR number or X Y M MT
+        chr_extracted = sumstats.loc[~is_chr_fixed,chrom].str.extract(r'^(chr)?(\d{1,3}|[XYM]|MT)$',flags=re.IGNORECASE|re.ASCII)[1]
 
-###############################################################################################################    
-# 20230128
-def fixpos(sumstats,pos="POS",status="STATUS",remove=False, verbose=True, lower_limit=0 , upper_limit=None , limit=250000000, log=Log()):
-        if upper_limit is None:
-            upper_limit = limit
-        if check_col(sumstats,pos,status) is not True:
-            if verbose: log.write(".fix_pos: Specified not detected..skipping...")
-            return sumstats
-        if verbose: log.write("Start to fix basepair positions...{}".format(_get_version()))
-        check_dataframe_shape(sumstats, log, verbose)  
+        is_chr_fixable = ~chr_extracted.isna()
+        if verbose: log.write(" -Variants with fixable chromosome notations:",sum(is_chr_fixable))  
+
+        # For not fixed variants, check if na
+        is_chr_na  = sumstats.loc[~is_chr_fixed, chrom].isna()
+        if sum(is_chr_na)>0 and verbose: 
+            log.write(" -Variants with NA chromosome notations:",sum(is_chr_na))  
         
-        all_var_num = len(sumstats)
-        #convert to numeric
-        is_pos_na = sumstats.loc[:,pos].isna()
+        # Check variants with CHR being not NA and not fixable
+        is_chr_invalid = (~is_chr_fixable)&(~is_chr_na)
+        if sum(is_chr_invalid)>0 and verbose: 
+            log.write(" -Variants with invalid chromosome notations:",sum(is_chr_invalid)) 
+            try:
+                log.write(" -A look at invalid chromosome notations:" , set(sumstats.loc[~is_chr_fixed,chrom][is_chr_invalid].head()))
+            except:
+                pass
+        elif verbose:
+            log.write(" -No unrecognized chromosome notations...")
         
-        try:
-            if str(sumstats[pos].dtype) == "string" or str(sumstats[pos].dtype) == "object":
-                sumstats.loc[:,pos] = sumstats.loc[:,pos].astype('string')
-                # if so, remove thousands separator
-                if verbose: log.write(' -Removing thousands separator "," or underbar "_" ...')
-                sumstats.loc[~is_pos_na, pos] = sumstats.loc[~is_pos_na, pos].str.replace(r'[,_]', '' ,regex=True)
-        except:
-            pass
+        # Assign good chr back to sumstats
+        sumstats.loc[is_chr_fixable.index,chrom] = chr_extracted[is_chr_fixable.index]
 
-        # convert POS to integer
-        try:
-            if verbose: log.write(' -Converting to Int64 data type ...')
-            sumstats[pos] = sumstats[pos].astype('Int64')
-        except:
-            if verbose: log.write(' -Force converting to Int64 data type ...')
-            sumstats[pos] = np.floor(pd.to_numeric(sumstats[pos], errors='coerce')).astype('Int64')
-        is_pos_fixed = ~sumstats.loc[:,pos].isna()
-        is_pos_invalid = (~is_pos_na)&(~is_pos_fixed)
+        # X, Y, MT to 23,24,25
+        xymt_list = [x[0].lower(),y[0].lower(),mt[0].lower(),x[0].upper(),y[0].upper(),mt[0].upper()]
         
-        sumstats.loc[is_pos_fixed,status]   = vchange_status(sumstats.loc[is_pos_fixed,status]  ,4,"975","630")
-        sumstats.loc[is_pos_invalid,status] = vchange_status(sumstats.loc[is_pos_invalid,status],4,"975","842")
+        # check if sumstats contain sex CHR
+        sex_chr = sumstats[chrom].isin(xymt_list)
         
-        # remove outlier, limit:250,000,000
-        if verbose: log.write(" -Position bound:({} , {:,})".format(lower_limit, upper_limit))
-        is_pos_na = sumstats.loc[:,pos].isna()
-        out_lier= ((sumstats[pos]<=lower_limit) | (sumstats[pos]>=upper_limit)) & (~is_pos_na)
-        if verbose: log.write(" -Removed outliers:",sum(out_lier))
-        sumstats = sumstats.loc[~out_lier,:]
-        #remove na
-        if remove is True: 
-            sumstats = sumstats.loc[~sumstats[pos].isna(),:]
-            remain_var_num = len(sumstats)
-            if verbose: log.write(" -Removed "+str(all_var_num - remain_var_num)+" variants with bad positions.")        
+        # if sumstats contain sex CHR
+        if sum(sex_chr)>0:
+            if verbose: log.write(" -Identifying non-autosomal chromosomes : {}, {}, and {} ...".format(x[0],y[0],mt[0]))
+            if verbose: log.write(" -Identified ",str(sum(sex_chr))," variants on sex chromosomes...")
+            
+            # convert "X, Y, MT" to numbers
+            convert_num_to_xymt={}
+            if x[0].lower() in sumstats[chrom].values or x[0].upper() in sumstats[chrom].values:
+                convert_num_to_xymt[x[0].lower()] = str(x[1])
+                convert_num_to_xymt[x[0].upper()] = str(x[1])
+                if verbose: log.write(" -Standardizing sex chromosome notations: {} to {}...".format(x[0], x[1]))
+            if y[0].lower() in sumstats[chrom].values or y[0].upper() in sumstats[chrom].values:
+                convert_num_to_xymt[y[0].lower()] = str(y[1])
+                convert_num_to_xymt[y[0].upper()] = str(y[1])
+                if verbose: log.write(" -Standardizing sex chromosome notations: {} to {}...".format(y[0], y[1]))
+            if mt[0].lower() in sumstats[chrom].values or mt[0].upper() in sumstats[chrom].values:
+                convert_num_to_xymt[mt[0].lower()] = str(mt[1])
+                convert_num_to_xymt[mt[0].upper()] = str(mt[1])
+                if verbose: log.write(" -Standardizing sex chromosome notations: {} to {}...".format(mt[0], mt[1]))
+            sumstats.loc[sex_chr,chrom] =sumstats.loc[sex_chr,chrom].map(convert_num_to_xymt)
         
-        if verbose: log.write(" -Converted all position to datatype Int64.")
-        if verbose: log.write("Finished fixing basepair position successfully!")
+        # change status code
+        sumstats.loc[is_chr_fixed,status] = vchange_status(sumstats.loc[is_chr_fixed,status],4,"986","520")
+        if len(is_chr_fixable.index)>0:
+            sumstats.loc[is_chr_fixable.index,status] = vchange_status(sumstats.loc[is_chr_fixable.index,status],4,"986","520")
+        if len(is_chr_fixable.index)>0:
+            sumstats.loc[is_chr_invalid.index,status] = vchange_status(sumstats.loc[is_chr_invalid.index,status],4,"986","743")
         
-        return sumstats
+        # check variants with unrecognized CHR
+        unrecognized_num = sum(~sumstats[chrom].isin(chrom_list))
+        if (remove is True) and unrecognized_num>0:
+            # remove variants with unrecognized CHR 
+            try:
+                if verbose: log.write(" -Valid CHR list: {} - {}".format(min([int(x) for x in chrom_list if x.isnumeric()]),max([int(x) for x in chrom_list if x.isnumeric()])))
+            except:
+                pass
+            if verbose: log.write(" -Removed "+ str(unrecognized_num)+ " variants with chromosome notations not in CHR list.") 
+            try:
+                log.write(" -A look at chromosome notations not in CHR list:" , set(sumstats.loc[~sumstats[chrom].isin(chrom_list),chrom].head()))
+            except:
+                pass
+            #sumstats = sumstats.loc[sumstats.index[sumstats[chrom].isin(chrom_list)],:]
+            good_chr = sumstats[chrom].isin(chrom_list)
+            sumstats = sumstats.loc[good_chr, :].copy()
+    else:
+        if verbose: log.write(" -All CHR are already fixed...") 
+        sumstats.loc[is_chr_fixed,status] = vchange_status(sumstats.loc[is_chr_fixed,status],4,"986","520")
+    
+    # Convert string to int
+    try:
+        sumstats[chrom] = sumstats[chrom].astype('Int64')
+    except:
+    #    # force convert
+        sumstats[chrom] = np.floor(pd.to_numeric(sumstats[chrom], errors='coerce')).astype('Int64')
+    
+    # filter out variants with CHR <=0
+    out_of_range_chr = sumstats[chrom] < minchr
+    out_of_range_chr = out_of_range_chr.fillna(False)
+    if sum(out_of_range_chr)>0:
+        if verbose: log.write(" -Sanity check for CHR...") 
+        if verbose:log.write(" -Removed {} variants with CHR < {}...".format(sum(out_of_range_chr),minchr))
+        sumstats = sumstats.loc[~out_of_range_chr,:]
+
+    finished(log,verbose,_end_line)
+    return sumstats
+
+###############################################################################################################    
+# 20230128
+def fixpos(sumstats,pos="POS",status="STATUS",remove=False, verbose=True, lower_limit=0 , upper_limit=None , limit=250000000, log=Log()):
+    ##start function with col checking##########################################################
+    _start_line = "fix basepair positions (POS)"
+    _end_line = "fixing basepair positions (POS)"
+    _start_cols =[pos,status]
+    _start_function = ".fix_pos()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+
+    if upper_limit is None:
+        upper_limit = limit
+        
+    all_var_num = len(sumstats)
+    #convert to numeric
+    is_pos_na = sumstats[pos].isna()
+    
+    try:
+        if str(sumstats[pos].dtype) == "string" or str(sumstats[pos].dtype) == "object":
+            sumstats[pos] = sumstats[pos].astype('string')
+            # if so, remove thousands separator
+            if verbose: log.write(' -Removing thousands separator "," or underbar "_" ...')
+            sumstats.loc[~is_pos_na, pos] = sumstats.loc[~is_pos_na, pos].str.replace(r'[,_]', '' ,regex=True)
+    except:
+        pass
+
+    # convert POS to integer
+    try:
+        if verbose: log.write(' -Converting to Int64 data type ...')
+        sumstats[pos] = sumstats[pos].astype('Int64')
+    except:
+        if verbose: log.write(' -Force converting to Int64 data type ...')
+        sumstats[pos] = np.floor(pd.to_numeric(sumstats[pos], errors='coerce')).astype('Int64')
+    is_pos_fixed = ~sumstats[pos].isna()
+    is_pos_invalid = (~is_pos_na)&(~is_pos_fixed)
+    
+    sumstats.loc[is_pos_fixed,status]   = vchange_status(sumstats.loc[is_pos_fixed,status]  ,4,"975","630")
+    sumstats.loc[is_pos_invalid,status] = vchange_status(sumstats.loc[is_pos_invalid,status],4,"975","842")
+    
+    # remove outlier, limit:250,000,000
+    if verbose: log.write(" -Position bound:({} , {:,})".format(lower_limit, upper_limit))
+    is_pos_na = sumstats[pos].isna()
+    out_lier= ((sumstats[pos]<=lower_limit) | (sumstats[pos]>=upper_limit)) & (~is_pos_na)
+    if verbose: log.write(" -Removed outliers:",sum(out_lier))
+    sumstats = sumstats.loc[~out_lier,:]
+    #remove na
+    if remove is True: 
+        sumstats = sumstats.loc[~sumstats[pos].isna(),:]
+        remain_var_num = len(sumstats)
+        if verbose: log.write(" -Removed "+str(all_var_num - remain_var_num)+" variants with bad positions.")        
+ 
+    finished(log,verbose,_end_line)
+    return sumstats
 
 ###############################################################################################################    
 # 20220514
 def fixallele(sumstats,ea="EA", nea="NEA",status="STATUS",remove=False,verbose=True,log=Log()):
-        # remove variants with alleles other than actgACTG
-        if check_col(sumstats,ea,nea,status) is not True:
-            if verbose: log.write("EA and NEA not detected..skipping...")
-            return sumstats
-        if verbose: log.write("Start to fix alleles...{}".format(_get_version()))
-        check_dataframe_shape(sumstats, log, verbose)  
-        
-        #if (ea not in sumstats.columns) or (nea not in sumstats.columns):
-        if verbose: log.write(" -Converted all bases to string datatype and UPPERCASE.")
-        
-        #try:
-        #    ea_missing = sum(sumstats[ea].isna())
-        #    nea_missing = sum(sumstats[nea].isna())
-        #    if sum(ea_missing)>0:
-        #        if verbose: log.write(" -Converting {} missing EA to letter N.".format(ea_missing))
-        #        sumstats.loc[:,ea] = sumstats.loc[:,ea].add_categories("N").fillna("N")
-        #    if sum(sumstats[nea].isna())>0:
-        #        if verbose: log.write(" -Converting {} missing NEA to letter N.".format(nea_missing))
-        #        sumstats.loc[:,nea] = sumstats.loc[:,nea].add_categories("N").fillna("N")
-        #except:
-        #    pass
-
-        categories = set(sumstats.loc[:,ea].str.upper())|set(sumstats.loc[:,nea].str.upper())|set("N")
-        categories = {x for x in categories if pd.notna(x)}
-
-        sumstats.loc[:,ea]=pd.Categorical(sumstats[ea].str.upper(),categories = categories) 
-        sumstats.loc[:,nea]=pd.Categorical(sumstats[nea].str.upper(),categories = categories) 
-        all_var_num = len(sumstats)
-        
-        ## check ATCG
-        bad_ea = sumstats[ea].str.contains("[^actgACTG]",na=True)
-        bad_nea = sumstats[nea].str.contains("[^actgACTG]",na=True)
-        good_ea  = ~bad_ea
-        good_nea = ~bad_nea
-
-        log.write(" -Variants with bad EA  : {}".format(sum(bad_ea)), verbose=verbose)
-        log.write(" -Variants with bad NEA : {}".format(sum(bad_nea)), verbose=verbose)
-        
-        ## check NA
-        is_eanea_na = sumstats[ea].isna() |  sumstats[nea].isna()
-        log.write(" -Variants with NA for EA or NEA: {}".format(sum(is_eanea_na)), verbose=verbose)
-        
-        ## check same alleles
-        not_variant = sumstats[nea] == sumstats[ea]
-        log.write(" -Variants with same EA and NEA: {}".format(sum(not_variant)), verbose=verbose)
+    ##start function with col checking##########################################################
+    _start_line = "fix alleles (EA and NEA)"
+    _end_line = "fixing alleles (EA and NEA)"
+    _start_cols =[ea, nea,status]
+    _start_function = ".fix_allele()"
+    _must_args ={}
 
-        ## sum up invalid variants
-        is_invalid = bad_ea | bad_nea | not_variant
-        
-        exclude  = bad_nea | bad_ea
-        
-        if verbose: 
-            if len(set(sumstats.loc[bad_ea,ea].head())) >0:
-                log.write(" -A look at the non-ATCG EA:",set(sumstats.loc[bad_ea,ea].head()),"...") 
-            if len(set(sumstats.loc[bad_nea,nea].head())) >0:
-                log.write(" -A look at the non-ATCG NEA:",set(sumstats.loc[bad_nea,nea].head()),"...") 
-        
-        if remove == True:
-            sumstats = sumstats.loc[(good_ea & good_nea),:].copy()
-            good_eanea_num = len(sumstats)
-            if verbose: log.write(" -Removed "+str(all_var_num - good_eanea_num)+" variants with NA alleles or alleles that contain bases other than A/C/T/G.")  
-            sumstats = sumstats.loc[(good_ea & good_nea & (~not_variant)),:].copy()
-            good_eanea_notsame_num = len(sumstats)
-            if verbose: log.write(" -Removed "+str(good_eanea_num - good_eanea_notsame_num)+" variants with same allele for EA and NEA.") 
-        else:
-            sumstats.loc[:,[ea,nea]] = sumstats.loc[:,[ea,nea]].fillna("N")
-            if verbose: log.write(" -Detected "+str(sum(exclude))+" variants with alleles that contain bases other than A/C/T/G .") 
-        categories = set(sumstats.loc[:,ea].str.upper())|set(sumstats.loc[:,nea].str.upper())|set("N")
-        sumstats.loc[:,ea]=pd.Categorical(sumstats[ea].str.upper(),categories = categories) 
-        sumstats.loc[:,nea]=pd.Categorical(sumstats[nea].str.upper(),categories = categories) 
-        
-        is_eanea_fixed = good_ea | good_nea
-        is_snp = (sumstats[ea].str.len()==1) &(sumstats[nea].str.len()==1)
-        is_indel = (sumstats[ea].str.len()!=sumstats[nea].str.len())
-        is_not_normalized = (sumstats[ea].str.len()>1) &(sumstats[nea].str.len()>1)
-        is_normalized = is_indel &( (sumstats[ea].str.len()==1) &(sumstats[nea].str.len()>1) | (sumstats[ea].str.len()>1) &(sumstats[nea].str.len()==1) )
-        
-        if sum(is_invalid)>0:
-            sumstats.loc[is_invalid, status]                      = vchange_status(sumstats.loc[is_invalid,status],                5,"9","6") 
-        if sum(is_eanea_na)>0:
-            sumstats.loc[is_eanea_na,status]                      = vchange_status(sumstats.loc[is_eanea_na, status],              5,"9","7")
-        if sum(is_eanea_fixed&is_not_normalized)>0:
-            sumstats.loc[is_eanea_fixed&is_not_normalized,status] = vchange_status(sumstats.loc[is_eanea_fixed&is_not_normalized,status], 5,"9","5")
-        if sum(is_eanea_fixed&is_snp)>0:
-            sumstats.loc[is_eanea_fixed&is_snp, status]           = vchange_status(sumstats.loc[is_eanea_fixed&is_snp,status],        5,"9","0")
-        if sum(is_eanea_fixed&is_indel)>0:
-            sumstats.loc[is_eanea_fixed&is_indel,status]          = vchange_status(sumstats.loc[is_eanea_fixed&is_indel, status],      5,"9","4")
-        if sum(is_eanea_fixed&is_normalized)>0:
-            sumstats.loc[is_eanea_fixed&is_normalized,status]     = vchange_status(sumstats.loc[is_eanea_fixed&is_normalized, status],  5,"4","3")
-        gc.collect()
-        if verbose: log.write("Finished fixing allele successfully!")
-        
-        return sumstats
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+    #try:
+    #    ea_missing = sum(sumstats[ea].isna())
+    #    nea_missing = sum(sumstats[nea].isna())
+    #    if sum(ea_missing)>0:
+    #        if verbose: log.write(" -Converting {} missing EA to letter N.".format(ea_missing))
+    #        sumstats[ea] = sumstats[ea].add_categories("N").fillna("N")
+    #    if sum(sumstats[nea].isna())>0:
+    #        if verbose: log.write(" -Converting {} missing NEA to letter N.".format(nea_missing))
+    #        sumstats[nea] = sumstats[nea].add_categories("N").fillna("N")
+    #except:
+    #    pass
+
+    if verbose: log.write(" -Converted all bases to string datatype and UPPERCASE.")
+    categories = set(sumstats[ea].str.upper())|set(sumstats[nea].str.upper())|set("N")
+    categories = {x for x in categories if pd.notna(x)}
+    sumstats[ea]=pd.Categorical(sumstats[ea].str.upper(),categories = categories) 
+    sumstats[nea]=pd.Categorical(sumstats[nea].str.upper(),categories = categories) 
+    all_var_num = len(sumstats)
+    
+    ## check ATCG
+    bad_ea = sumstats[ea].str.contains("[^actgACTG]",na=True)
+    bad_nea = sumstats[nea].str.contains("[^actgACTG]",na=True)
+    good_ea  = ~bad_ea
+    good_nea = ~bad_nea
+
+    log.write(" -Variants with bad EA  : {}".format(sum(bad_ea)), verbose=verbose)
+    log.write(" -Variants with bad NEA : {}".format(sum(bad_nea)), verbose=verbose)
+    
+    ## check NA
+    is_eanea_na = sumstats[ea].isna() |  sumstats[nea].isna()
+    log.write(" -Variants with NA for EA or NEA: {}".format(sum(is_eanea_na)), verbose=verbose)
+    
+    ## check same alleles
+    not_variant = sumstats[nea] == sumstats[ea]
+    log.write(" -Variants with same EA and NEA: {}".format(sum(not_variant)), verbose=verbose)
+
+    ## sum up invalid variants
+    is_invalid = bad_ea | bad_nea | not_variant
+    
+    exclude  = bad_nea | bad_ea
+    
+    if verbose: 
+        if len(set(sumstats.loc[bad_ea,ea].head())) >0:
+            log.write(" -A look at the non-ATCG EA:",set(sumstats.loc[bad_ea,ea].head()),"...") 
+        if len(set(sumstats.loc[bad_nea,nea].head())) >0:
+            log.write(" -A look at the non-ATCG NEA:",set(sumstats.loc[bad_nea,nea].head()),"...") 
+    
+    if remove == True:
+        sumstats = sumstats.loc[(good_ea & good_nea),:].copy()
+        good_eanea_num = len(sumstats)
+        if verbose: log.write(" -Removed "+str(all_var_num - good_eanea_num)+" variants with NA alleles or alleles that contain bases other than A/C/T/G.")  
+        sumstats = sumstats.loc[(good_ea & good_nea & (~not_variant)),:].copy()
+        good_eanea_notsame_num = len(sumstats)
+        if verbose: log.write(" -Removed "+str(good_eanea_num - good_eanea_notsame_num)+" variants with same allele for EA and NEA.") 
+    else:
+        sumstats[[ea,nea]] = sumstats[[ea,nea]].fillna("N")
+        if verbose: log.write(" -Detected "+str(sum(exclude))+" variants with alleles that contain bases other than A/C/T/G .") 
+    categories = set(sumstats[ea].str.upper())|set(sumstats[nea].str.upper())|set("N")
+    sumstats[ea]=pd.Categorical(sumstats[ea].str.upper(),categories = categories) 
+    sumstats[nea]=pd.Categorical(sumstats[nea].str.upper(),categories = categories) 
+    
+    is_eanea_fixed = good_ea | good_nea
+    is_snp = (sumstats[ea].str.len()==1) &(sumstats[nea].str.len()==1)
+    is_indel = (sumstats[ea].str.len()!=sumstats[nea].str.len())
+    is_not_normalized = (sumstats[ea].str.len()>1) &(sumstats[nea].str.len()>1)
+    is_normalized = is_indel &( (sumstats[ea].str.len()==1) &(sumstats[nea].str.len()>1) | (sumstats[ea].str.len()>1) &(sumstats[nea].str.len()==1) )
+    
+    if sum(is_invalid)>0:
+        sumstats.loc[is_invalid, status]                      = vchange_status(sumstats.loc[is_invalid,status],                5,"9","6") 
+    if sum(is_eanea_na)>0:
+        sumstats.loc[is_eanea_na,status]                      = vchange_status(sumstats.loc[is_eanea_na, status],              5,"9","7")
+    if sum(is_eanea_fixed&is_not_normalized)>0:
+        sumstats.loc[is_eanea_fixed&is_not_normalized,status] = vchange_status(sumstats.loc[is_eanea_fixed&is_not_normalized,status], 5,"9","5")
+    if sum(is_eanea_fixed&is_snp)>0:
+        sumstats.loc[is_eanea_fixed&is_snp, status]           = vchange_status(sumstats.loc[is_eanea_fixed&is_snp,status],        5,"9","0")
+    if sum(is_eanea_fixed&is_indel)>0:
+        sumstats.loc[is_eanea_fixed&is_indel,status]          = vchange_status(sumstats.loc[is_eanea_fixed&is_indel, status],      5,"9","4")
+    if sum(is_eanea_fixed&is_normalized)>0:
+        sumstats.loc[is_eanea_fixed&is_normalized,status]     = vchange_status(sumstats.loc[is_eanea_fixed&is_normalized, status],  5,"4","3")
+
+    finished(log,verbose,_end_line)
+    return sumstats
 
 ###############################################################################################################   
 # 20220721
 
 def parallelnormalizeallele(sumstats,snpid="SNPID",rsid="rsID",pos="POS",nea="NEA",ea="EA" ,status="STATUS",n_cores=1,verbose=True,log=Log()):
-    if check_col(sumstats,pos,ea,nea,status) is not True:
-        if verbose: log.write("WARNING! .normalize(): specified columns not detected..skipping...")
-        return sumstats
-    
-    if verbose: log.write("Start to normalize variants...{}".format(_get_version()))
-    check_dataframe_shape(sumstats, log, verbose)  
+    ##start function with col checking##########################################################
+    _start_line = "normalize indels"
+    _end_line = "normalizing indels"
+    _start_cols =[ea, nea,status]
+    _start_function = ".normalize()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+
     #variants_to_check = status_match(sumstats[status],5,[4,5]) #
     #r'\w\w\w\w[45]\w\w'
     variants_to_check = sumstats[status].str[4].str.match(r'4|5', case=False, flags=0, na=False)
@@ -742,7 +820,8 @@ def parallelnormalizeallele(sumstats,snpid="SNPID",rsid="rsID",pos="POS",nea="NE
             n_cores=1  
         pool = Pool(n_cores)
         map_func = partial(normalizeallele,pos=pos,nea=nea,ea=ea,status=status)
-        df_split = np.array_split(sumstats.loc[variants_to_check,[pos,nea,ea,status]], n_cores)
+        #df_split = np.array_split(sumstats.loc[variants_to_check,[pos,nea,ea,status]], n_cores)
+        df_split = _df_split(sumstats.loc[variants_to_check,[pos,nea,ea,status]], n_cores)
         normalized_pd = pd.concat(pool.map(map_func,df_split))
         pool.close()
         pool.join()
@@ -772,16 +851,16 @@ def parallelnormalizeallele(sumstats,snpid="SNPID",rsid="rsID",pos="POS",nea="NE
         else:
             log.write(" -All variants are already normalized..")
     ###################################################################################################################
-    categories = set(sumstats.loc[:,ea])|set(sumstats.loc[:,nea]) |set(normalized_pd.loc[:,ea]) |set(normalized_pd.loc[:,nea])
-    sumstats.loc[:,ea]  = pd.Categorical(sumstats.loc[:,ea],categories = categories) 
-    sumstats.loc[:,nea] = pd.Categorical(sumstats.loc[:,nea],categories = categories ) 
+    categories = set(sumstats[ea])|set(sumstats[nea]) |set(normalized_pd.loc[:,ea]) |set(normalized_pd.loc[:,nea])
+    sumstats[ea]  = pd.Categorical(sumstats[ea],categories = categories) 
+    sumstats[nea] = pd.Categorical(sumstats[nea],categories = categories ) 
     sumstats.loc[variants_to_check,[pos,nea,ea,status]] = normalized_pd.values
     try:
-        sumstats.loc[:,pos] = sumstats.loc[:,pos].astype('Int64')
+        sumstats[pos] = sumstats[pos].astype('Int64')
     except:
-        sumstats.loc[:,pos] = np.floor(pd.to_numeric(sumstats.loc[:,pos], errors='coerce')).astype('Int64')
+        sumstats[pos] = np.floor(pd.to_numeric(sumstats[pos], errors='coerce')).astype('Int64')
   
-    if verbose: log.write("Finished normalizing variants successfully!")
+    finished(log,verbose,_end_line)
     return sumstats
 
 def normalizeallele(sumstats,pos="POS" ,nea="NEA",ea="EA",status="STATUS"):
@@ -846,6 +925,52 @@ def add_tolerence(stats, float_tolerence, mode):
         stats = (stats[0] , stats[1] + float_tolerence if stats[0]!=float("Inf") else float("Inf"))
     return stats
 
+
+def check_range(sumstats, var_range, header, coltocheck, cols_to_check, log, verbose, dtype="Int64"):
+    pre_number=len(sumstats)
+    if header in coltocheck and header in sumstats.columns:
+        cols_to_check.append(header)
+        if header=="STATUS": 
+            if verbose: log.write(" -Checking STATUS and converting STATUS to categories....") 
+            categories = {str(j+i) for j in [1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
+            sumstats[header] = pd.Categorical(sumstats[header],categories=categories)
+            return sumstats
+        
+        if dtype in ["Int64","Int32","int","int32","in64"]:
+            if verbose: log.write(" -Checking if {} <= {} <= {} ...".format( var_range[0] ,header, var_range[1])) 
+            sumstats[header] = np.floor(pd.to_numeric(sumstats[header], errors='coerce')).astype(dtype)
+        
+        elif dtype in ["Float64","Float32","float","float64","float32"]:
+            log.write(" -Checking if {} < {} < {} ...".format( var_range[0] ,header, var_range[1]),verbose=verbose) 
+            sumstats[header] = pd.to_numeric(sumstats[header], errors='coerce').astype(dtype)
+        
+        is_valid = (sumstats[header]>=var_range[0]) & (sumstats[header]<=var_range[1])
+        is_valid = is_valid.fillna(False)
+
+        if header=="P":
+            is_low_p =  sumstats["P"] == 0 
+            if sum(is_low_p) >0:
+                log.warning("Extremely low P detected (P=0 or P < minimum positive value of float64) : {}".format(sum(is_low_p)))
+                log.warning("Please consider using MLOG10P instead.")
+
+        if sum(~is_valid)>0:
+            try:
+                if "SNPID" in sumstats.columns:
+                    id_to_use = "SNPID"
+                elif "rsID" in sumstats.columns:
+                    id_to_use = "rsID"
+                invalid_ids = sumstats.loc[~is_valid, id_to_use].head().astype("string")
+                invalid_values = sumstats.loc[~is_valid, header].head().astype("string").fillna("NA")
+                log.write("  -Examples of invalid variants({}): {} ...".format(id_to_use, ",".join(invalid_ids.to_list()) ), verbose=verbose) 
+                log.write("  -Examples of invalid values ({}): {} ...".format(header, ",".join(invalid_values.to_list()) ), verbose=verbose) 
+            except:
+                pass
+
+        sumstats = sumstats.loc[is_valid,:]
+        after_number=len(sumstats)
+        log.write(" -Removed {} variants with bad/na {}.".format(pre_number - after_number, header), verbose=verbose) 
+    return sumstats
+
 def sanitycheckstats(sumstats,
                      coltocheck=None,
                      n=(0,2**31-1),
@@ -853,8 +978,10 @@ def sanitycheckstats(sumstats,
                      ncontrol=(0,2**31-1),
                      eaf=(0,1),
                      mac=(0,2**31-1),
+                     maf=(0,0.5),
                      chisq=(0,float("Inf")),
                      z=(-9999,9999),
+                     t=(-99999,99999),
                      f=(0,float("Inf")),
                      p=(0,1),
                      mlog10p=(0,9999),
@@ -885,10 +1012,30 @@ def sanitycheckstats(sumstats,
         HR_95U: float64  , HR_95L >0
         INFO:   float32  , 1>=INFO>0
         Z       float64  , -9999 < Z < 9999
+        T       float64  , -99999 < T < 99999
         F       float64  , F > 0 
     '''
+    ##start function with col checking##########################################################
+    _start_line = "perform sanity check for statistics"
+    _end_line = "sanity check for statistics"
+    _start_cols =[]
+    _start_function = ".check_sanity()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
 
+    if verbose: log.write(" -Comparison tolerance for floats: {}".format(float_tolerence)) 
     eaf = add_tolerence(eaf, float_tolerence, "lr")
+    maf = add_tolerence(maf, float_tolerence, "lr")
     beta = add_tolerence(beta, float_tolerence, "lr")
     se = add_tolerence(se, float_tolerence, "lr")
     mlog10p = add_tolerence(mlog10p, float_tolerence, "lr")
@@ -903,233 +1050,83 @@ def sanitycheckstats(sumstats,
     p = add_tolerence(p, float_tolerence, "lr")
     f = add_tolerence(f, float_tolerence, "lr")
     chisq = add_tolerence(chisq, float_tolerence, "lr")
-
-
+    ############################################################################################
     ## add direction
     if coltocheck is None:
         coltocheck = ["P","MLOG10P","INFO","Z","BETA","SE","EAF","CHISQ","F","N","N_CASE","N_CONTROL","OR","OR_95L","OR_95U","HR","HR_95L","HR_95U","STATUS"]
-    if verbose: log.write("Start sanity check for statistics...{}".format(_get_version())) 
-    check_dataframe_shape(sumstats, log, verbose)
+    
     cols_to_check=[]
     oringinal_number=len(sumstats)
     sumstats = sumstats.copy()
     
-    if verbose: log.write(" -Comparison tolerance for floats: {}".format(float_tolerence)) 
-    ###SAMPLE SIZE################################################################################################################################################
-    pre_number=len(sumstats)
-    if "N" in coltocheck and "N" in sumstats.columns:
-        cols_to_check.append("N")
-        if verbose: log.write(" -Checking if ",n[0],"<=N<=",n[1]," ...") 
-        sumstats.loc[:,"N"] = np.floor(pd.to_numeric(sumstats.loc[:,"N"], errors='coerce')).astype("Int64")
-        sumstats = sumstats.loc[(sumstats["N"]>=n[0]) & (sumstats["N"]<=n[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad N.") 
-    pre_number=len(sumstats)
-    if "N_CASE" in coltocheck and "N_CASE" in sumstats.columns:
-        cols_to_check.append("N_CASE")
-        if verbose: log.write(" -Checking if ",ncase[0],"<=N_CASE<=",ncase[1]," ...") 
-        sumstats.loc[:,"N_CASE"] = np.floor(pd.to_numeric(sumstats.loc[:,"N_CASE"], errors='coerce')).astype("Int64")
-        sumstats = sumstats.loc[(sumstats["N_CASE"]>=ncase[0]) & (sumstats["N_CASE"]<=ncase[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad N_CASE.") 
-    pre_number=len(sumstats)
-    if "N_CONTROL" in coltocheck and "N_CONTROL" in sumstats.columns:
-        cols_to_check.append("N_CONTROL")
-        if verbose: log.write(" -Checking if ",ncontrol[0],"<=N_CONTROL<=",ncontrol[1]," ...") 
-        sumstats.loc[:,"N_CONTROL"] = np.floor(pd.to_numeric(sumstats.loc[:,"N_CONTROL"], errors='coerce')).astype("Int64")
-        sumstats = sumstats.loc[(sumstats["N_CONTROL"]>=ncontrol[0]) & (sumstats["N_CONTROL"]<=ncontrol[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad N_CONTROL.") 
+    ###Int64 ################################################################################################################################################
+    sumstats = check_range(sumstats, var_range=n, header="N", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="Int64")
+    sumstats = check_range(sumstats, var_range=ncase, header="N_CASE", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="Int64")
+    sumstats = check_range(sumstats, var_range=ncontrol, header="N_CONTROL", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="Int64")
 
+    ###float32 ################################################################################################################################################
+    sumstats = check_range(sumstats, var_range=eaf, header="EAF", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float32")
+    sumstats = check_range(sumstats, var_range=maf, header="MAF", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float32")
+    sumstats = check_range(sumstats, var_range=info, header="INFO", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float32")
 
-    ###ALLELE FREQUENCY################################################################################################################################################
-    pre_number=len(sumstats)    
-    if "EAF" in coltocheck and "EAF" in sumstats.columns:
-        cols_to_check.append("EAF")
-        if verbose: log.write(" -Checking if ",eaf[0],"<EAF<",eaf[1]," ...") 
-        sumstats.loc[:,"EAF"] = pd.to_numeric(sumstats.loc[:,"EAF"], errors='coerce').astype("float32")
-        sumstats = sumstats.loc[(sumstats["EAF"]>eaf[0]) & (sumstats["EAF"]<eaf[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad EAF.") 
-    
-    pre_number=len(sumstats)    
-    if "EAF" in coltocheck and "EAF" in sumstats.columns and "N" in coltocheck and "N" in sumstats.columns:
-        if verbose: log.write(" -Checking if ",mac[0],"<=MAC<=",mac[1]," ...") 
-        sumstats["_MAF"]=sumstats["EAF"]
-        sumstats.loc[sumstats["EAF"]>0.5,"_MAF"] = 1 - sumstats.loc[sumstats["EAF"]>0.5,"EAF"]
-        sumstats["_MAC"] = np.floor(pd.to_numeric(sumstats.loc[:,"_MAF"] * sumstats.loc[:,"N"], errors='coerce')).astype("int64")
-        macl = ( sumstats["_MAC"] >= mac[0]) 
-        macu = ( sumstats["_MAC"] <= mac[1]) 
-        sumstats = sumstats.loc[macl&macu,:]
-        sumstats = sumstats.drop(labels=["_MAF","_MAC"],axis=1)
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad MAC.") 
-
-    ###TEST STATISTICS################################################################################################################################################
-    pre_number=len(sumstats)    
-    if "CHISQ" in coltocheck and "CHISQ" in sumstats.columns:
-        cols_to_check.append("CHISQ")
-        if verbose: log.write(" -Checking if ",chisq[0],"<CHISQ<",chisq[1]," ...") 
-        sumstats.loc[:,"CHISQ"] = pd.to_numeric(sumstats.loc[:,"CHISQ"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["CHISQ"]>chisq[0]) & (sumstats["CHISQ"]<chisq[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad CHISQ.") 
-    
-    pre_number=len(sumstats)    
-    if "Z" in coltocheck and "Z" in sumstats.columns:
-        cols_to_check.append("Z")
-        if verbose: log.write(" -Checking if ",z[0],"<Z<",z[1]," ...") 
-        sumstats.loc[:,"Z"] = pd.to_numeric(sumstats.loc[:,"Z"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["Z"]>z[0]) & (sumstats["Z"]<z[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad Z.") 
-    
-    pre_number=len(sumstats)    
-    if "F" in coltocheck and "F" in sumstats.columns:
-        cols_to_check.append("F")
-        if verbose: log.write(" -Checking if ",f[0],"<F<",f[1]," ...") 
-        sumstats.loc[:,"F"] = pd.to_numeric(sumstats.loc[:,"F"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["F"]>f[0]) & (sumstats["F"]<f[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad F.") 
-
-    ###P ################################################################################################################################################
-    pre_number=len(sumstats) 
-    if "P" in coltocheck and "P" in sumstats.columns:
-        cols_to_check.append("P")
-        if verbose: log.write(" -Checking if ",p[0],"< P <",p[1]," ...") 
-        sumstats.loc[:,"P"] = pd.to_numeric(sumstats.loc[:,"P"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["P"]>p[0]) & (sumstats["P"]<p[1]),:]
-        
-        is_low_p =  sumstats["P"] == 0 
-        if sum(is_low_p) >0:
-            log.write(" -WARNING! Extremely low P detected (P=0 or P < minimum positive value of float64) : {}".format(sum(is_low_p)), verbose=verbose)
-            log.write(" -WARNING! Please consider using MLOG10P instead.", verbose=verbose)
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad P.") 
-    
-    pre_number=len(sumstats)
-    if "MLOG10P" in coltocheck and "MLOG10P" in sumstats.columns:
-        cols_to_check.append("MLOG10P")
-        if verbose: log.write(" -Checking if ",mlog10p[0],"<MLOG10P<",mlog10p[1]," ...") 
-        sumstats.loc[:,"MLOG10P"] = pd.to_numeric(sumstats.loc[:,"MLOG10P"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["MLOG10P"]>mlog10p[0]) & (sumstats["MLOG10P"]<mlog10p[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad MLOG10P.") 
-    
-    ###EFFECT ################################################################################################################################################
-    pre_number=len(sumstats)    
-    if "BETA" in coltocheck and "BETA" in sumstats.columns:
-        cols_to_check.append("BETA")
-        if verbose: log.write(" -Checking if ",beta[0],"<BETA<",beta[1]," ...") 
-        sumstats.loc[:,"BETA"] = pd.to_numeric(sumstats.loc[:,"BETA"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["BETA"]>beta[0]) & (sumstats["BETA"]<beta[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad BETA.") 
-            
-    pre_number=len(sumstats)    
-    if "SE" in coltocheck and "SE" in sumstats.columns:
-        cols_to_check.append("SE")
-        if verbose: log.write(" -Checking if ",se[0],"<SE<",se[1]," ...") 
-        sumstats.loc[:,"SE"] = pd.to_numeric(sumstats.loc[:,"SE"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["SE"]>se[0]) & (sumstats["SE"]<se[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad SE.") 
-            
-    pre_number=len(sumstats)    
-    if "OR" in coltocheck and "OR" in sumstats.columns:
-        cols_to_check.append("OR")
-        if verbose: log.write(" -Checking if ",OR[0],"<log(OR)<",OR[1]," ...") 
-        sumstats.loc[:,"OR"] = pd.to_numeric(sumstats.loc[:,"OR"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(np.log(sumstats["OR"])>OR[0]) & (np.log(sumstats["OR"])<OR[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad OR.") 
-            
-    pre_number=len(sumstats)   
-    if "OR_95L" in coltocheck and "OR_95L" in sumstats.columns:
-        cols_to_check.append("OR_95L")
-        if verbose: log.write(" -Checking if ",OR_95L[0],"<OR_95L<",OR_95L[1]," ...") 
-        sumstats.loc[:,"OR_95L"] = pd.to_numeric(sumstats.loc[:,"OR_95L"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["OR_95L"]>OR_95L[0]) & (sumstats["OR_95L"]<OR_95L[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad OR_95L.") 
-            
-    pre_number=len(sumstats)    
-    if "OR_95U" in coltocheck and "OR_95U" in sumstats.columns:
-        cols_to_check.append("OR_95U")
-        if verbose: log.write(" -Checking if ",OR_95U[0],"<OR_95U<",OR_95U[1]," ...") 
-        sumstats.loc[:,"OR_95U"] = pd.to_numeric(sumstats.loc[:,"OR_95U"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["OR_95U"]>OR_95U[0]) & (sumstats["OR_95U"]<OR_95U[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad OR_95U.") 
-    
-    pre_number=len(sumstats)    
-    if "HR" in coltocheck and "HR" in sumstats.columns:
-        cols_to_check.append("HR")
-        if verbose: log.write(" -Checking if ",HR[0],"<log(HR)<",HR[1]," ...") 
-        sumstats.loc[:,"HR"] = pd.to_numeric(sumstats.loc[:,"HR"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(np.log(sumstats["HR"])>HR[0]) & (np.log(sumstats["HR"])<HR[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad HR.") 
-            
-    pre_number=len(sumstats)   
-    if "HR_95L" in coltocheck and "HR_95L" in sumstats.columns:
-        cols_to_check.append("HR_95L")
-        if verbose: log.write(" -Checking if ",HR_95L[0],"<HR_95L<",HR_95L[1]," ...") 
-        sumstats.loc[:,"HR_95L"] = pd.to_numeric(sumstats.loc[:,"HR_95L"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["HR_95L"]>HR_95L[0]) & (sumstats["HR_95L"]<HR_95L[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad HR_95L.") 
-            
-    pre_number=len(sumstats)    
-    if "HR_95U" in coltocheck and "HR_95U" in sumstats.columns:
-        cols_to_check.append("HR_95U")
-        if verbose: log.write(" -Checking if ",HR_95U[0],"<HR_95U<",HR_95U[1]," ...") 
-        sumstats.loc[:,"HR_95U"] = pd.to_numeric(sumstats.loc[:,"HR_95U"], errors='coerce').astype("float64")
-        sumstats = sumstats.loc[(sumstats["HR_95U"]>HR_95U[0]) & (sumstats["HR_95U"]<HR_95U[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad HR_95U.") 
-    #INFO #################################################################################################################
-    pre_number=len(sumstats)    
-    if "INFO" in coltocheck and "INFO" in sumstats.columns:
-        cols_to_check.append("INFO")
-        if verbose: log.write(" -Checking if ",info[0],"<INFO<",info[1]," ...") 
-        sumstats.loc[:,"INFO"] = pd.to_numeric(sumstats.loc[:,"INFO"], errors='coerce').astype("float32")
-        sumstats = sumstats.loc[(sumstats["INFO"]>info[0]) & (sumstats["INFO"]<info[1]),:]
-        after_number=len(sumstats)
-        if verbose: log.write(" -Removed "+str(pre_number - after_number)+" variants with bad INFO.") 
-    ###STATUS ################################################################################################################################################
-    pre_number=len(sumstats)    
-    if "STATUS" in coltocheck and "STATUS" in sumstats.columns:
-        cols_to_check.append("STATUS")
-        if verbose: log.write(" -Checking STATUS and converting STATUS to categories....") 
-        categories = {str(j+i) for j in [1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
-        sumstats.loc[:,"STATUS"] = pd.Categorical(sumstats["STATUS"],categories=categories)
-
-    #pre_number=len(sumstats)  
-    #sumstats = sumstats.dropna(subset=cols_to_check)
-    after_number=len(sumstats)
-    #if verbose:log.write(" -Removed {} variants with NAs in the checked columns...".format(pre_number - after_number))
+    ###float64 ################################################################################################################################################
+    sumstats = check_range(sumstats, var_range=chisq, header="CHISQ", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=z, header="Z", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=t, header="T", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=f, header="F", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=p, header="P", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=mlog10p, header="MLOG10P", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=beta, header="BETA", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=se, header="SE", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=OR, header="OR", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=OR_95L, header="OR_95L", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=OR_95U, header="OR_95U", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=HR, header="HR", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=HR_95L, header="HR_95L", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    sumstats = check_range(sumstats, var_range=HR_95U, header="HR_95U", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="float64")
+    ###STATUS ###############################################################################################################################################
+    sumstats = check_range(sumstats, var_range=None, header="STATUS", coltocheck=coltocheck, cols_to_check=cols_to_check, log=log, verbose=verbose, dtype="category")
 
-    if verbose: log.write(" -Removed "+str(oringinal_number - after_number)+" variants with bad statistics in total.") 
-    if verbose:
-        log.write(" -Data types for each column:")
-        check_datatype(sumstats,verbose=verbose, log=log)
-    if verbose: log.write("Finished sanity check successfully!")
+    after_number=len(sumstats)
+    log.write(" -Removed "+str(oringinal_number - after_number)+" variants with bad statistics in total.",verbose=verbose) 
+    log.write(" -Data types for each column:",verbose=verbose)
+    check_datatype(sumstats,verbose=verbose, log=log)
+    finished(log,verbose,_end_line)
     return sumstats
 
 ### check consistency #############################################################################################################################################
 
-def _check_data_consistency(sumstats, rtol=1e-3, atol=1e-3, equal_nan=True, verbose=True,log=Log()):
-    if verbose: log.write("Start to check data consistency across columns...{}".format(_get_version())) 
-    check_dataframe_shape(sumstats, log, verbose)
+def _check_data_consistency(sumstats, beta="BETA", se="SE", p="P",mlog10p="MLOG10P",rtol=1e-3, atol=1e-3, equal_nan=True, verbose=True,log=Log()):
+    ##start function with col checking##########################################################
+    _start_line = "check data consistency across columns"
+    _end_line = "checking data consistency across columns"
+    _start_cols =[]
+    _start_function = ".check_data_consistency()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+
     log.write(" -Tolerance: {} (Relative) and {} (Absolute)".format(rtol, atol),verbose=verbose)
+    check_status = 0
     
-    
-    if "SNPID" not in sumstats.columns:
+    if "SNPID" in sumstats.columns:
+        id_to_use = "SNPID"
+    elif "rsID" in sumstats.columns:
         id_to_use = "rsID"
     else:
-        id_to_use = "SNPID"
+        log.write(" -SNPID/rsID not available...SKipping",verbose=verbose)
+        log.write("Finished checking data consistency across columns.",verbose=verbose) 
+        return 0
+    
     
     if "BETA" in sumstats.columns and "SE" in sumstats.columns:
         if "MLOG10P" in sumstats.columns:
@@ -1138,10 +1135,11 @@ def _check_data_consistency(sumstats, rtol=1e-3, atol=1e-3, equal_nan=True, verb
             is_close = np.isclose(betase_derived_mlog10p, sumstats["MLOG10P"], rtol=rtol, atol=atol, equal_nan=equal_nan)
             diff = betase_derived_mlog10p - sumstats["MLOG10P"]
             if sum(~is_close)>0:
-                log.write("  -Not consistent: {} variant(s)".format(sum(~is_close),verbose=verbose))
-                log.write("  -Variant {} with max difference: {} with {}".format(id_to_use, sumstats.loc[diff.idxmax(),id_to_use], diff.max(),verbose=verbose))
+                log.write("  -Not consistent: {} variant(s)".format(sum(~is_close)),verbose=verbose)
+                log.write("  -Variant {} with max difference: {} with {}".format(id_to_use, sumstats.loc[diff.idxmax(),id_to_use], diff.max()),verbose=verbose)
             else:
                 log.write("  -Variants with inconsistent values were not detected." ,verbose=verbose)
+            check_status=1
         
         if "P" in sumstats.columns:
             log.write(" -Checking if BETA/SE-derived-P is consistent with P...",verbose=verbose)
@@ -1149,10 +1147,11 @@ def _check_data_consistency(sumstats, rtol=1e-3, atol=1e-3, equal_nan=True, verb
             is_close = np.isclose(betase_derived_p, sumstats["P"], rtol=rtol, atol=atol, equal_nan=equal_nan)
             diff = betase_derived_p - sumstats["P"]
             if sum(~is_close)>0:
-                log.write("  -Not consistent: {} variant(s)".format(sum(~is_close),verbose=verbose))
-                log.write("  -Variant {} with max difference: {} with {}".format(id_to_use, sumstats.loc[diff.idxmax(),id_to_use], diff.max(),verbose=verbose))
+                log.write("  -Not consistent: {} variant(s)".format(sum(~is_close)),verbose=verbose)
+                log.write("  -Variant {} with max difference: {} with {}".format(id_to_use, sumstats.loc[diff.idxmax(),id_to_use], diff.max()),verbose=verbose)
             else:
                 log.write("  -Variants with inconsistent values were not detected." ,verbose=verbose)
+            check_status=1
     
     if "MLOG10P" in sumstats.columns and "P" in sumstats.columns:
         log.write(" -Checking if MLOG10P-derived-P is consistent with P...",verbose=verbose)
@@ -1160,25 +1159,30 @@ def _check_data_consistency(sumstats, rtol=1e-3, atol=1e-3, equal_nan=True, verb
         is_close = np.isclose(mlog10p_derived_p, sumstats["P"], rtol=rtol, atol=atol, equal_nan=equal_nan)
         diff = mlog10p_derived_p - sumstats["P"]
         if sum(~is_close)>0:
-            log.write("  -Not consistent: {} variant(s)".format(sum(~is_close),verbose=verbose))
-            log.write("  -Variant {} with max difference: {} with {}".format(id_to_use, sumstats.loc[diff.idxmax(),id_to_use], diff.max(),verbose=verbose))
+            log.write("  -Not consistent: {} variant(s)".format(sum(~is_close)),verbose=verbose)
+            log.write("  -Variant {} with max difference: {} with {}".format(id_to_use, sumstats.loc[diff.idxmax(),id_to_use], diff.max()),verbose=verbose)
         else:
             log.write("  -Variants with inconsistent values were not detected." ,verbose=verbose)
+        check_status=1
 
     if "N" in sumstats.columns and "N_CONTROL" in sumstats.columns and "N_CASE" in sumstats.columns:
         if verbose: log.write(" -Checking if N is consistent with N_CASE + N_CONTROL ...") 
-        is_close = sumstats.loc[:,"N"] == sumstats.loc[:,"N_CASE"] + sumstats.loc[:,"N_CONTROL"] 
-        #is_close = np.isclose(sumstats.loc[:,"N"], sumstats.loc[:,"N_CASE"] + sumstats.loc[:,"N_CONTROL"] , rtol=rtol, atol=atol, equal_nan=equal_nan)
-        diff = abs(sumstats.loc[:,"N"] - (sumstats.loc[:,"N_CASE"] + sumstats.loc[:,"N_CONTROL"] ))
+        is_close = sumstats["N"] == sumstats["N_CASE"] + sumstats["N_CONTROL"] 
+        #is_close = np.isclose(sumstats["N"], sumstats["N_CASE"] + sumstats["N_CONTROL"] , rtol=rtol, atol=atol, equal_nan=equal_nan)
+        diff = abs(sumstats["N"] - (sumstats["N_CASE"] + sumstats["N_CONTROL"] ))
         if sum(~is_close)>0:
-            log.write("  -Not consistent: {} variant(s)".format(sum(~is_close),verbose=verbose))
-            log.write("  -Variant {} with max difference: {} with {}".format(id_to_use, sumstats.loc[diff.idxmax(),id_to_use], diff.max(),verbose=verbose))
+            log.write("  -Not consistent: {} variant(s)".format(sum(~is_close)),verbose=verbose)
+            log.write("  -Variant {} with max difference: {} with {}".format(id_to_use, sumstats.loc[diff.idxmax(),id_to_use], diff.max()),verbose=verbose)
         else:
             log.write("  -Variants with inconsistent values were not detected." ,verbose=verbose)
+        check_status=1
+        
+    if check_status==1:
+        log.write(" -Note: if the max difference is greater than expected, please check your original sumstats.",verbose=verbose)
+    else:
+        log.write(" -No availalbe columns for data consistency checking...Skipping...",verbose=verbose)
+    finished(log,verbose,_end_line)
 
-    log.write(" -Note: if the max difference is greater than expected, please check your original sumstats.",verbose=verbose)
-
-    if verbose: log.write("Finished checking data consistency across columns.") 
 ###############################################################################################################
 # 20220426
 def get_reverse_complementary_allele(a):
@@ -1201,11 +1205,81 @@ def flip_direction(string):
         else: #sometime it is 0
             flipped_string+=char
     return flipped_string
-    
+
+def flip_by_swap(sumstats, matched_index, log, verbose):
+    if ("NEA" in sumstats.columns) and ("EA" in sumstats.columns) :
+        if verbose: log.write(" -Swapping column: NEA <=> EA...") 
+        sumstats.loc[matched_index,['NEA','EA']] = sumstats.loc[matched_index,['EA','NEA']].values
+    return sumstats
+
+def flip_by_inverse(sumstats, matched_index, log, verbose, cols=None, factor=1):
+    if "OR" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: OR = 1 / OR...") 
+        sumstats.loc[matched_index,"OR"] =   factor / sumstats.loc[matched_index,"OR"].values
+    if "OR_95L" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: OR_95U = 1 / OR_95L...") 
+        sumstats.loc[matched_index,"OR_95U"] =   factor / sumstats.loc[matched_index,"OR_95L"].values
+    if "OR_95U" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: OR_95L = 1 / OR_95U...") 
+        sumstats.loc[matched_index,"OR_95L"] =   factor / sumstats.loc[matched_index,"OR_95U"].values
+    if "HR" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: HR = 1 / HR...") 
+        sumstats.loc[matched_index,"HR"] =   factor / sumstats.loc[matched_index,"HR"].values
+    if "HR_95L" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: HR_95U = 1 / HR_95L...") 
+        sumstats.loc[matched_index,"HR_95U"] =   factor / sumstats.loc[matched_index,"HR_95L"].values
+    if "HR_95U" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: HR_95L = 1 / HR_95U...") 
+        sumstats.loc[matched_index,"HR_95L"] =   factor / sumstats.loc[matched_index,"HR_95U"].values
+    return sumstats
+
+def flip_by_subtract(sumstats, matched_index, log, verbose, cols=None, factor=1):
+    if "EAF" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: EAF = 1 - EAF...") 
+        sumstats.loc[matched_index,"EAF"] =   factor - sumstats.loc[matched_index,"EAF"].values
+    return sumstats
+
+def flip_by_sign(sumstats, matched_index, log, verbose, cols=None):
+    if "BETA" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: BETA = - BETA...") 
+        sumstats.loc[matched_index,"BETA"] =     - sumstats.loc[matched_index,"BETA"].values
+    if "BETA_95L" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: BETA_95U = - BETA_95L...") 
+        sumstats.loc[matched_index,"BETA_95U"] =     - sumstats.loc[matched_index,"BETA_95L"].values
+    if "BETA_95U" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: BETA_95L = - BETA_95U...") 
+        sumstats.loc[matched_index,"BETA_95L"] =     - sumstats.loc[matched_index,"BETA_95U"].values
+    if "Z" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: Z = - Z...") 
+        sumstats.loc[matched_index,"Z"] =     - sumstats.loc[matched_index,"Z"].values
+    if "T" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: T = - T...") 
+        sumstats.loc[matched_index,"Z"] =     - sumstats.loc[matched_index,"T"].values
+    if "DIRECTION" in sumstats.columns:
+        if verbose: log.write(" -Flipping column: DIRECTION +-?0 <=> -+?0 ...") 
+        sumstats.loc[matched_index,"DIRECTION"] =   sumstats.loc[matched_index,"DIRECTION"].apply(flip_direction)
+    return sumstats
+
 def flipallelestats(sumstats,status="STATUS",verbose=True,log=Log()):
-    
-    check_dataframe_shape(sumstats, log, verbose)
-    
+    ##start function with col checking##########################################################
+    _start_line = "adjust statistics based on STATUS code"
+    _end_line = "adjusting statistics based on STATUS code"
+    _start_cols =[]
+    _start_function = ".check_data_consistency()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+
+    if_stats_flipped = False
     ###################get reverse complementary####################
     pattern = r"\w\w\w\w\w[45]\w"  
     #matched_index = status_match(sumstats[status],6,[4,5]) #
@@ -1217,107 +1291,49 @@ def flipallelestats(sumstats,status="STATUS",verbose=True,log=Log()):
             if verbose: log.write(" -Converting to reverse complement : EA and NEA...") 
             reverse_complement_nea = sumstats.loc[matched_index,'NEA'].apply(lambda x :get_reverse_complementary_allele(x)) 
             reverse_complement_ea = sumstats.loc[matched_index,'EA'].apply(lambda x :get_reverse_complementary_allele(x)) 
-            categories = set(sumstats.loc[:,'EA'])|set(sumstats.loc[:,'NEA']) |set(reverse_complement_ea) |set(reverse_complement_nea)
-            sumstats.loc[:,'EA']=pd.Categorical(sumstats.loc[:,'EA'],categories = categories) 
-            sumstats.loc[:,'NEA']=pd.Categorical(sumstats.loc[:,'NEA'],categories = categories ) 
+            categories = set(sumstats['EA'])|set(sumstats['NEA']) |set(reverse_complement_ea) |set(reverse_complement_nea)
+            sumstats['EA']=pd.Categorical(sumstats['EA'],categories = categories) 
+            sumstats['NEA']=pd.Categorical(sumstats['NEA'],categories = categories ) 
             sumstats.loc[matched_index,['NEA']] = reverse_complement_nea
             sumstats.loc[matched_index,['EA']] = reverse_complement_ea
             sumstats.loc[matched_index,status] = vchange_status(sumstats.loc[matched_index,status], 6, "4","2")
             if verbose: log.write(" -Changed the status for flipped variants : xxxxx4x -> xxxxx2x")
-
+        if_stats_flipped = True
     ###################flip ref####################
     pattern = r"\w\w\w\w\w[35]\w"  
     #matched_index = status_match(sumstats[status],6,[3,5]) #sumstats[status].str.match(pattern)
     matched_index = sumstats[status].str[5].str.match(r"3|5")
     if sum(matched_index)>0:
-        if verbose: log.write("Start to flip allele-specific stats for SNPs with status xxxxx[35]x: alt->ea , ref->nea ...{}".format(_get_version())) 
+        if verbose: log.write("Start to flip allele-specific stats for SNPs with status xxxxx[35]x: ALT->EA , REF->NEA ...{}".format(_get_version())) 
         if verbose: log.write(" -Flipping "+ str(sum(matched_index)) +" variants...") 
-        if ("NEA" in sumstats.columns) and ("EA" in sumstats.columns) :
-            if verbose: log.write(" -Swapping column: NEA <=> EA...") 
-            sumstats.loc[matched_index,['NEA','EA']] = sumstats.loc[matched_index,['EA','NEA']].values
-        if "BETA" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: BETA = - BETA...") 
-            sumstats.loc[matched_index,"BETA"] =     - sumstats.loc[matched_index,"BETA"].values
-        if "BETA_95L" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: BETA_95L = - BETA_95L...") 
-            sumstats.loc[matched_index,"BETA_95L"] =     - sumstats.loc[matched_index,"BETA_95L"].values
-        if "BETA_95U" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: BETA_95U = - BETA_95U...") 
-            sumstats.loc[matched_index,"BETA_95U"] =     - sumstats.loc[matched_index,"BETA_95U"].values
-        if "EAF" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: EAF = 1 - EAF...") 
-            sumstats.loc[matched_index,"EAF"] =   1 - sumstats.loc[matched_index,"EAF"].values
-        if "OR" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: OR = 1 / OR...") 
-            sumstats.loc[matched_index,"OR"] =   1 / sumstats.loc[matched_index,"OR"].values
-        if "OR_95L" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: OR_95L = 1 / OR_95L...") 
-            sumstats.loc[matched_index,"OR_95L"] =   1 / sumstats.loc[matched_index,"OR_95L"].values
-        if "OR_95U" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: OR_95U = 1 / OR_95U...") 
-            sumstats.loc[matched_index,"OR_95U"] =   1 / sumstats.loc[matched_index,"OR_95U"].values
-        if "HR" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: HR = 1 / HR...") 
-            sumstats.loc[matched_index,"HR"] =   1 / sumstats.loc[matched_index,"HR"].values
-        if "HR_95L" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: HR_95L = 1 / HR_95L...") 
-            sumstats.loc[matched_index,"HR_95L"] =   1 / sumstats.loc[matched_index,"HR_95L"].values
-        if "HR_95U" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: HR_95U = 1 / HR_95U...") 
-            sumstats.loc[matched_index,"HR_95U"] =   1 / sumstats.loc[matched_index,"HR_95U"].values
-        if "DIRECTION" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: DIRECTION +-? <=> -+? ...") 
-            sumstats.loc[matched_index,"DIRECTION"] =   sumstats.loc[matched_index,"DIRECTION"].apply(flip_direction)
+        
+        flip_by_swap(sumstats, matched_index, log, verbose)
+        flip_by_sign(sumstats, matched_index, log, verbose, cols=None)
+        flip_by_subtract(sumstats, matched_index, log, verbose, cols=None, factor=1)
+        flip_by_inverse(sumstats, matched_index, log, verbose, cols=None, factor=1)
+        
         #change status    
         if verbose: log.write(" -Changed the status for flipped variants : xxxxx[35]x -> xxxxx[12]x") 
         sumstats.loc[matched_index,status] = vchange_status(sumstats.loc[matched_index,status], 6, "35","12")
+        if_stats_flipped = True
         
     ###################flip ref for undistingushable indels####################
     pattern = r"\w\w\w\w[123][67]6"  
     #matched_index = status_match(sumstats[status],6,[1,2,3])|status_match(sumstats[status],6,[6,7])|status_match(sumstats[status],7,6) #sumstats[status].str.match(pattern)
     matched_index = sumstats[status].str[4:].str.match(r"[123][67]6")
     if sum(matched_index)>0:
-        if verbose: log.write("Start to flip allele-specific stats for standardized indels with status xxxx[123][67][6]: alt->ea , ref->nea...{}".format(_get_version())) 
+        if verbose: log.write("Start to flip allele-specific stats for standardized indels with status xxxx[123][67][6]: ALT->EA , REF->NEA...{}".format(_get_version())) 
         if verbose: log.write(" -Flipping "+ str(sum(matched_index)) +" variants...") 
-        if ("NEA" in sumstats.columns) and ("EA" in sumstats.columns) :
-            if verbose: log.write(" -Swapping column: NEA <=> EA...") 
-            sumstats.loc[matched_index,['NEA','EA']] = sumstats.loc[matched_index,['EA','NEA']].values
-        if "BETA" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: BETA = - BETA...") 
-            sumstats.loc[matched_index,"BETA"] =     - sumstats.loc[matched_index,"BETA"].values
-        if "BETA_95L" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: BETA_95L = - BETA_95L...") 
-            sumstats.loc[matched_index,"BETA_95L"] =     - sumstats.loc[matched_index,"BETA_95L"].values
-        if "BETA_95U" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: BETA_95U = - BETA_95U...") 
-            sumstats.loc[matched_index,"BETA_95U"] =     - sumstats.loc[matched_index,"BETA_95U"].values
-        if "EAF" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: EAF = 1 - EAF...") 
-            sumstats.loc[matched_index,"EAF"] =   1 - sumstats.loc[matched_index,"EAF"].values
-        if "OR" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: OR = 1 / OR...") 
-            sumstats.loc[matched_index,"OR"] =   1 / sumstats.loc[matched_index,"OR"].values
-        if "OR_95L" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: OR_95L = 1 / OR_95L...") 
-            sumstats.loc[matched_index,"OR_95L"] =   1 / sumstats.loc[matched_index,"OR_95L"].values
-        if "OR_95U" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: OR_95U = 1 / OR_95U...") 
-            sumstats.loc[matched_index,"OR_95U"] =   1 / sumstats.loc[matched_index,"OR_95U"].values
-        if "HR" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: HR = 1 / HR...") 
-            sumstats.loc[matched_index,"HR"] =   1 / sumstats.loc[matched_index,"HR"].values
-        if "HR_95L" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: HR_95L = 1 / HR_95L...") 
-            sumstats.loc[matched_index,"HR_95L"] =   1 / sumstats.loc[matched_index,"HR_95L"].values
-        if "HR_95U" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: HR_95U = 1 / HR_95U...") 
-            sumstats.loc[matched_index,"HR_95U"] =   1 / sumstats.loc[matched_index,"HR_95U"].values
-        if "DIRECTION" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: DIRECTION +-? <=> -+? ...") 
-            sumstats.loc[matched_index,"DIRECTION"] =   sumstats.loc[matched_index,"DIRECTION"].apply(flip_direction)
+        
+        flip_by_swap(sumstats, matched_index, log, verbose)
+        flip_by_sign(sumstats, matched_index, log, verbose, cols=None)
+        flip_by_subtract(sumstats, matched_index, log, verbose, cols=None, factor=1)
+        flip_by_inverse(sumstats, matched_index, log, verbose, cols=None, factor=1)
+        
         #change status    
         if verbose: log.write(" -Changed the status for flipped variants xxxx[123][67]6 -> xxxx[123][67]4") 
         sumstats.loc[matched_index,status] = vchange_status(sumstats.loc[matched_index,status], 7, "6","4")
+        if_stats_flipped = True
          # flip ref
     ###################flip statistics for reverse strand panlindromic variants####################
     pattern = r"\w\w\w\w\w[012]5"  
@@ -1326,43 +1342,20 @@ def flipallelestats(sumstats,status="STATUS",verbose=True,log=Log()):
     if sum(matched_index)>0:
         if verbose: log.write("Start to flip allele-specific stats for palindromic SNPs with status xxxxx[12]5: (-)strand <=> (+)strand...{}".format(_get_version())) 
         if verbose: log.write(" -Flipping "+ str(sum(matched_index)) +" variants...") 
-        if "BETA" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: BETA = - BETA...") 
-            sumstats.loc[matched_index,"BETA"] =     - sumstats.loc[matched_index,"BETA"].values
-        if "BETA_95L" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: BETA_95L = - BETA_95L...") 
-            sumstats.loc[matched_index,"BETA_95L"] =     - sumstats.loc[matched_index,"BETA_95L"].values
-        if "BETA_95U" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: BETA_95U = - BETA_95U...") 
-            sumstats.loc[matched_index,"BETA_95U"] =     - sumstats.loc[matched_index,"BETA_95U"].values
-        if "EAF" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: EAF = 1 - EAF...") 
-            sumstats.loc[matched_index,"EAF"] =   1 - sumstats.loc[matched_index,"EAF"].values
-        if "OR" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: OR = 1 / OR...") 
-            sumstats.loc[matched_index,"OR"] =   1 / sumstats.loc[matched_index,"OR"].values
-        if "OR_95L" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: OR_95L = 1 / OR_95L...") 
-            sumstats.loc[matched_index,"OR_95L"] =   1 / sumstats.loc[matched_index,"OR_95L"].values
-        if "OR_95U" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: OR_95U = 1 / OR_95U...") 
-            sumstats.loc[matched_index,"OR_95U"] =   1 / sumstats.loc[matched_index,"OR_95U"].values
-        if "HR" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: HR = 1 / HR...") 
-            sumstats.loc[matched_index,"HR"] =   1 / sumstats.loc[matched_index,"HR"].values
-        if "HR_95L" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: HR_95L = 1 / HR_95L...") 
-            sumstats.loc[matched_index,"HR_95L"] =   1 / sumstats.loc[matched_index,"HR_95L"].values
-        if "HR_95U" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: HR_95U = 1 / HR_95U...") 
-            sumstats.loc[matched_index,"HR_95U"] =   1 / sumstats.loc[matched_index,"HR_95U"].values
-        if "DIRECTION" in sumstats.columns:
-            if verbose: log.write(" -Flipping column: DIRECTION +-? <=> -+? ...") 
-            sumstats.loc[matched_index,"DIRECTION"] =   sumstats.loc[matched_index,"DIRECTION"].apply(flip_direction)
+
+        flip_by_sign(sumstats, matched_index, log, verbose, cols=None)
+        flip_by_subtract(sumstats, matched_index, log, verbose, cols=None, factor=1)
+        flip_by_inverse(sumstats, matched_index, log, verbose, cols=None, factor=1)
+        
         #change status    
         if verbose: log.write(" -Changed the status for flipped variants:  xxxxx[012]5: ->  xxxxx[012]2") 
         sumstats.loc[matched_index,status] = vchange_status(sumstats.loc[matched_index,status], 7, "5","2")
-    if verbose: log.write("Finished converting successfully!")
+        if_stats_flipped = True
+
+    if if_stats_flipped == True:
+        finished(log, verbose, "adjusting")
+    else:
+        finished(log, verbose, "adjusting with no statistics changed.")
     return sumstats
 ""
 
@@ -1371,8 +1364,8 @@ def flipallelestats(sumstats,status="STATUS",verbose=True,log=Log()):
 # 20220426
 def liftover_snv(row,chrom,converter,to_build):
     status_pre=""
-    status_end=row[1][2]+"9"+row[1][4]+"99"  
-    pos_0_based = int(row[0]) - 1
+    status_end=row.iloc[1][2]+"9"+row.iloc[1][4]+"99"  
+    pos_0_based = int(row.iloc[0]) - 1
     results = converter[chrom][pos_0_based]
     if converter[chrom][pos_0_based]:
         # return chrom, pos_1_based
@@ -1402,13 +1395,25 @@ def liftover_variant(sumstats,
     return sumstats
 
 def parallelizeliftovervariant(sumstats,n_cores=1,chrom="CHR", pos="POS", from_build="19", to_build="38",status="STATUS",remove=True, verbose=True,log=Log()):
-    if check_col(sumstats,chrom,pos,status) is not True:
-        if verbose: log.write("WARNING! .liftover(): specified columns not detected..skipping...")
-        return sumstats
-    if verbose: log.write("Start to perform liftover...{}".format(_get_version()))
-    check_dataframe_shape(sumstats, log, verbose)  
-    if verbose: log.write(" -CPU Cores to use :",n_cores)
-    if verbose: log.write(" -Performing liftover ...")
+    ##start function with col checking##########################################################
+    _start_line = "perform liftover"
+    _end_line = "liftover"
+    _start_cols =[chrom,pos,status]
+    _start_function = ".liftover()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            n_cores=n_cores,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+
     if verbose: log.write(" -Creating converter : hg" + from_build +" to hg"+ to_build)
     # valid chr and pos
     pattern = r"\w\w\w0\w\w\w"  
@@ -1420,11 +1425,12 @@ def parallelizeliftovervariant(sumstats,n_cores=1,chrom="CHR", pos="POS", from_b
         if sum(to_lift)<10000:
             n_cores=1
     
-        df_split = np.array_split(sumstats.loc[:,[chrom,pos,status]], n_cores)
+        #df_split = np.array_split(sumstats[[chrom,pos,status]], n_cores)
+        df_split = _df_split(sumstats[[chrom,pos,status]], n_cores)
         pool = Pool(n_cores)
         #df = pd.concat(pool.starmap(func, df_split))
         func=liftover_variant
-        sumstats.loc[:,[chrom,pos,status]] = pd.concat(pool.map(partial(func,chrom=chrom,pos=pos,from_build=from_build,to_build=to_build,status=status),df_split))
+        sumstats[[chrom,pos,status]] = pd.concat(pool.map(partial(func,chrom=chrom,pos=pos,from_build=from_build,to_build=to_build,status=status),df_split))
         pool.close()
         pool.join()
     ############################################################################
@@ -1439,18 +1445,29 @@ def parallelizeliftovervariant(sumstats,n_cores=1,chrom="CHR", pos="POS", from_b
     sumstats = fixchr(sumstats,chrom=chrom,add_prefix="",remove=remove, verbose=True)
     sumstats = fixpos(sumstats,pos=pos,remove=remove, verbose=True)
     
-    if verbose: log.write("Finished liftover successfully!")
+    finished(log,verbose,_end_line)
     return sumstats
 
 ###############################################################################################################
 # 20220426
 def sortcoordinate(sumstats,chrom="CHR",pos="POS",reindex=True,verbose=True,log=Log()):
-    if check_col(sumstats,chrom,pos) is not True:
-        if verbose: log.write(".liftover(): specified columns not detected..skipping...")
-        return sumstats
-    
-    if verbose: log.write("Start to sort the genome coordinates...{}".format(_get_version()))
-    check_dataframe_shape(sumstats, log, verbose)
+    ##start function with col checking##########################################################
+    _start_line = "sort the genome coordinates"
+    _end_line = "sorting coordinates"
+    _start_cols =[chrom,pos]
+    _start_function = ".sort_coordinate()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
     
     try:
         if sumstats[pos].dtype == "Int64":
@@ -1460,50 +1477,144 @@ def sortcoordinate(sumstats,chrom="CHR",pos="POS",reindex=True,verbose=True,log=
             sumstats[pos]  = np.floor(pd.to_numeric(sumstats[pos], errors='coerce')).astype('Int64')
     except:
         pass
-
-    if verbose: log.write(" -Sorting genome coordinates...")
     sumstats = sumstats.sort_values(by=[chrom,pos],ascending=True,ignore_index=True)
-    if verbose: log.write("Finished sorting genome coordinates successfully!")
-    gc.collect()
+    
+    finished(log,verbose,_end_line)
     return sumstats
 ###############################################################################################################
 # 20230430 added HR HR_95 BETA_95 N_CASE N_CONTROL
-def sortcolumn(sumstats,verbose=True,log=Log(),order = [
+def sortcolumn(sumstats,verbose=True,log=Log(),order = None):
+    ##start function with col checking##########################################################
+    _start_line = "reorder the columns"
+    _end_line = "reordering the columns"
+    _start_cols =[]
+    _start_function = ".sort_column()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    ############################################################################################
+
+    if order is None:
+        order = [
         "SNPID","rsID", "CHR", "POS", "EA", "NEA", "EAF", "MAF", "BETA", "SE","BETA_95L","BETA_95U", "Z","T","F",
-        "CHISQ", "P", "MLOG10P", "OR", "OR_95L", "OR_95U","HR", "HR_95L", "HR_95U","INFO", "N","N_CASE","N_CONTROL","DIRECTION","I2","P_HET","DOF","SNPR2","STATUS"
-           ]):
-    if verbose: log.write("Start to reorder the columns...{}".format(_get_version()))
-    check_dataframe_shape(sumstats, log, verbose)   
-    
+        "CHISQ", "P", "MLOG10P", "OR", "OR_95L", "OR_95U","HR", "HR_95L", "HR_95U","INFO", "N","N_CASE","N_CONTROL","DIRECTION","I2","P_HET","DOF","SNPR2","STATUS"]
     output_columns = []
     for i in order:
         if i in sumstats.columns: output_columns.append(i)
     for i in sumstats.columns:
         if i not in order: output_columns.append(i)
     if verbose: log.write(" -Reordering columns to    :", ",".join(output_columns))
-    sumstats = sumstats.loc[:, output_columns]
-    if verbose: log.write("Finished sorting columns successfully!")
+    sumstats = sumstats[ output_columns]
+
+    finished(log,verbose,_end_line)
     return sumstats
 
-def check_col(df,*args):
+
+###############################################################################################################
+def start_to(sumstats, 
+             log, 
+             verbose, 
+             start_line,
+             end_line,
+             start_cols,
+             start_function,
+             ref_vcf=None,
+             ref_fasta=None,
+             n_cores=None,
+             ref_tsv=None,
+             **args
+             ):
+    
+    log.write("Start to {}...{}".format(start_line,_get_version()), verbose=verbose)
+    
+    check_dataframe_shape(sumstats=sumstats, 
+                          log=log, 
+                          verbose=verbose)  
+    
+    is_enough_col = check_col(sumstats.columns, 
+                              verbose=verbose, 
+                              log=log, 
+                              cols=start_cols, 
+                              function=start_function)
+    
+    if is_enough_col==True:
+        if n_cores is not None:
+            log.write(" -Number of threads/cores to use: {}".format(n_cores))
+        if ref_vcf is not None:
+            log.write(" -Reference VCF: {}".format(ref_vcf))
+        if ref_fasta is not None:
+            log.write(" -Reference FASTA: {}".format(ref_fasta))
+        if ref_tsv is not None:
+            log.write(" -Reference TSV: {}".format(ref_tsv))
+        
+        is_args_valid = True
+        for key, value in args.items():
+            is_args_valid = is_args_valid & check_arg(log, verbose, key, value, start_function)
+        is_enough_col = is_args_valid & is_enough_col
+
+    if  is_enough_col == False:
+        skipped(log, verbose, end_line)
+
+    return is_enough_col
+
+def finished(log, verbose, end_line):
+    log.write("Finished {}.".format(end_line), verbose=verbose)
+    gc.collect()
+
+def skipped(log, verbose, end_line):
+    log.write("Skipped {}.".format(end_line), verbose=verbose)
+    gc.collect()
+
+def check_arg(log, verbose, key, value, function):
+    if value is None:
+        log.warning("Necessary argument {} for {} is not provided!".format(key, function))
+        return False
+    return True
+
+def check_col(df_col_names, verbose=True, log=Log(), cols=None, function=None):
     not_in_df=[]
-    for i in args:
+    for i in cols:
         if type(i) is str:
-            if i in df.columns:
+            # single check
+            if i in df_col_names:
                 continue
             else:
                 not_in_df.append(i)
         else:
+            # paried check
             count=0
             for j in i:
-                if j in df.columns:
+                if j not in df_col_names:
+                    not_in_df.append(j)
                     count+=1
-            if count==0:
-                return False
-                print(" -Specified columns names was not detected. Please check:"+",".join(i))
-    
+
     if len(not_in_df)>0:
+        if function is None:
+            to_show_title=" "
+        else:
+            to_show_title = " for {} ".format(function)
+        log.warning("Necessary columns{}were not detected:{}".format(to_show_title, ",".join(not_in_df)))
+        skipped(log, verbose, end_line=function)
         return False
-        print(" -Specified columns names was not detected. Please check:"+",".join(not_in_df))
+    
     return True
 
+###############################################################################################################
+def _df_split(dataframe, n):
+    chunks = []
+    chunk_size = int(dataframe.shape[0] // n)+1
+
+    for index in range(0, dataframe.shape[0], chunk_size):
+        chunks.append(
+            dataframe.iloc[index:index + chunk_size]
+        )
+
+    return chunks