gwaslab 3.6.5__py3-none-any.whl → 3.6.7__py3-none-any.whl

gwaslab/hm_harmonize_sumstats.py

@@ -27,7 +27,7 @@ from gwaslab.bd_common_data import _maketrans
 from gwaslab.g_vchange_status import vchange_status
 from gwaslab.g_version import _get_version
 from gwaslab.cache_manager import CacheManager, PALINDROMIC_INDEL, NON_PALINDROMIC
-
+from gwaslab.g_vchange_status import STATUS_CATEGORIES
 #rsidtochrpos
 #checkref
 #parallelizeassignrsid
@@ -357,10 +357,7 @@ def oldcheckref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status=
     
     log.write("\n",end="",show_time=False,verbose=verbose) 
         
-    CATEGORIES = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
-    sumstats[status] = pd.Categorical(sumstats[status],categories=CATEGORIES)
-    #sumstats[status] = sumstats[status].astype("string")
-
+    sumstats[status] = pd.Categorical(sumstats[status],categories=STATUS_CATEGORIES)
 
     available_to_check =sum( (~sumstats[pos].isna()) & (~sumstats[nea].isna()) & (~sumstats[ea].isna()))
     status_0=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[0]\w", case=False, flags=0, na=False))
@@ -681,9 +678,7 @@ def checkref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="ST
         sumstats.loc[to_check_ref,status] = check_status(sumstats_to_check, all_records_dict, log=log, verbose=verbose)
         log.write(" -Finished checking records", verbose=verbose) 
     
-    CATEGORIES = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
-    sumstats[status] = pd.Categorical(sumstats[status],categories=CATEGORIES)
-    #sumstats[status] = sumstats[status].astype("string")
+    sumstats[status] = pd.Categorical(sumstats[status],categories=STATUS_CATEGORIES)
 
     available_to_check =sum( (~sumstats[pos].isna()) & (~sumstats[nea].isna()) & (~sumstats[ea].isna()))
     status_0=sum(sumstats["STATUS"].str.match("\w\w\w\w\w[0]\w", case=False, flags=0, na=False))

gwaslab/{io_load_ld.py → io/io_load_ld.py}

@@ -1,8 +1,6 @@
-
 import scipy.sparse as sparse
 import numpy as np
 import pandas as pd
-from gwaslab.hm_casting import _merge_mold_with_sumstats_by_chrpos
 import subprocess
 import os
 import re
@@ -10,16 +8,22 @@ import gc
 import pandas as pd
 import numpy as np
 from gwaslab.g_Log import Log
-from gwaslab.qc_fix_sumstats import start_to
-from gwaslab.qc_fix_sumstats import finished
-from gwaslab.util_in_get_sig import getsig
-from gwaslab.util_ex_process_ref import _process_plink_input_files
 from gwaslab.g_version import _checking_plink_version
-from gwaslab.util_in_filter_value import _exclude_hla
-from gwaslab.util_ex_calculate_ldmatrix import _extract_variants_in_locus
-from gwaslab.util_ex_calculate_ldmatrix import _export_snplist_and_locus_sumstats
-from gwaslab.viz_plot_regional2 import _get_lead_id
-from gwaslab.util_ex_calculate_ldmatrix import _extract_variants_in_locus
+
+from gwaslab.hm.hm_casting import _merge_mold_with_sumstats_by_chrpos
+
+from gwaslab.qc.qc_fix_sumstats import start_to
+from gwaslab.qc.qc_fix_sumstats import finished
+
+from gwaslab.util.util_in_get_sig import getsig
+from gwaslab.util.util_ex_process_ref import _process_plink_input_files
+from gwaslab.util.util_in_filter_value import _exclude_hla
+from gwaslab.util.util_ex_calculate_ldmatrix import _extract_variants_in_locus
+from gwaslab.util.util_ex_calculate_ldmatrix import _export_snplist_and_locus_sumstats
+from gwaslab.util.util_ex_calculate_ldmatrix import _extract_variants_in_locus
+
+from gwaslab.viz.viz_plot_regional2 import _get_lead_id
+
 
 def tofinemapping_using_ld(sumstats, 
                   study=None, 
@@ -72,7 +76,7 @@ def tofinemapping_using_ld(sumstats,
         sig_df = getsig(sumstats,id="SNPID",chrom="CHR",pos="POS",p="P"+suffixes[0],**getlead_args)
     else:
         sig_df = sumstats.loc[sumstats["SNPID"].isin(loci),:]
-
+    log.write(" -Number of loci: {}...".format(len(sig_df)),verbose=verbose)
     # Drop duplicate!!!!
     log.write(" -Dropping duplicated SNPIDs...",verbose=verbose)
     sumstats = sumstats.drop_duplicates(subset=["SNPID"]).copy()
@@ -447,7 +451,6 @@ def _merge_ld_map_with_sumstats(row,
     # matching by SNPID
     # preserve bim keys (use intersection of keys from both frames, similar to a SQL inner join; preserve the order of the left keys.)
     combined_df = pd.merge(ld_map, locus_sumstats, on=["CHR","POS"],how="inner")
-    
     # match allele
     perfect_match =  ((combined_df["EA"] == combined_df["EA_bim"]) & (combined_df["NEA"] == combined_df["NEA_bim"]) ) 
     log.write("   -Variants with perfect matched alleles:{}".format(sum(perfect_match)))

gwaslab/{io_preformat_input.py → io/io_preformat_input.py}

@@ -3,14 +3,16 @@ import numpy as np
 import scipy.stats as ss
 import gzip
 import os
+import re
 import gc
-from gwaslab.bd_common_data import get_format_dict
-from gwaslab.qc_fix_sumstats import sortcolumn
-from gwaslab.qc_fix_sumstats import _process_build
-from gwaslab.qc_check_datatype import check_datatype
-from gwaslab.qc_check_datatype import quick_convert_datatype
-from gwaslab.qc_check_datatype import check_dataframe_memory_usage
+from gwaslab.bd.bd_common_data import get_format_dict
+from gwaslab.qc.qc_fix_sumstats import sortcolumn
+from gwaslab.qc.qc_fix_sumstats import _process_build
+from gwaslab.qc.qc_check_datatype import check_datatype
+from gwaslab.qc.qc_check_datatype import quick_convert_datatype
+from gwaslab.qc.qc_check_datatype import check_dataframe_memory_usage
 from gwaslab.g_headers import _check_overlap_with_reserved_keys
+from gwaslab.g_vchange_status import STATUS_CATEGORIES
 #20221030
 def preformat(sumstats,
           fmt=None,
@@ -122,62 +124,22 @@ def preformat(sumstats,
         
         if "sep" not in readargs.keys():
             readargs["sep"] = "\t"
+    else:
+        meta_data = None
         
 #########################################################################################################################################################      
     
-    # check chr-separated path / vcf / then print header.            
-    try:
-        if type(sumstats) is str:
-            ## loading data from path #################################################
-            inpath = sumstats
-            ###load sumstats by each chromosome #################################################  
-            if "@" in inpath:
-                log.write(" -Detected @ in path: load sumstats by each chromosome...",verbose=verbose)
-                inpath_chr_list=[]
-                inpath_chr_num_list=[]
-                for chromosome in list(range(1,26))+["x","y","X","Y","MT","mt","m","M"]:
-                    inpath_chr = inpath.replace("@",str(chromosome))  
-                    if isfile_casesensitive(inpath_chr):
-                        inpath_chr_num_list.append(str(chromosome))
-                        inpath_chr_list.append(inpath_chr)        
-                log.write(" -Chromosomes detected:",",".join(inpath_chr_num_list),verbose=verbose)
-                readargs_header = get_readargs_header(inpath = inpath_chr_list[0], readargs = readargs)
-                row_one = pd.read_table(inpath_chr_list[0],**readargs_header)
-                # columns in the sumstats
-                raw_cols = row_one.columns
-            else:
-            ##### loading data from tabular file#################################################
-                readargs_header = get_readargs_header(inpath = inpath, readargs = readargs)
-                row_one = pd.read_table(inpath,**readargs_header)
-                raw_cols = row_one.columns
-            
-            if fmt=="vcf":
-                # expanded
-                format_cols = list(row_one["FORMAT"].str.split(":"))[0]
-                # fixed + study1 + expanded
-                raw_cols = meta_data["format_fixed"] + [raw_cols[9]] + format_cols
+    # check chr-separated path / vcf / then print header.     
+    inpath, inpath_chr_list, inpath_chr_num_list, format_cols, raw_cols, usecols, dtype_dictionary  = check_path_and_header(sumstats, 
+                                                                                                                    fmt, 
+                                                                                                                    meta_data, 
+                                                                                                                    readargs, 
+                                                                                                                    usecols, 
+                                                                                                                    dtype_dictionary, 
+                                                                                                                    rename_dictionary, 
+                                                                                                                    log, 
+                                                                                                                    verbose)     
 
-            ###################################################################################### 
-        elif type(sumstats) is pd.DataFrame:
-            ## loading data from dataframe
-            raw_cols = sumstats.columns
-        
-        ################################################
-        for key,value in rename_dictionary.items():
-            # check avaiable keys  key->raw header
-            # usecols : a list of raw headers to load from file/DataFrame 
-            if key in raw_cols:
-                usecols.append(key)
-            if value in ["EA","NEA"]:
-                dtype_dictionary[key]="category"
-            if value in ["STATUS"]:
-                dtype_dictionary[key]="string"     
-            if value in ["CHR"]:
-                dtype_dictionary[key]="string"  
-    
-    except ValueError:
-        raise ValueError("Please input a path or a pd.DataFrame, and make sure the separator is correct and the columns you specified are in the file.")
-        
     ###################################################################################################################################################
     ## check columns/datatype to use 
     if snpid:
@@ -359,7 +321,7 @@ def preformat(sumstats,
     try:
         if type(sumstats) is str:
             ## loading data from path
-            inpath = sumstats
+            #inpath = sumstats
             if "@" in inpath:
                 log.write("Start to initialize gl.Sumstats from files with pattern :" + inpath,verbose=verbose)
                 sumstats_chr_list=[]
@@ -445,14 +407,14 @@ def preformat(sumstats,
         sumstats["N_CONTROL"] = ncontrol
     
     ### status ######################################################################################################
-    if status is None:
-        sumstats = process_status(sumstats=sumstats,build=build,log=log,verbose=verbose)
+    
+    sumstats = process_status(sumstats=sumstats,build=build,status=status,log=log,verbose=verbose)
     
     ## ea/nea, ref/alt ##############################################################################################
     sumstats = process_allele(sumstats=sumstats,log=log,verbose=verbose)
         
     ## NEAF to EAF ###########################################################################################################
-    if neaf is not None :
+    if neaf is not None or ("NEAF" in sumstats.columns and "EAF" not in sumstats.columns):
         sumstats = process_neaf(sumstats=sumstats,log=log,verbose=verbose)
 
     ## reodering ###################################################################################################  
@@ -562,9 +524,15 @@ def process_neaf(sumstats,log,verbose):
     log.write(" -NEAF is specified...",verbose=verbose) 
     pre_number=len(sumstats)
     log.write(" -Checking if 0<= NEAF <=1 ...",verbose=verbose) 
-    sumstats["EAF"] = pd.to_numeric(sumstats["EAF"], errors='coerce')
-    sumstats = sumstats.loc[(sumstats["EAF"]>=0) & (sumstats["EAF"]<=1),:]
-    sumstats["EAF"] = 1- sumstats["EAF"]
+    if "NEAF" in sumstats.columns:
+        sumstats["NEAF"] = pd.to_numeric(sumstats["NEAF"], errors='coerce')
+        sumstats = sumstats.loc[(sumstats["NEAF"]>=0) & (sumstats["NEAF"]<=1),:]
+        sumstats["EAF"] = 1- sumstats["NEAF"]
+        sumstats.drop(columns=["NEAF"], inplace=True)
+    else:
+        sumstats["EAF"] = pd.to_numeric(sumstats["EAF"], errors='coerce')
+        sumstats = sumstats.loc[(sumstats["EAF"]>=0) & (sumstats["EAF"]<=1),:]
+        sumstats["EAF"] = 1- sumstats["EAF"]
     log.write(" -Converted NEAF to EAF.",verbose=verbose) 
     after_number=len(sumstats)
     log.write(" -Removed "+str(pre_number - after_number)+" variants with bad NEAF.",verbose=verbose) 
@@ -599,13 +567,14 @@ def process_allele(sumstats,log,verbose):
         sumstats["NEA"]=sumstats["NEA"].astype("category")  
     return sumstats
 
-def process_status(sumstats,build,log,verbose):
-    log.write(" -Initiating a status column: STATUS ...",verbose=verbose)
-    #sumstats["STATUS"] = int(build)*(10**5) +99999
-    build = _process_build(build,log,verbose)
-    sumstats["STATUS"] = build +"99999"
-    categories = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
-    sumstats["STATUS"] = pd.Categorical(sumstats["STATUS"],categories=categories)
+def process_status(sumstats,build,status, log,verbose):
+    if status is None:
+        log.write(" -Initiating a status column: STATUS ...",verbose=verbose)
+        #sumstats["STATUS"] = int(build)*(10**5) +99999
+        build = _process_build(build,log,verbose)
+        sumstats["STATUS"] = build +"99999"
+
+    sumstats["STATUS"] = pd.Categorical(sumstats["STATUS"],categories=STATUS_CATEGORIES)
     return sumstats
 
 
@@ -649,4 +618,114 @@ def _load_variants_with_pattern(inpath,usecols,dtype_dictionary,readargs,rename_
     log.write(" -Loading only variants with pattern :  {} ...".format(snpid_pat),verbose=verbose)
     sumstats_filtered = pd.concat([chunk[chunk[chunk_snpid].str.match(snpid_pat, case=False,na=False) ] for chunk in sumstats_iter])
     log.write(" -Loaded {} variants with pattern : {} ...".format(len(sumstats_filtered), snpid_pat),verbose=verbose)
-    return sumstats_filtered
+    return sumstats_filtered
+
+
+def check_path_and_header(sumstats=None, 
+                          fmt=None, 
+                          meta_data=None, 
+                          readargs=None, 
+                          usecols=None, 
+                          dtype_dictionary=None, 
+                          rename_dictionary=None, 
+                          log=None, 
+                          verbose=None):
+    
+
+    if type(sumstats) is str:
+        ## loading data from path #################################################
+        inpath = sumstats
+        
+        try:
+            format_cols, raw_cols, inpath_chr_list, inpath_chr_num_list = process_inpath_and_load_header(inpath, fmt, meta_data,  readargs, log, verbose)
+        
+        except (FileNotFoundError, IndexError):
+            log.warning("Loading {} failed...Tesing if compressed/uncompressed...".format(inpath),verbose=verbose)
+            try:
+                if inpath[-3:]==".gz":
+                    inpath = inpath[:-3]
+                    log.write(" -Trying to load {}...".format(inpath),verbose=verbose)
+                    format_cols, raw_cols, inpath_chr_list, inpath_chr_num_list =process_inpath_and_load_header(inpath, fmt, meta_data,  readargs, log, verbose)
+                else:
+                    inpath = inpath+".gz"
+                    log.write(" -Trying to load {}...".format(inpath),verbose=verbose)
+                    format_cols, raw_cols, inpath_chr_list, inpath_chr_num_list = process_inpath_and_load_header(inpath, fmt, meta_data,  readargs, log, verbose)
+            except:
+                raise ValueError("Please input a valid path, and make sure the separator is correct and the columns you specified are in the file.") 
+
+        ###################################################################################### 
+    elif type(sumstats) is pd.DataFrame:
+        inpath = None
+        format_cols = None
+        inpath_chr_list = None
+        inpath_chr_num_list = None
+        ## loading data from dataframe
+        raw_cols = sumstats.columns
+
+    ################################################
+    for key,value in rename_dictionary.items():
+        # check avaiable keys  key->raw header
+        # usecols : a list of raw headers to load from file/DataFrame 
+        if key in raw_cols:
+            usecols.append(key)
+        if value in ["EA","NEA"]:
+            dtype_dictionary[key]="category"
+        if value in ["STATUS"]:
+            dtype_dictionary[key]="string"     
+        if value in ["CHR"]:
+            dtype_dictionary[key]="string"  
+
+    return inpath, inpath_chr_list, inpath_chr_num_list, format_cols, raw_cols, usecols, dtype_dictionary
+
+def process_inpath_and_load_header(inpath, fmt, meta_data,  readargs, log, verbose):
+    
+    format_cols = None
+    inpath_chr_list = None
+    inpath_chr_num_list = None
+
+    if "@" in inpath:
+        log.write(" -Detected @ in path: load sumstats by each chromosome...",verbose=verbose)
+        inpath_chr_list=[]
+        inpath_chr_num_list=[]
+        
+        # create a regex pattern for matching
+        pat = os.path.basename(inpath).replace("@","(\w+)")
+        
+        # get dir
+        dirname = os.path.dirname(inpath)
+        
+        # all files in the directory
+        files = os.listdir(dirname)
+        
+        files.sort()
+
+        for file in files:
+            # match
+            result = re.match(pat, file)
+            if result:
+                # get chr
+                chr_matched = str(result.group(1))
+                inpath_chr_num_list.append(chr_matched)
+                inpath_chr_list.append(inpath.replace("@",str(chr_matched))  )
+        
+        log.write(" -Chromosomes detected:",",".join(inpath_chr_num_list),verbose=verbose)
+
+        #if inpath_chr_list is empty-> IndexError
+        readargs_header = get_readargs_header(inpath = inpath_chr_list[0], readargs = readargs)
+        row_one = pd.read_table(inpath_chr_list[0],**readargs_header)
+        # columns in the sumstats
+        raw_cols = row_one.columns
+    else:
+    ##### loading data from tabular file#################################################
+    #if file not found, FileNotFoundError
+        readargs_header = get_readargs_header(inpath = inpath, readargs = readargs)
+        row_one = pd.read_table(inpath,**readargs_header)
+        raw_cols = row_one.columns
+
+    if fmt=="vcf":
+        # expanded
+        format_cols = list(row_one["FORMAT"].str.split(":"))[0]
+        # fixed + study1 + expanded
+        raw_cols = meta_data["format_fixed"] + [raw_cols[9]] + format_cols
+
+    return format_cols, raw_cols, inpath_chr_list, inpath_chr_num_list

gwaslab/{io_preformat_input_polars.py → io/io_preformat_input_polars.py}

@@ -5,12 +5,12 @@ import scipy.stats as ss
 import gzip
 import os
 import gc
-from gwaslab.bd_common_data import get_format_dict
-from gwaslab.qc_fix_sumstats import sortcolumn
-from gwaslab.qc_fix_sumstats import _process_build
-from gwaslab.qc_check_datatype_polars import check_datatype
-from gwaslab.qc_check_datatype_polars import quick_convert_datatype
-from gwaslab.qc_check_datatype_polars import check_dataframe_memory_usage
+from gwaslab.bd.bd_common_data import get_format_dict
+from gwaslab.qc.qc_fix_sumstats import sortcolumn
+from gwaslab.qc.qc_fix_sumstats import _process_build
+from gwaslab.qc.qc_check_datatype_polars import check_datatype_polars
+from gwaslab.qc.qc_check_datatype_polars import quick_convert_datatype
+from gwaslab.qc.qc_check_datatype_polars import check_dataframe_memory_usage
 from gwaslab.g_headers import _check_overlap_with_reserved_keys
 #20221030
 def preformatp(sumstats,
@@ -433,7 +433,7 @@ def preformatp(sumstats,
     #sumstats = sortcolumn(sumstats=sumstats,log=log,verbose=verbose)    
     sumstats = quick_convert_datatype(sumstats,log=log,verbose=verbose)
     
-    check_datatype(sumstats,log=log,verbose=verbose)
+    check_datatype_polars(sumstats,log=log,verbose=verbose)
     #gc.collect()
     check_dataframe_memory_usage(sumstats,log=log,verbose=verbose)
     

gwaslab/{io_read_pipcs.py → io/io_read_pipcs.py}

@@ -1,7 +1,7 @@
 import pandas as pd
 from gwaslab.g_Log import Log
-from gwaslab.qc_check_datatype import check_datatype
-from gwaslab.qc_check_datatype import check_dataframe_memory_usage
+from gwaslab.qc.qc_check_datatype import check_datatype
+from gwaslab.qc.qc_check_datatype import check_dataframe_memory_usage
 import re
 import os
 
@@ -11,7 +11,8 @@ def _read_pipcs(data,
                 group=None,
                 studie_names=None,
                 log=Log(),
-                verbose=True):
+                verbose=True,
+                **readcsv_kwargs):
     
     log.write("Start to load PIP and CREDIBLE_SET_INDEX from file...",verbose=verbose)
     log.write(" -File:{}".format(output_prefix),verbose=verbose)
@@ -32,14 +33,14 @@ def _read_pipcs(data,
         pipcs_single_list=[]
         for index,pipcs_path in enumerate(pipcs_path_list):
             log.write(" -Loading {}:".format(pipcs_loci_list[index]) + pipcs_path)
-            pipcs_single = pd.read_csv(pipcs_path)
+            pipcs_single = pd.read_csv(pipcs_path,**readcsv_kwargs)
             if "LOCUS" not in pipcs_single.columns:
                 pipcs_single["LOCUS"]=pipcs_loci_list[index]
             pipcs_single_list.append(pipcs_single)
 
         pipcs = pd.concat(pipcs_single_list, axis=0, ignore_index=True) 
     else:
-        pipcs = pd.read_csv("{}".format(output_prefix))
+        pipcs = pd.read_csv("{}".format(output_prefix),**readcsv_kwargs)
     
     if "CHR" not in pipcs.columns:
         log.write(" -Merging CHR and POS from main dataframe...",verbose=verbose)

gwaslab/{io_read_tabular.py → io/io_read_tabular.py}

@@ -1,7 +1,7 @@
 import pandas as pd
-from gwaslab.bd_common_data import get_formats_list
+from gwaslab.bd.bd_common_data import get_formats_list
 from gwaslab.g_Log import Log
-from gwaslab.bd_common_data import get_format_dict
+from gwaslab.bd.bd_common_data import get_format_dict
 
 def _read_tabular(path, fmt, **kwargs):
     

gwaslab/{io_to_formats.py → io/io_to_formats.py}

@@ -7,16 +7,19 @@ from pysam import tabix_compress
 from pysam import tabix_index
 from datetime import datetime
 from datetime import date
-from gwaslab.io_preformat_input import print_format_info
-from gwaslab.bd_common_data import get_formats_list
 from gwaslab.g_Log import Log
-from gwaslab.bd_common_data import get_format_dict
-from gwaslab.bd_common_data import get_number_to_chr
 from gwaslab.g_version import gwaslab_info
-from gwaslab.bd_get_hapmap3 import gethapmap3
-from gwaslab.util_in_filter_value import _exclude_hla
-from gwaslab.util_in_filter_value import _exclude
-from gwaslab.util_in_filter_value import _extract
+
+from gwaslab.io.io_preformat_input import print_format_info
+
+from gwaslab.bd.bd_common_data import get_format_dict
+from gwaslab.bd.bd_common_data import get_number_to_chr
+from gwaslab.bd.bd_common_data import get_formats_list
+from gwaslab.bd.bd_get_hapmap3 import gethapmap3
+
+from gwaslab.util.util_in_filter_value import _exclude_hla
+from gwaslab.util.util_in_filter_value import _exclude
+from gwaslab.util.util_in_filter_value import _extract
 # to vcf
 # to fmt
     ## vcf
@@ -402,11 +405,11 @@ def tofmt(sumstats,
     
 ####################################################################################################################
 def _write_tabular(sumstats,rename_dictionary, path, tab_fmt, to_csvargs, to_tabular_kwargs, log, verbose):
-    chr_header = rename_dictionary["CHR"]
     if tab_fmt=="tsv" or tab_fmt=="csv":
         try:
             log.write(f"  -Fast to csv mode...",verbose=verbose)
             if "@" in path:
+                chr_header = rename_dictionary["CHR"]
                 log.write(f"  -@ detected: writing each chromosome to a single file...",verbose=verbose)
                 log.write("  -Chromosomes:{}...".format(list(sumstats["CHR"].unique())),verbose=verbose)
                 for single_chr in list(sumstats["CHR"].unique()):
@@ -420,6 +423,7 @@ def _write_tabular(sumstats,rename_dictionary, path, tab_fmt, to_csvargs, to_tab
         except:
             log.write(f"Error in using fast_to_csv. Falling back to original implementation.",verbose=verbose)
             if "@" in path:
+                chr_header = rename_dictionary["CHR"]
                 log.write(f"  -@ detected: writing each chromosome to a single file...",verbose=verbose)
                 log.write("  -Chromosomes:{}...".format(list(sumstats["CHR"].unique())),verbose=verbose)
                 for single_chr in list(sumstats["CHR"].unique()):

gwaslab/{io_to_pickle.py → io/io_to_pickle.py}

@@ -44,4 +44,19 @@ def load_data_from_pickle(path,usecols=None):
                 existing_cols.append(i)
         data = data.loc[:,existing_cols]
         gc.collect()
-    return data
+    return data
+
+def _offload(df,path,log):
+    with open(path, 'wb') as file:
+        pickle.dump(df, file)
+        log.write("Dumpping dataframe to : ", path)
+
+def _reload(path,log):
+    with open(path, 'rb') as file:
+        df =  pickle.load(file)
+        log.write("Loaded dataframe back from : ", path)
+    try:
+        os.remove(path)
+    except:
+        pass
+    return df

gwaslab/{qc_check_datatype_polars.py → qc/qc_check_datatype_polars.py}

@@ -56,7 +56,7 @@ dtype_dict ={
     'P_RANDOM' :[pl.Float64()]
     }
 
-def check_datatype(sumstats, verbose=True, log=Log()):
+def check_datatype_polars(sumstats, verbose=True, log=Log()):
     
     #try:
     headers = []
@@ -112,7 +112,7 @@ def quick_convert_datatype(sumstats, log, verbose):
                     pass
     return sumstats
 
-def check_dataframe_shape(sumstats, log, verbose):
+def check_dataframe_shape_polars(sumstats, log, verbose):
     memory_in_mb = sumstats.estimated_size(unit="mb") 
     try:
         log.write(" -Current Dataframe shape : {} x {} ; Memory usage: {:.2f} MB".format(len(sumstats),len(sumstats.columns),memory_in_mb), verbose=verbose)