gwaslab 3.4.35__tar.gz → 3.4.37__tar.gz

{gwaslab-3.4.35/src/gwaslab.egg-info → gwaslab-3.4.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gwaslab
-Version: 3.4.35
+Version: 3.4.37
 Summary: A collection of handy tools for GWAS SumStats
 Author-email: Yunye <yunye@gwaslab.com>
 Project-URL: Homepage, https://cloufield.github.io/gwaslab/
@@ -8,19 +8,18 @@ Project-URL: Github, https://github.com/Cloufield/gwaslab
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: <=3.10,>=3.8
+Requires-Python: <=3.10,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pandas!=1.5,!=2.0,>=1.3
+Requires-Dist: pandas!=1.5,>=1.3
 Requires-Dist: numpy>=1.21.2
 Requires-Dist: matplotlib!=3.7.2,>=3.5
 Requires-Dist: seaborn>=0.11.1
-Requires-Dist: scipy>=1.6.2
+Requires-Dist: scipy>=1.12
 Requires-Dist: pySAM<0.20,>=0.18.1
 Requires-Dist: Biopython>=1.79
 Requires-Dist: adjustText<=0.8,>=0.7.3
 Requires-Dist: liftover>=1.1.13
-Requires-Dist: statsmodels==0.13
 Requires-Dist: scikit-allel>=1.3.5
 Requires-Dist: pyensembl==2.2.3
 Requires-Dist: gtfparse==1.3.0
@@ -45,7 +44,7 @@ Note: GWASLab is being updated very frequently for now. I will release the first
 ## Install
 
 ```
-pip install gwaslab==3.4.32
+pip install gwaslab==3.4.35
 ```
 
 

{gwaslab-3.4.35 → gwaslab-3.4.37}/README.md

@@ -18,7 +18,7 @@ Note: GWASLab is being updated very frequently for now. I will release the first
 ## Install
 
 ```
-pip install gwaslab==3.4.32
+pip install gwaslab==3.4.35
 ```
 
 

{gwaslab-3.4.35 → gwaslab-3.4.37}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gwaslab"
-version = "3.4.35"
+version = "3.4.37"
 authors = [
   { name="Yunye", email="yunye@gwaslab.com" },
 ]
@@ -16,22 +16,21 @@ description = "A collection of handy tools for GWAS SumStats"
 readme = "README.md"
 
 dependencies = [
-    "pandas>=1.3,!=1.5,!=2.0",
+    "pandas>=1.3,!=1.5",
     "numpy>=1.21.2",
     "matplotlib>=3.5,!=3.7.2",
     "seaborn>=0.11.1",
-    "scipy>=1.6.2",
+    "scipy>=1.12",
     "pySAM>=0.18.1,<0.20",
     "Biopython>=1.79",
     "adjustText>=0.7.3, <=0.8",
     "liftover>=1.1.13",
-    "statsmodels==0.13",
     "scikit-allel>=1.3.5",
     "pyensembl==2.2.3",
     "gtfparse==1.3.0"
 ]
 
-requires-python = ">=3.8,<=3.10"
+requires-python = ">=3.9,<=3.10"
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",

{gwaslab-3.4.35 → gwaslab-3.4.37}/src/gwaslab/__init__.py

@@ -42,6 +42,6 @@ from gwaslab.util_in_calculate_power import get_power
 from gwaslab.util_in_calculate_power import get_beta
 from gwaslab.viz_plot_trumpetplot import plot_power
 from gwaslab.viz_plot_trumpetplot import plot_power_x
-from gwaslab.util_ex_process_h5 import process_ref_vcf
+from gwaslab.util_ex_process_h5 import process_vcf_to_hfd5
 from gwaslab.util_ex_run_susie import _run_susie_rss as run_susie_rss
 from gwaslab.io_read_tabular import _read_tabular as read_tabular

{gwaslab-3.4.35 → gwaslab-3.4.37}/src/gwaslab/bd_common_data.py

@@ -146,12 +146,14 @@ def get_number_to_NC(build,inverse=False):
 def get_NC_to_number(build):
     return get_number_to_NC(build=build,inverse=True)
 
-def get_chr_list(add_number=False,n=25):
+def get_chr_list(add_number=False,n=25,only_number=False):
     chrom_list=[str(i) for i in range(1,n+1)]+["X","Y","M","MT"]
     
-    if add_number is True:
+    if add_number == True:
         chrom_list = [str(i) for i in range(1,n+1)] + ["X","Y","M","MT"] + [i for i in range(1,n+1)]
 
+    if only_number ==True:
+        chrom_list = [i for i in range(1,n+1)]
     return chrom_list
 
 def get_chr_to_number(out_chr=False,xymt=["X","Y","MT"],xymt_num=[23,24,25]):

{gwaslab-3.4.35 → gwaslab-3.4.37}/src/gwaslab/g_Sumstats.py

@@ -52,7 +52,8 @@ from gwaslab.bd_common_data import get_format_dict
 from gwaslab.bd_common_data import get_formats_list
 from gwaslab.g_version import _show_version
 from gwaslab.g_version import gwaslab_info
-from gwaslab.g_meta import init_meta
+from gwaslab.g_meta import _init_meta
+from gwaslab.g_meta import _append_meta_record
 from gwaslab.util_ex_run_clumping import _clump
 from gwaslab.util_ex_calculate_ldmatrix import tofinemapping
 from gwaslab.util_ex_calculate_prs import _calculate_prs
@@ -60,6 +61,7 @@ from gwaslab.viz_plot_mqqplot import mqqplot
 from gwaslab.viz_plot_trumpetplot import plottrumpet
 from gwaslab.viz_plot_compare_af import plotdaf
 from gwaslab.util_ex_run_susie import _run_susie_rss
+from gwaslab.qc_fix_sumstats import _check_data_consistency
 import gc
 
 #20220309
@@ -119,10 +121,9 @@ class Sumstats():
         self.log = Log()
 
         # meta information
-        self.meta = init_meta() 
+        self.meta = _init_meta() 
         self.build = build
-        self.meta["gwaslab"]["study_name"] = study
-        #self.meta["gwaslab"]["genome_build"] = build
+        self.meta["gwaslab"]["study_name"] =  study
         self.meta["gwaslab"]["species"] = species
         
         # initialize attributes for clumping and finmapping
@@ -217,8 +218,22 @@ class Sumstats():
         return lookupstatus(self.data[status])
     
     def set_build(self, build, verbose=True):
-        self.data = _set_build(self.data, build=build, log=self.log,verbose=verbose)
+        self.data, self.meta["gwaslab"]["genome_build"] = _set_build(self.data, build=build, log=self.log,verbose=verbose)
         gc.collect()
+
+    def infer_build(self,**args):
+        self.data, self.meta["gwaslab"]["genome_build"] = inferbuild(self.data,**args)
+    
+    def liftover(self,to_build, from_build=None,**args):
+        if from_build is None:
+            if self.meta["gwaslab"]["genome_build"]=="99":
+                self.data, self.meta["gwaslab"]["genome_build"] = inferbuild(self.data,**args)
+            from_build = self.meta["gwaslab"]["genome_build"]
+        self.data = parallelizeliftovervariant(self.data,from_build=from_build, to_build=to_build, log=self.log,**args)
+        self.meta["is_sorted"] = False
+        self.meta["is_harmonised"] = False
+        self.meta["gwaslab"]["genome_build"]=to_build
+
 # QC ######################################################################################
     #clean the sumstats with one line
     def basic_check(self,
@@ -231,20 +246,23 @@ class Sumstats():
                     fixpos_args={},
                     fixallele_args={},
                     sanitycheckstats_args={},
+                    consistencycheck_args={},
                     normalize=True,
                     normalizeallele_args={},
                     verbose=True):
         ###############################################
         # try to fix data without dropping any information
         self.data = fixID(self.data,verbose=verbose, **fixid_args)
-        if remove_dup is True:
-            self.data = removedup(self.data,log=self.log,verbose=verbose,**removedup_args)
         self.data = fixchr(self.data,log=self.log,remove=remove,verbose=verbose,**fixchr_agrs)
         self.data = fixpos(self.data,log=self.log,remove=remove,verbose=verbose,**fixpos_args)
         self.data = fixallele(self.data,log=self.log,remove=remove,verbose=verbose,**fixallele_args)
         self.data = sanitycheckstats(self.data,log=self.log,verbose=verbose,**sanitycheckstats_args)
+        _check_data_consistency(self.data,log=self.log,verbose=verbose,**consistencycheck_args)
+        
         if normalize is True:
             self.data = parallelnormalizeallele(self.data,n_cores=n_cores,verbose=verbose,log=self.log,**normalizeallele_args)
+        if remove_dup is True:
+            self.data = removedup(self.data,log=self.log,verbose=verbose,**removedup_args)
         self.data = sortcoordinate(self.data,verbose=verbose,log=self.log)
         self.data = sortcolumn(self.data,verbose=verbose,log=self.log)
         self.meta["is_sorted"] = True
@@ -329,9 +347,9 @@ class Sumstats():
             
             self.data= parallelinferstrand(self.data,ref_infer = ref_infer,ref_alt_freq=ref_alt_freq,maf_threshold=maf_threshold,
                                               n_cores=n_cores,log=self.log,**inferstrand_args)
-            
-            self.meta["gwaslab"]["references"]["ref_infer"] = ref_infer
 
+            self.meta["gwaslab"]["references"]["ref_infer"] = _append_meta_record(self.meta["gwaslab"]["references"]["ref_infer"] , ref_infer)
+            
             self.data =flipallelestats(self.data,log=self.log,**flipallelestats_args)
             
             gc.collect()
@@ -341,13 +359,18 @@ class Sumstats():
             
             self.data = parallelizeassignrsid(self.data,path=ref_rsid_tsv,ref_mode="tsv",
                                                  n_cores=n_cores,log=self.log,**assignrsid_args)
+            
+            
+
             self.meta["gwaslab"]["references"]["ref_rsid_tsv"] = ref_rsid_tsv
             gc.collect()
+
         if ref_rsid_vcf is not None:
-            
             self.data = parallelizeassignrsid(self.data,path=ref_rsid_vcf,ref_mode="vcf",
                                                  n_cores=n_cores,log=self.log,**assignrsid_args)   
-            self.meta["gwaslab"]["references"]["ref_rsid_vcf"] = ref_rsid_vcf
+
+            self.meta["gwaslab"]["references"]["ref_rsid_vcf"] = _append_meta_record(self.meta["gwaslab"]["references"]["ref_rsid_vcf"] , ref_rsid_vcf)
+            
             gc.collect()
         ######################################################    
         if remove is True:
@@ -376,17 +399,23 @@ class Sumstats():
         self.data = removedup(self.data,log=self.log,**args)
     def check_sanity(self,**args):
         self.data = sanitycheckstats(self.data,log=self.log,**args)
-    # 
+    def check_data_consistency(self, **args):
+        _check_data_consistency(self.data,log=self.log,**args)
     def check_id(self,**args):
         pass
-    def check_ref(self,**args):
-        self.data = checkref(self.data,log=self.log,**args)
-    def infer_strand(self,**args):
-        self.data = parallelinferstrand(self.data,log=self.log,**args)
+    
+    def check_ref(self,ref_seq,**args):
+        self.meta["gwaslab"]["references"]["ref_seq"] = ref_seq
+        self.data = checkref(self.data,ref_seq,log=self.log,**args)
+    def infer_strand(self,ref_infer,**args):
+        self.meta["gwaslab"]["references"]["ref_infer"] = _append_meta_record(self.meta["gwaslab"]["references"]["ref_infer"] , ref_infer)
+        self.data = parallelinferstrand(self.data,ref_infer=ref_infer,log=self.log,**args)
+    
     def flip_allele_stats(self,**args):
         self.data = flipallelestats(self.data,log=self.log,**args)
     def normalize_allele(self,**args):
         self.data = parallelnormalizeallele(self.data,log=self.log,**args)
+    
     def assign_rsid(self,
                     ref_rsid_tsv=None,
                     ref_rsid_vcf=None,
@@ -396,21 +425,15 @@ class Sumstats():
             self.meta["gwaslab"]["references"]["ref_rsid_tsv"] = ref_rsid_tsv
         if ref_rsid_vcf is not None:
             self.data = parallelizeassignrsid(self.data,path=ref_rsid_vcf,ref_mode="vcf",log=self.log,**args)   
-            self.meta["gwaslab"]["references"]["ref_rsid_vcf"] = ref_rsid_vcf
+            self.meta["gwaslab"]["references"]["ref_rsid_vcf"] = _append_meta_record(self.meta["gwaslab"]["references"]["ref_rsid_vcf"] , ref_rsid_vcf)
+    
     def rsid_to_chrpos(self,**args):
         self.data = rsidtochrpos(self.data,log=self.log,**args)
+        
     def rsid_to_chrpos2(self,**args):
         self.data = parallelrsidtochrpos(self.data,log=self.log,**args)
 
-    def liftover(self,to_build, from_build=None,**args):
-        if from_build is None:
-            if self.meta["gwaslab"]["genome_build"]=="99":
-                self.data, self.meta["gwaslab"]["genome_build"] = inferbuild(self.data,**args)
-            from_build = self.meta["gwaslab"]["genome_build"]
-        self.data = parallelizeliftovervariant(self.data,from_build=from_build, to_build=to_build, log=self.log,**args)
-        self.meta["is_sorted"] = False
-        self.meta["is_harmonised"] = False
-        self.meta["gwaslab"]["genome_build"]=to_build
+
     ############################################################################################################
     
     def sort_coordinate(self,**sort_args):
@@ -420,11 +443,10 @@ class Sumstats():
         self.data = sortcolumn(self.data,log=self.log,**args)
     
     ############################################################################################################
-    def fill_data(self, **args):
-        self.data = filldata(self.data,**args)
-    
-    def infer_build(self,**args):
-        self.data, self.meta["gwaslab"]["genome_build"] = inferbuild(self.data,**args)
+    def fill_data(self, verbose=True, **args):
+        self.data = filldata(self.data, verbose=verbose, **args)
+        self.data = sortcolumn(self.data, verbose=verbose, log=self.log)
+
 # utilities ############################################################################################################
     # filter series ######################################################################
     def get_flanking(self, inplace=False,**args):
@@ -485,11 +507,12 @@ class Sumstats():
     
     def check_af(self,ref_infer,**args):
         self.data = parallelecheckaf(self.data,ref_infer=ref_infer,log=self.log,**args)
-        self.meta["gwaslab"]["references"]["ref_infer_daf"] = ref_infer
-    
+        self.meta["gwaslab"]["references"]["ref_infer_daf"] = _append_meta_record(self.meta["gwaslab"]["references"]["ref_infer_daf"] , ref_infer)
+        
     def infer_af(self,ref_infer,**args):
         self.data = paralleleinferaf(self.data,ref_infer=ref_infer,log=self.log,**args)
         self.meta["gwaslab"]["references"]["ref_infer_af"] = ref_infer
+        self.meta["gwaslab"]["references"]["ref_infer_af"] = _append_meta_record(self.meta["gwaslab"]["references"]["ref_infer_af"] , ref_infer)
       
     def plot_daf(self, **args):
         fig,outliers = plotdaf(self.data, **args)

{gwaslab-3.4.35 → gwaslab-3.4.37}/src/gwaslab/g_meta.py

@@ -1,6 +1,6 @@
 from gwaslab.g_version import gwaslab_info
 
-def init_meta():
+def _init_meta():
     metadata = {"gwaslab":{
                         "gwaslab_version": gwaslab_info()["version"],
                         "study_name":"Sumstats_1",
@@ -23,7 +23,11 @@ def init_meta():
                             "ref_rsid_tsv":"Unknown",
                             "ref_rsid_vcf":"Unknown",
                             "ref_seq":"Unknown",
-                            "ref_infer":"Unknown"
+                            "ref_infer":"Unknown",
+                            "ref_infer_af":"Unknown",
+                            "ref_infer_daf":"Unknown",
+                            "ref_rsid_to_chrpos_tsv":"Unknown",
+                            "ref_rsid_to_chrpos_vcf":"Unknown"
                         }
                     },
                      "genotyping_technology":"Unknown", 
@@ -45,4 +49,10 @@ def init_meta():
                      "coordinate_system":"1-based",
                      "sex": "M|F|combined"
                      }
-    return metadata.copy()
+    return metadata.copy()
+
+def _append_meta_record(old, new):
+    if old == "Unknown" or old== "Unchecked":
+        return new
+    else:
+        return "{}, {}".format(old, new)

{gwaslab-3.4.35 → gwaslab-3.4.37}/src/gwaslab/g_version.py

@@ -15,8 +15,8 @@ def _get_version():
 def gwaslab_info():
     # version meta information
     dic={
-   "version":"3.4.35",
-   "release_date":"20240123"
+   "version":"3.4.37",
+   "release_date":"20240129"
     }
     return dic   
 

{gwaslab-3.4.35 → gwaslab-3.4.37}/src/gwaslab/hm_harmonize_sumstats.py

@@ -11,6 +11,8 @@ import gc
 from gwaslab.g_Log import Log
 from gwaslab.qc_fix_sumstats import fixchr
 from gwaslab.qc_fix_sumstats import fixpos
+from gwaslab.qc_fix_sumstats import sortcolumn
+from gwaslab.qc_check_datatype import check_dataframe_shape
 from gwaslab.bd_common_data import get_number_to_chr
 from gwaslab.bd_common_data import get_chr_list
 from gwaslab.bd_common_data import get_chr_to_number
@@ -27,7 +29,7 @@ from gwaslab.g_version import _get_version
 
 ###~!!!!
 def rsidtochrpos(sumstats,
-         path="", snpid="SNPID",
+         path=None, ref_rsid_to_chrpos_tsv=None, snpid="SNPID",
          rsid="rsID", chrom="CHR",pos="POS",ref_rsid="rsID",ref_chr="CHR",ref_pos="POS", build="19",
               overwrite=False,remove=False,chunksize=5000000,verbose=True,log=Log()):
     '''
@@ -35,9 +37,12 @@ def rsidtochrpos(sumstats,
     '''
     #########################################################################################################
     if verbose:  log.write("Start to update chromosome and position information based on rsID...{}".format(_get_version()))  
-    if verbose:  log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns))   
+    check_dataframe_shape(sumstats, log, verbose)
     if verbose:  log.write(" -rsID dictionary file: "+ path)  
     
+    if ref_rsid_to_chrpos_tsv is not None:
+        path = ref_rsid_to_chrpos_tsv
+
     if snpid in sumstats.columns and sum(sumstats[rsid].isna())>0:
         if verbose:  log.write(" -Filling na in rsID columns with SNPID...")  
         sumstats.loc[sumstats[rsid].isna(),rsid] = sumstats.loc[sumstats[rsid].isna(),snpid]
@@ -75,6 +80,7 @@ def rsidtochrpos(sumstats,
     if verbose:  log.write(" -Updating CHR and POS finished.Start to re-fixing CHR and POS... ")
     sumstats = fixchr(sumstats,verbose=verbose)
     sumstats = fixpos(sumstats,verbose=verbose)
+    sumstats = sortcolumn(sumstats,verbose=verbose)
     return sumstats
     #################################################################################################### 
 
@@ -96,9 +102,19 @@ def merge_chrpos(sumstats_part,all_groups_max,path,build,status):
     return sumstats_part
 
 
-def parallelrsidtochrpos(sumstats, rsid="rsID", chrom="CHR",pos="POS", path=None,build="99",status="STATUS",
+def parallelrsidtochrpos(sumstats, rsid="rsID", chrom="CHR",pos="POS", path=None, ref_rsid_to_chrpos_vcf = None, ref_rsid_to_chrpos_hdf5 = None, build="99",status="STATUS",
                          n_cores=4,block_size=20000000,verbose=True,log=Log()):
+    
+    if ref_rsid_to_chrpos_hdf5 is not None:
+        path = ref_rsid_to_chrpos_hdf5
+    elif ref_rsid_to_chrpos_vcf is not None:
+        vcf_file_name = os.path.basename(ref_rsid_to_chrpos_vcf)
+        vcf_dir_path = os.path.dirname(ref_rsid_to_chrpos_vcf)
+        path = "{}/{}.rsID_CHR_POS_groups_{}.h5".format(vcf_dir_path,vcf_file_name,int(block_size))
+
     if verbose:  log.write("Start to assign CHR and POS using rsIDs...{}".format(_get_version()))
+    check_dataframe_shape(sumstats, log, verbose)
+
     if path is None:
         raise ValueError("Please provide path to hdf5 file.")
     
@@ -164,13 +180,16 @@ def parallelrsidtochrpos(sumstats, rsid="rsID", chrom="CHR",pos="POS", path=None
     # merge back
     if verbose:  log.write(" -Append data... ")
     sumstats = pd.concat([sumstats_rs,sumstats_nonrs],ignore_index=True)
+    
     del sumstats_rs
     del sumstats_nonrs
     gc.collect()
     
     # check
-    sumstats = fixchr(sumstats,verbose=True)
-    sumstats = fixpos(sumstats,verbose=True)
+    sumstats = fixchr(sumstats,verbose=verbose)
+    sumstats = fixpos(sumstats,verbose=verbose)
+    sumstats = sortcolumn(sumstats,verbose=verbose)
+
     pool.close()
     pool.join()
     gc.collect()
@@ -235,7 +254,7 @@ def check_status(row,record):
 
 def checkref(sumstats,ref_path,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="STATUS",chr_dict=get_chr_to_number(),remove=False,verbose=True,log=Log()):
     if verbose: log.write("Start to check if NEA is aligned with reference sequence...{}".format(_get_version()))
-    if verbose: log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns)) 
+    check_dataframe_shape(sumstats, log, verbose)
     if verbose: log.write(" -Reference genome fasta file: "+ ref_path)  
     if verbose: log.write(" -Checking records: ", end="")  
     chromlist = get_chr_list(add_number=True)
@@ -381,7 +400,7 @@ def parallelizeassignrsid(sumstats, path, ref_mode="vcf",snpid="SNPID",rsid="rsI
         assign rsID based on chr:pos
         '''
         if verbose:  log.write("Start to annotate rsID based on chromosome and position information...{}".format(_get_version()))  
-        if verbose:  log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns))   
+        check_dataframe_shape(sumstats, log, verbose)
         if verbose:  log.write(" -SNPID-rsID text file: "+ path)  
         
         standardized_normalized = sumstats["STATUS"].str.match("\w\w\w[0][01234][0126]\w", case=False, flags=0, na=False)
@@ -517,7 +536,7 @@ def parallelinferstrand(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.40,
                        chr="CHR",pos="POS",ref="NEA",alt="EA",eaf="EAF",status="STATUS",
                        chr_dict=None,verbose=True,log=Log()):
     if verbose: log.write("Start to infer strand for palindromic SNPs...{}".format(_get_version()))
-    if verbose: log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns))   
+    check_dataframe_shape(sumstats, log, verbose)
     if verbose: log.write(" -Reference vcf file:", ref_infer)   
 
     chr_dict = auto_check_vcf_chr_dict(ref_infer, chr_dict, verbose, log)
@@ -528,8 +547,8 @@ def parallelinferstrand(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.40,
     if "p" in mode:
         # ref_alt_freq INFO in vcf was provided
         if ref_alt_freq is not None:
+            
             if verbose: log.write(" -Alternative allele frequency in INFO:", ref_alt_freq)  
-
             ## checking \w\w\w\w[0]\w\w -> standardized and normalized snp
             good_chrpos =  sumstats[status].str.match(r'\w\w\w[0][0]\w\w', case=False, flags=0, na=False) 
             palindromic = good_chrpos & is_palindromic(sumstats[[ref,alt]],a1=ref,a2=alt)   
@@ -538,22 +557,28 @@ def parallelinferstrand(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.40,
             ##not palindromic : change status
             sumstats.loc[not_palindromic_snp,status] = vchange_status(sumstats.loc[not_palindromic_snp,status], 7 ,"9","0")  
             if verbose: log.write(" -Identified ", sum(palindromic)," palindromic SNPs...")
-
+            
             #palindromic but can not infer
-            maf_can_infer   = (sumstats.loc[:,eaf] < maf_threshold) | (sumstats.loc[:,eaf] > 1 - maf_threshold) 
+            maf_can_infer   = (sumstats.loc[:,eaf] < maf_threshold) | (sumstats.loc[:,eaf] > 1 - maf_threshold)
+            
             sumstats.loc[palindromic&(~maf_can_infer),status] = vchange_status(sumstats.loc[palindromic&(~maf_can_infer),status],7,"9","7")
+            
+            #palindromic WITH UNKNWON OR UNCHECKED STATUS
+            unknow_palindromic = sumstats[status].str.match(r'\w\w\w\w\w[012][89]', case=False, flags=0, na=False) 
 
+            unknow_palindromic_to_check = palindromic & maf_can_infer & unknow_palindromic
+            
+            if verbose: log.write(" -After filtering by MAF< {} , {} palindromic SNPs with unknown strand will be inferred...".format(maf_threshold, sum(unknow_palindromic_to_check)))
 
-            if verbose: log.write(" -After filtering by MAF< ", maf_threshold ," , the strand of ", sum(palindromic & maf_can_infer)," palindromic SNPs will be inferred...")
             ######################################################################################### 
-            if sum(palindromic & maf_can_infer)>0:
-                if sum(palindromic & maf_can_infer)<10000: 
+            if sum(unknow_palindromic_to_check)>0:
+                if sum(unknow_palindromic_to_check)<10000: 
                     n_cores=1  
-                df_split = np.array_split(sumstats.loc[(palindromic & maf_can_infer),[chr,pos,ref,alt,eaf,status]], n_cores)
+                df_split = np.array_split(sumstats.loc[unknow_palindromic_to_check,[chr,pos,ref,alt,eaf,status]], n_cores)
                 pool = Pool(n_cores)
                 map_func = partial(check_strand,chr=chr,pos=pos,ref=ref,alt=alt,eaf=eaf,status=status,ref_infer=ref_infer,ref_alt_freq=ref_alt_freq,chr_dict=chr_dict) 
                 status_inferred = pd.concat(pool.map(map_func,df_split))
-                sumstats.loc[(palindromic & maf_can_infer),status] = status_inferred.values
+                sumstats.loc[unknow_palindromic_to_check,status] = status_inferred.values
             pool.close()
             pool.join()
             #########################################################################################
@@ -650,7 +675,7 @@ def parallelinferstrand(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.40,
 def parallelecheckaf(sumstats,ref_infer,ref_alt_freq=None,maf_threshold=0.4,column_name="DAF",suffix="",n_cores=1, chr="CHR",pos="POS",ref="NEA",alt="EA",eaf="EAF",status="STATUS",chr_dict=None,force=False, verbose=True,log=Log()):
         
     if verbose: log.write("Start to check the difference between EAF and reference vcf alt frequency ...{}".format(_get_version()))
-    if verbose: log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns))   
+    check_dataframe_shape(sumstats, log, verbose)  
     if verbose: log.write(" -Reference vcf file:", ref_infer)   
     if verbose: log.write(" -CPU Cores to use :",n_cores)
     
@@ -718,7 +743,7 @@ def check_daf(chr,start,end,ref,alt,eaf,vcf_reader,alt_freq,chr_dict=None):
 def paralleleinferaf(sumstats,ref_infer,ref_alt_freq=None,n_cores=1, chr="CHR",pos="POS",ref="NEA",alt="EA",eaf="EAF",status="STATUS",chr_dict=None,force=False, verbose=True,log=Log()):
         
     if verbose: log.write("Start to infer the AF and reference vcf alt frequency ...{}".format(_get_version()))
-    if verbose: log.write(" -Current Dataframe shape :",len(sumstats)," x ", len(sumstats.columns))   
+    check_dataframe_shape(sumstats, log, verbose)   
     if verbose: log.write(" -Reference vcf file:", ref_infer)   
     if verbose: log.write(" -CPU Cores to use :",n_cores)
     

{gwaslab-3.4.35 → gwaslab-3.4.37}/src/gwaslab/io_preformat_input.py

@@ -8,6 +8,7 @@ from gwaslab.bd_common_data import get_format_dict
 from gwaslab.qc_fix_sumstats import sortcolumn
 from gwaslab.qc_fix_sumstats import _process_build
 from gwaslab.qc_check_datatype import check_datatype
+from gwaslab.qc_check_datatype import check_dataframe_memory_usage
 
 #20221030
 def preformat(sumstats,
@@ -353,6 +354,8 @@ def preformat(sumstats,
     sumstats = sortcolumn(sumstats=sumstats,log=log,verbose=verbose)    
     check_datatype(sumstats,log=log,verbose=verbose)
     gc.collect()
+    check_dataframe_memory_usage(sumstats,log=log,verbose=verbose)
+    
     if verbose: log.write("Finished loading data successfully!")
     return sumstats
 
@@ -457,13 +460,18 @@ def process_neaf(sumstats,log,verbose):
     return sumstats
 
 def process_allele(sumstats,log,verbose):
+    
     if "EA" in sumstats.columns:
+
         if "REF" in sumstats.columns and "ALT" in sumstats.columns:
+
             if "NEA" not in sumstats.columns:
                 if verbose: log.write(" NEA not available: assigning REF to NEA...") 
                 sumstats["NEA"]=sumstats["REF"]    
+            
             if verbose: log.write(" -EA,REF and ALT columns are available: assigning NEA...") 
             ea_alt = sumstats["EA"]==sumstats["ALT"]
+            
             if verbose: log.write(" -For variants with EA == ALT : assigning REF to NEA ...") 
             sumstats.loc[ea_alt,"NEA"] = sumstats.loc[ea_alt,"REF"]
             
@@ -474,6 +482,7 @@ def process_allele(sumstats,log,verbose):
             #sumstats = sumstats.drop(labels=["REF","ALT"],axis=1)
             sumstats["REF"]=sumstats["REF"].astype("category") 
             sumstats["ALT"]=sumstats["ALT"].astype("category") 
+            
         sumstats["EA"]=sumstats["EA"].astype("category")     
     if "NEA" in sumstats.columns:
         sumstats["NEA"]=sumstats["NEA"].astype("category")  

{gwaslab-3.4.35 → gwaslab-3.4.37}/src/gwaslab/qc_check_datatype.py

@@ -87,3 +87,17 @@ def verify_datatype(header, dtype):
             return "F"
     else:
         return "NA"
+
+def check_dataframe_shape(sumstats, log, verbose):
+    memory_in_mb = sumstats.memory_usage().sum()/1024/1024
+    try:
+        log.write(" -Current Dataframe shape : {} x {} ; Memory usage: {:.2f} MB".format(len(sumstats),len(sumstats.columns),memory_in_mb), verbose=verbose)
+    except:
+        log.write(" -WARNING! Error: cannot get Dataframe shape...", verbose=verbose)
+    
+def check_dataframe_memory_usage(sumstats, log, verbose):
+    memory_in_mb = sumstats.memory_usage().sum()/1024/1024
+    try:
+        log.write(" -Current Dataframe memory usage: {:.2f} MB".format(memory_in_mb), verbose=verbose)
+    except:
+        log.write(" -WARNING! Error: cannot get Memory usage...", verbose=verbose)