gwaslab 3.4.48__py3-none-any.whl → 3.4.49__py3-none-any.whl

gwaslab/bd_common_data.py

@@ -274,7 +274,9 @@ def get_gtf(chrom, build="19",source="ensembl"):
         gtf = pd.DataFrame(columns=["seqname","start","end","strand","feature","gene_biotype","gene_id","gene_name"])
     return gtf
 
-
+def get_chain(from_build="19", to_build="38"):    
+    chain_path = check_and_download("{}to{}".format(from_build, to_build))
+    return chain_path
 ####################################################################################################################
 def gtf_to_protein_coding(gtfpath,log=Log(),verbose=True):
     protein_coding_path = gtfpath[:-6]+"protein_coding.gtf.gz"

gwaslab/data/reference.json

@@ -90,12 +90,20 @@
     "1kg_dbsnp151_hg38_x_md5":"48c05eeb1454c0dd4cbee3cb26382e8e",
     "recombination_hg19":"https://www.dropbox.com/s/wbesl8haxknonuc/recombination_hg19.tar.gz?dl=1",
     "recombination_hg38":"https://www.dropbox.com/s/vuo8mvqx0fpibzj/recombination_hg38.tar.gz?dl=1",
-    "ensembl_hg19_gtf":"https://ftp.ensembl.org/pub/grch37/current/gtf/homo_sapiens/Homo_sapiens.GRCh37.87.chr.gtf.gz",
+    "ensembl_hg19_gtf":"https://ftp.ensembl.org/pub/grch37/release-87/gtf/homo_sapiens/Homo_sapiens.GRCh37.87.chr.gtf.gz",
     "ensembl_hg38_gtf":"https://ftp.ensembl.org/pub/release-109/gtf/homo_sapiens//Homo_sapiens.GRCh38.109.chr.gtf.gz",
     "refseq_hg19_gtf":"https://ftp.ncbi.nlm.nih.gov/refseq/H_sapiens/annotation/GRCh37_latest/refseq_identifiers/GRCh37_latest_genomic.gtf.gz",
     "refseq_hg38_gtf":"https://ftp.ncbi.nlm.nih.gov/refseq/H_sapiens/annotation/GRCh38_latest/refseq_identifiers/GRCh38_latest_genomic.gtf.gz",
     "testlink":"https://www.dropbox.com/s/8u7capwge0ihshu/EAS.chr22.split_norm_af.1kgp3v5.vcf.gz?dl=1",
-    "testlink_tbi":"https://www.dropbox.com/s/hdneg53t6u1j6ib/EAS.chr22.split_norm_af.1kgp3v5.vcf.gz.tbi?dl=1"
+    "testlink_tbi":"https://www.dropbox.com/s/hdneg53t6u1j6ib/EAS.chr22.split_norm_af.1kgp3v5.vcf.gz.tbi?dl=1",
+    "19to38":"https://hgdownload.soe.ucsc.edu/goldenPath/hg19/liftOver/hg19ToHg38.over.chain.gz",
+    "19to13":"https://s3-us-west-2.amazonaws.com/human-pangenomics/T2T/CHM13/assemblies/chain/v1_nflo/hg19-chm13v2.chain",
+    "38to19":"https://hgdownload.soe.ucsc.edu/goldenPath/hg38/liftOver/hg38ToHg19.over.chain.gz",
+    "38to13":"https://s3-us-west-2.amazonaws.com/human-pangenomics/T2T/CHM13/assemblies/chain/v1_nflo/grch38-chm13v2.chain",
+    "13to19":"https://s3-us-west-2.amazonaws.com/human-pangenomics/T2T/CHM13/assemblies/chain/v1_nflo/chm13v2-hg19.chain",
+    "13to38":"https://s3-us-west-2.amazonaws.com/human-pangenomics/T2T/CHM13/assemblies/chain/v1_nflo/chm13v2-grch38.chain",
+    "18to19":"https://hgdownload.soe.ucsc.edu/goldenPath/hg18/liftOver/hg18ToHg19.over.chain.gz",
+    "18to38":"https://hgdownload.soe.ucsc.edu/goldenPath/hg18/liftOver/hg18ToHg38.over.chain.gz"
 }
 
 

gwaslab/g_Sumstats.py

@@ -8,6 +8,8 @@ from gwaslab.io_preformat_input import preformat
 from gwaslab.io_to_formats import _to_format
 from gwaslab.g_Log import Log
 from gwaslab.qc_fix_sumstats import fixID
+from gwaslab.qc_fix_sumstats import flipSNPID
+from gwaslab.qc_fix_sumstats import stripSNPID
 from gwaslab.qc_fix_sumstats import removedup
 from gwaslab.qc_fix_sumstats import fixchr
 from gwaslab.qc_fix_sumstats import fixpos
@@ -123,6 +125,8 @@ class Sumstats():
              snpr2=None,
              status=None,
              other=[],
+             chrom_pat=None,
+             snpid_pat=None,
              usekeys=None,
              direction=None,
              verbose=True,
@@ -205,6 +209,8 @@ class Sumstats():
           status=status,
           other=other,
           usekeys=usekeys,
+          chrom_pat=chrom_pat,
+          snpid_pat=snpid_pat,
           verbose=verbose,
           readargs=readargs,
           log=self.log)
@@ -418,6 +424,10 @@ class Sumstats():
     #customizable API to build your own QC pipeline
     def fix_id(self,**kwargs):
         self.data = fixID(self.data,log=self.log,**kwargs)
+    def flip_snpid(self,**kwargs):
+        self.data = flipSNPID(self.data,log=self.log,**kwargs)
+    def strip_snpid(self,**kwargs):
+        self.data = stripSNPID(self.data,log=self.log,**kwargs)
     def fix_chr(self,**kwargs):
         self.data = fixchr(self.data,log=self.log,**kwargs)
     def fix_pos(self,**kwargs):

gwaslab/g_vchange_status.py

@@ -1,6 +1,6 @@
 import pandas as pd
 
-CATEGORIES = {str(j+i) for j in [1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
+CATEGORIES = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
 
 def vchange_status(status,digit,before,after):
     dic={}

gwaslab/g_version.py

@@ -15,8 +15,8 @@ def _get_version():
 def gwaslab_info():
     # version meta information
     dic={
-   "version":"3.4.48",
-   "release_date":"20240822"
+   "version":"3.4.49",
+   "release_date":"20241019"
     }
     return dic   
 

gwaslab/hm_harmonize_sumstats.py

@@ -21,6 +21,7 @@ from gwaslab.qc_check_datatype import check_dataframe_shape
 from gwaslab.bd_common_data import get_number_to_chr
 from gwaslab.bd_common_data import get_chr_list
 from gwaslab.bd_common_data import get_chr_to_number
+from gwaslab.bd_common_data import get_number_to_NC
 from gwaslab.bd_common_data import _maketrans
 from gwaslab.g_vchange_status import vchange_status
 from gwaslab.g_version import _get_version
@@ -355,7 +356,7 @@ def oldcheckref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status=
     
     log.write("\n",end="",show_time=False,verbose=verbose) 
         
-    CATEGORIES = {str(j+i) for j in [1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
+    CATEGORIES = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
     sumstats[status] = pd.Categorical(sumstats[status],categories=CATEGORIES)
     #sumstats[status] = sumstats[status].astype("string")
 
@@ -674,7 +675,7 @@ def checkref(sumstats,ref_seq,chrom="CHR",pos="POS",ea="EA",nea="NEA",status="ST
         sumstats.loc[to_check_ref,status] = check_status(sumstats_to_check, all_records_dict, log=log, verbose=verbose)
         log.write(" -Finished checking records", verbose=verbose) 
     
-    CATEGORIES = {str(j+i) for j in [1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
+    CATEGORIES = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
     sumstats[status] = pd.Categorical(sumstats[status],categories=CATEGORIES)
     #sumstats[status] = sumstats[status].astype("string")
 
@@ -1496,17 +1497,21 @@ def infer_af(chr,start,end,ref,alt,vcf_reader,alt_freq,chr_dict=None):
 def auto_check_vcf_chr_dict(vcf_path, vcf_chr_dict, verbose, log):    
     if vcf_path is not None:
         if vcf_chr_dict is None:
-            log.write(" -Checking prefix for chromosomes in vcf files..." ,verbose=verbose)  
-            prefix = check_vcf_chr_prefix(vcf_path) 
+            log.write(" -Checking chromosome notations in VCF/BCF files..." ,verbose=verbose)  
+            vcf_chr_dict = check_vcf_chr_NC(vcf_path, log, verbose)
+            if vcf_chr_dict is not None:
+                return vcf_chr_dict
+            log.write(" -Checking prefix for chromosomes in VCF/BCF files..." ,verbose=verbose)  
+            prefix = check_vcf_chr_prefix(vcf_path, log,verbose) 
             if prefix is not None:
                 log.write(" -Prefix for chromosomes: ",prefix) 
                 vcf_chr_dict = get_number_to_chr(prefix=prefix)
             else:
-                log.write(" -No prefix for chromosomes in the VCF files." ,verbose=verbose)  
+                log.write(" -No prefix for chromosomes in the VCF/BCF files." ,verbose=verbose)  
                 vcf_chr_dict = get_number_to_chr()
     return vcf_chr_dict
 
-def check_vcf_chr_prefix(vcf_bcf_path):
+def check_vcf_chr_prefix(vcf_bcf_path,log,verbose):
     vcf_bcf = VariantFile(vcf_bcf_path)
     for i in list(vcf_bcf.header.contigs):
         m = re.search('(chr|Chr|CHR)([0-9xXyYmM]+)', i)
@@ -1514,5 +1519,16 @@ def check_vcf_chr_prefix(vcf_bcf_path):
             return m.group(1)
     else:
         return None
-    
+
+def check_vcf_chr_NC(vcf_bcf_path,log,verbose):
+    vcf_bcf = VariantFile(vcf_bcf_path)
+    for i in list(vcf_bcf.header.contigs):
+        if i in get_number_to_NC(build="19").values():
+            log.write("  -RefSeq ID detected (hg19) in VCF/BCF...",verbose=verbose) 
+            return get_number_to_NC(build="19")
+        elif i in get_number_to_NC(build="38").values():
+            log.write("  -RefSeq ID detected (hg38) in VCF/BCF...",verbose=verbose) 
+            return get_number_to_NC(build="38")
+    else:
+        return None
 

gwaslab/io_preformat_input.py

@@ -56,6 +56,8 @@ def preformat(sumstats,
           build=None,
           other=[],
           usekeys=None,
+          chrom_pat=None,
+          snpid_pat=None,
           verbose=False,
           readargs=None,
           log=None):
@@ -84,7 +86,10 @@ def preformat(sumstats,
         if "format_separator" in meta_data.keys():
             if "sep" not in readargs.keys():
                 readargs["sep"] = meta_data["format_separator"]
-        
+            else:
+                if readargs["sep"] != meta_data["format_separator"]:
+                    log.write('  - format_separator will be changed to: "{}"'.format(readargs["sep"]),verbose=verbose)
+
         if "format_na" in meta_data.keys():
             readargs["na_values"] = meta_data["format_na"]
         
@@ -92,7 +97,7 @@ def preformat(sumstats,
             readargs["comment"] = meta_data["format_comment"]
         
         if "sep" not in readargs.keys():
-             readargs["sep"] = "\t"
+            readargs["sep"] = "\t"
         
 #########################################################################################################################################################      
     
@@ -323,10 +328,30 @@ def preformat(sumstats,
                 skip_rows = get_skip_rows(inpath)
                 readargs["skiprows"] = skip_rows
                 log.write("Start to initialize gl.Sumstats from file :" + inpath,verbose=verbose)
-                sumstats = pd.read_table(inpath,
-                                 usecols=set(usecols),
-                                 dtype=dtype_dictionary,
-                                 **readargs)
+                if chrom_pat is not None:
+                    sumstats = _load_single_chr(inpath,
+                                                usecols,
+                                                dtype_dictionary,
+                                                readargs=readargs,
+                                                rename_dictionary=rename_dictionary,
+                                                chrom_pat=chrom_pat,
+                                                log=log,
+                                                verbose=verbose)
+                elif snpid_pat is not None: 
+                                
+                    sumstats = _load_variants_with_pattern(inpath,
+                                                usecols,
+                                                dtype_dictionary,
+                                                readargs=readargs,
+                                                rename_dictionary=rename_dictionary,
+                                                snpid_pat=snpid_pat,
+                                                log=log,
+                                                verbose=verbose)
+                else:
+                    sumstats = pd.read_table(inpath,
+                                    usecols=set(usecols),
+                                    dtype=dtype_dictionary,
+                                    **readargs)
 
         elif type(sumstats) is pd.DataFrame:
             ## loading data from dataframe
@@ -520,9 +545,49 @@ def process_status(sumstats,build,log,verbose):
     #sumstats["STATUS"] = int(build)*(10**5) +99999
     build = _process_build(build,log,verbose)
     sumstats["STATUS"] = build +"99999"
-    categories = {str(j+i) for j in [1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
+    categories = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
     sumstats["STATUS"] = pd.Categorical(sumstats["STATUS"],categories=categories)
     return sumstats
 
 
-    
+def _load_single_chr(inpath,usecols,dtype_dictionary,readargs,rename_dictionary,chrom_pat,log,verbose):
+    
+    sumstats_iter = pd.read_table(inpath,
+                usecols=set(usecols),
+                dtype=dtype_dictionary, 
+                iterator=True, 
+                chunksize=500000,
+                **readargs)
+    # get chr 
+    for k,v in rename_dictionary.items():
+        if v=="CHR":
+            if k in usecols:
+                log.write(" -Columns used to filter variants: {}".format(k),verbose=verbose)
+                chunk_chrom = k
+                break
+
+    log.write(" -Loading only variants on chromosome with pattern : {} ...".format(chrom_pat),verbose=verbose)
+    sumstats_filtered = pd.concat([chunk[chunk[chunk_chrom].str.match(chrom_pat, case=False,na=False) ] for chunk in sumstats_iter])
+    log.write(" -Loaded {} variants on chromosome with pattern :{} ...".format(len(sumstats_filtered), chrom_pat),verbose=verbose)
+    return sumstats_filtered
+
+def _load_variants_with_pattern(inpath,usecols,dtype_dictionary,readargs,rename_dictionary,snpid_pat,log,verbose):
+    
+    sumstats_iter = pd.read_table(inpath,
+                usecols=set(usecols),
+                dtype=dtype_dictionary, 
+                iterator=True, 
+                chunksize=500000,
+                **readargs)
+    # get chr 
+    for k,v in rename_dictionary.items():
+        if v=="SNPID":
+            if k in usecols:
+                log.write(" -Columns used to filter variants: {}".format(k),verbose=verbose)
+                chunk_snpid = k
+                break
+
+    log.write(" -Loading only variants with pattern :  {} ...".format(snpid_pat),verbose=verbose)
+    sumstats_filtered = pd.concat([chunk[chunk[chunk_snpid].str.match(snpid_pat, case=False,na=False) ] for chunk in sumstats_iter])
+    log.write(" -Loaded {} variants with pattern : {} ...".format(len(sumstats_filtered), snpid_pat),verbose=verbose)
+    return sumstats_filtered

gwaslab/io_to_formats.py

@@ -342,7 +342,7 @@ def tofmt(sumstats,
         meta_data,rename_dictionary = get_format_dict(fmt,inverse=True)
         print_format_info(fmt=fmt, meta_data=meta_data,rename_dictionary=rename_dictionary,verbose=verbose, log=log, output=True)
         
-        ymal_path = path + "."+suffix+".tsv-meta.ymal"
+        yaml_path = path + "."+suffix+".tsv-meta.yaml"
         path = path + "."+suffix+".tsv.gz"
         log.write(" -Output path:",path, verbose=verbose) 
         
@@ -361,7 +361,7 @@ def tofmt(sumstats,
             md5_value = calculate_md5sum_file(path)
         
         ## update ssf-style meta data and export to yaml file
-        _configure_ssf_meta(sumstats, fmt, ssfmeta, meta, meta_data, path, md5_value, ymal_path, log, verbose)
+        _configure_ssf_meta(sumstats, fmt, ssfmeta, meta, meta_data, path, md5_value, yaml_path, log, verbose)
         
         return sumstats  
     
@@ -476,7 +476,7 @@ def _configure_output_cols_and_args(sumstats, rename_dictionary, cols, no_status
     return sumstats, to_csvargs
 
 
-def _configure_ssf_meta(sumstats, fmt, ssfmeta, meta, meta_data, path, md5_value, ymal_path, log, verbose):
+def _configure_ssf_meta(sumstats, fmt, ssfmeta, meta, meta_data, path, md5_value, yaml_path, log, verbose):
     ### calculate meta data
     if "EAF" in sumstats.columns:
         min_maf = sumstats["EAF"].min()
@@ -506,8 +506,8 @@ def _configure_ssf_meta(sumstats, fmt, ssfmeta, meta, meta_data, path, md5_value
         sumstats_meta_copy["gwaslab"]["samples"]["sample_size_min"] = n_min
         sumstats_meta_copy["gwaslab"]["samples"]["sample_size_median"] = n_median
         sumstats_meta_copy["gwaslab"]["variants"]["variant_number"] = len(sumstats)
-        log.write(" -Exporting SSF-style meta data to {}".format(ymal_path),verbose=verbose) 
-        with open(ymal_path, 'w') as outfile:
+        log.write(" -Exporting SSF-style meta data to {}".format(yaml_path),verbose=verbose) 
+        with open(yaml_path, 'w') as outfile:
             yaml.dump(sumstats_meta_copy, outfile)
 
 

gwaslab/qc_fix_sumstats.py

@@ -5,6 +5,7 @@ import numpy as np
 from itertools import repeat
 from multiprocessing import  Pool
 from liftover import get_lifter
+from liftover import ChainFile
 from functools import partial
 from gwaslab.g_vchange_status import vchange_status
 from gwaslab.g_vchange_status import status_match
@@ -19,6 +20,7 @@ from gwaslab.g_version import _get_version
 from gwaslab.util_in_fill_data import _convert_betase_to_mlog10p
 from gwaslab.util_in_fill_data import _convert_betase_to_p
 from gwaslab.util_in_fill_data import _convert_mlog10p_to_p
+from gwaslab.bd_common_data import get_chain
 #process build
 #setbuild
 #fixID
@@ -43,9 +45,15 @@ def _process_build(build,log,verbose):
     if str(build).lower() in ["hg19","19","37","b37","grch37"]:
         log.write(" -Genomic coordinates are based on GRCh37/hg19...", verbose=verbose)
         final_build = "19"
+    elif str(build).lower() in ["hg18","18","36","b36","grch36"]:
+        log.write(" -Genomic coordinates are based on GRCh36/hg18...", verbose=verbose)
+        final_build = "18"
     elif str(build).lower() in ["hg38","38","b38","grch38"]:
         log.write(" -Genomic coordinates are based on GRCh38/hg38...", verbose=verbose)
         final_build = "38"
+    elif str(build).lower() in ["t2t","hs1","chm13","13"]:
+        log.write(" -Genomic coordinates are based on T2T-CHM13...", verbose=verbose)
+        final_build = "13"
     else:
         log.warning("Version of genomic coordinates is unknown...", verbose=verbose)
         final_build = "99"
@@ -358,6 +366,76 @@ def fixID(sumstats,
 
 ""
 
+def stripSNPID(sumstats,snpid="SNPID",overwrite=False,verbose=True,log=Log()):  
+    '''
+    flip EA and NEA SNPid   CHR:POS:EA:NEA -> CHR:POS:NEA:EA
+    '''
+    ##start function with col checking##########################################################
+    _start_line = "strip SNPID"
+    _end_line = "stripping SNPID"
+    _start_cols =["SNPID"]
+    _start_function = ".strip_snpid()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    log.write(" -Checking if SNPID is (xxx:)CHR:POS:ATCG_Allele:ATCG_Allele(:xxx)...(separator: - ,: , _)",verbose=verbose)
+    is_chrposrefalt = sumstats[snpid].str.contains(r'[:_-]?\w+[:_-]\d+[:_-][ATCG]+[:_-][ATCG]+[:_-]?', case=False, flags=0, na=False)
+    # check if SNPID is NA
+    is_snpid_na = sumstats[snpid].isna()
+    
+    log.write(" -Stripping {} non-NA fixable SNPIDs...".format(sum(is_chrposrefalt)),verbose=verbose)
+
+    # flip 
+    sumstats.loc[is_chrposrefalt,snpid] = \
+        sumstats.loc[is_chrposrefalt,snpid].str.extract(r'[:_-]?(chr)?(\w+[:_-]\d+[:_-][ATCG]+[:_-][ATCG]+)[:_-]?',flags=re.IGNORECASE|re.ASCII)[1].astype("string")  
+
+    finished(log,verbose,_end_line)
+    return sumstats
+
+def flipSNPID(sumstats,snpid="SNPID",overwrite=False,verbose=True,log=Log()):  
+    '''
+    flip EA and NEA SNPid   CHR:POS:EA:NEA -> CHR:POS:NEA:EA
+    '''
+    ##start function with col checking##########################################################
+    _start_line = "flip SNPID"
+    _end_line = "flipping SNPID"
+    _start_cols =["SNPID"]
+    _start_function = ".flip_snpid()"
+    _must_args ={}
+
+    is_enough_info = start_to(sumstats=sumstats,
+                            log=log,
+                            verbose=verbose,
+                            start_line=_start_line,
+                            end_line=_end_line,
+                            start_cols=_start_cols,
+                            start_function=_start_function,
+                            **_must_args)
+    if is_enough_info == False: return sumstats
+    log.warning("This function only flips alleles in SNPID without changing EA, NEA, STATUS or any statistics.")
+    log.write(" -Checking if SNPID is CHR:POS:ATCG_Allele:ATCG_Allele...(separator: - ,: , _)",verbose=verbose)
+    is_chrposrefalt = sumstats[snpid].str.match(r'^\w+[:_-]\d+[:_-][ATCG]+[:_-][ATCG]+$', case=False, flags=0, na=False)
+    # check if SNPID is NA
+    is_snpid_na = sumstats[snpid].isna()
+    
+    log.write(" -Flipping {} non-NA fixable SNPIDs...".format(sum(is_chrposrefalt)),verbose=verbose)
+
+    # flip 
+    sumstats.loc[is_chrposrefalt,snpid] = \
+        sumstats.loc[is_chrposrefalt,snpid].str.extract(r'^(chr)?(\w+[:_-]\d+)[:_-]([ATCG]+)[:_-]([ATCG]+)$',flags=re.IGNORECASE|re.ASCII)[1].astype("string")  \
+        + ":"+sumstats.loc[is_chrposrefalt,snpid].str.extract(r'^(chr)?(\w+)[:_-](\d+)[:_-]([ATCG]+)[:_-]([ATCG]+)$',flags=re.IGNORECASE|re.ASCII)[4].astype("string") \
+        + ":"+sumstats.loc[is_chrposrefalt,snpid].str.extract(r'^(chr)?(\w+)[:_-](\d+)[:_-]([ATCG]+)[:_-]([ATCG]+)$',flags=re.IGNORECASE|re.ASCII)[3].astype("string")
+
+    finished(log,verbose,_end_line)
+    return sumstats
 
 ###############################################################################################################
 # 20230128 
@@ -1041,7 +1119,7 @@ def check_range(sumstats, var_range, header, coltocheck, cols_to_check, log, ver
         cols_to_check.append(header)
         if header=="STATUS": 
             log.write(" -Checking STATUS and converting STATUS to categories....", verbose=verbose) 
-            categories = {str(j+i) for j in [1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
+            categories = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
             sumstats[header] = pd.Categorical(sumstats[header],categories=categories)
             return sumstats
         
@@ -1496,11 +1574,19 @@ def liftover_variant(sumstats,
              pos="POS", 
              status="STATUS",
              from_build="19", 
-             to_build="38"):
+             to_build="38",
+             chain=None):
+    
     try:
-        converter = get_lifter("hg"+from_build,"hg"+to_build,one_based=True)
+        if chain is None:
+            converter = get_lifter(from_build,to_build,one_based=True)
+        else:
+            converter = ChainFile(chain, one_based=True)
     except:
-        converter = get_lifter("hg"+from_build,"hg"+to_build)
+        if chain is None:
+            converter = get_lifter(from_build,to_build)
+        else:
+            converter = ChainFile(chain)
 
     dic= get_number_to_chr(in_chr=False,xymt=["X","Y","M"])
     dic2= get_chr_to_number(out_chr=False)
@@ -1513,7 +1599,7 @@ def liftover_variant(sumstats,
         sumstats.loc[variants_on_chrom_to_convert,chrom]    =  lifted.str[0].map(dic2).astype("Int64")
     return sumstats
 
-def parallelizeliftovervariant(sumstats,n_cores=1,chrom="CHR", pos="POS", from_build="19", to_build="38",status="STATUS",remove=True, verbose=True,log=Log()):
+def parallelizeliftovervariant(sumstats,n_cores=1,chrom="CHR", pos="POS", from_build="19", to_build="38",status="STATUS",remove=True,chain=None, verbose=True,log=Log()):
     ##start function with col checking##########################################################
     _start_line = "perform liftover"
     _end_line = "liftover"
@@ -1532,8 +1618,21 @@ def parallelizeliftovervariant(sumstats,n_cores=1,chrom="CHR", pos="POS", from_b
                             **_must_args)
     if is_enough_info == False: return sumstats
     ############################################################################################
+    
+    lifter_from_build = _process_build(from_build,log=log,verbose=False)
+    lifter_to_build = _process_build(to_build,log=log,verbose=False)
 
-    log.write(" -Creating converter : hg" + from_build +" to hg"+ to_build, verbose=verbose)
+    if chain is not None:
+        log.write(" -Creating converter using ChainFile: {}".format(chain), verbose=verbose)
+    else:
+        try:
+            chain = get_chain(from_build=from_build, to_build=to_build)
+            log.write(" -Creating converter using ChainFile: {}".format(chain), verbose=verbose)
+        except:
+            chain = None
+            lifter_from_build=from_build
+            lifter_to_build=to_build
+    log.write(" -Creating converter : {} -> {}".format(lifter_from_build, lifter_to_build), verbose=verbose)
     # valid chr and pos
     pattern = r"\w\w\w0\w\w\w"  
     to_lift = sumstats[status].str.match(pattern)
@@ -1549,7 +1648,7 @@ def parallelizeliftovervariant(sumstats,n_cores=1,chrom="CHR", pos="POS", from_b
         pool = Pool(n_cores)
         #df = pd.concat(pool.starmap(func, df_split))
         func=liftover_variant
-        sumstats[[chrom,pos,status]] = pd.concat(pool.map(partial(func,chrom=chrom,pos=pos,from_build=from_build,to_build=to_build,status=status),df_split))
+        sumstats[[chrom,pos,status]] = pd.concat(pool.map(partial(func,chrom=chrom,pos=pos,from_build=from_build,to_build=to_build,status=status,chain=chain),df_split))
         pool.close()
         pool.join()
     ############################################################################

gwaslab/util_in_fill_data.py

@@ -184,7 +184,8 @@ def fill_mlog10p(sumstats,log,verbose=True,filled_count=0):
     else:
         return 0,filled_count
     return 1,filled_count
-def fill_extreme_mlog10p(sumstats,log,verbose=True,filled_count=0):
+
+def fill_extreme_mlog10p(sumstats,df,log,verbose=True,filled_count=0):
     # ref: https://stackoverflow.com/questions/46416027/how-to-compute-p-values-from-z-scores-in-r-when-the-z-score-is-large-pvalue-muc/46416222#46416222
     if "Z" in sumstats.columns:
         # P -> MLOG10P
@@ -198,6 +199,10 @@ def fill_extreme_mlog10p(sumstats,log,verbose=True,filled_count=0):
         log.write("  - Filling MLOG10P using Z column...", verbose=verbose)
         sumstats = fill_extreme_mlog10(sumstats, "Z")
         filled_count +=1
+    elif "CHISQ" in sumstats.columns and "DOF" in sumstats.columns:
+        log.write("  - Filling MLOG10P using CHISQ and DOF column...", verbose=verbose)
+        sumstats = fill_extreme_mlog10_chisq(sumstats, "CHISQ", df)
+        filled_count +=1
     else:
         return 0,filled_count
     return 1,filled_count
@@ -223,6 +228,19 @@ def fill_extreme_mlog10(sumstats, z):
     sumstats["P_EXPONENT"]= exponent
     return sumstats
 
+def fill_extreme_mlog10_chisq(sumstats, chisq, df):
+    #https://stackoverflow.com/a/46416222/199475
+    log_pvalue = ss.chi2.logsf(sumstats[chisq], sumstats[df])
+
+    log10_pvalue = log_pvalue/np.log(10)
+    
+    mantissa = 10**(log10_pvalue %1)
+    exponent = log10_pvalue // 1
+    sumstats["MLOG10P"] = -log10_pvalue
+    sumstats["P_MANTISSA"]= mantissa
+    sumstats["P_EXPONENT"]= exponent
+    return sumstats
+
 ####################################################################################################################
 def fill_iteratively(sumstats,raw_to_fill,log,only_sig,df,extreme,verbose,sig_level):
     to_fill = raw_to_fill.copy()
@@ -260,7 +278,7 @@ def fill_iteratively(sumstats,raw_to_fill,log,only_sig,df,extreme,verbose,sig_le
     # p to -log10(P)  ###############################################################################################
         if "MLOG10P" in to_fill:
             if extreme==True:
-                status,filled_count = fill_extreme_mlog10p(sumstats,log,verbose=verbose,filled_count=filled_count)
+                status,filled_count = fill_extreme_mlog10p(sumstats,df, log,verbose=verbose,filled_count=filled_count)
                 filled_count +=1
             elif "P" not in sumstats.columns:
                 fill_p(sumstats,log,verbose=verbose)

gwaslab/viz_plot_miamiplot2.py

@@ -290,7 +290,11 @@ def plot_miami2(
     
 
     #####################################################################################################################    
-    
+    ax1l, ax1r = ax5.get_xlim()
+    ax5l, ax5r = ax1.get_xlim()
+    ax1.set_xlim([min(ax1l,ax5l), max(ax1r,ax5r)])
+    ax5.set_xlim([min(ax1l,ax5l), max(ax1r,ax5r)])
+    #####################################################################################################################
     ax5.set_xlabel("")
     #ax5.set_xticks(chrom_df)
     ax5.set_xticklabels([])

gwaslab/viz_plot_stackedregional.py

@@ -95,6 +95,10 @@ def plot_stacked_mqq(objects,
         if "family" not in title_args.keys():
             title_args["family"] = "Arial"
     # create figure and axes ##################################################################################################################
+    #
+    # subplot_height : subplot height
+    # figsize : Width, height in inches
+
     if mode=="r":
         if len(vcfs)==1:
             vcfs = vcfs *len(sumstats_list)
@@ -107,14 +111,17 @@ def plot_stacked_mqq(objects,
         else:
             height_ratios = [1 for i in range(n_plot_plus_gene_track-1)]+[gene_track_height]
         
-        fig_args["figsize"] = [16,subplot_height*n_plot_plus_gene_track]
+        if "figsize" not in fig_args.keys():
+            fig_args["figsize"] = [16,subplot_height*n_plot_plus_gene_track]
+
         fig, axes = plt.subplots(n_plot_plus_gene_track, 1, sharex=True, 
                              gridspec_kw={'height_ratios': height_ratios},
                              **fig_args)
         plt.subplots_adjust(hspace=region_hspace)
     elif mode=="m":
         n_plot = len(sumstats_list)
-        fig_args["figsize"] = [10,subplot_height*n_plot]
+        if "figsize" not in fig_args.keys():
+            fig_args["figsize"] = [10,subplot_height*n_plot]
         fig, axes = plt.subplots(n_plot, 1, sharex=True, 
                              gridspec_kw={'height_ratios': [1 for i in range(n_plot)]},
                              **fig_args)
@@ -122,8 +129,8 @@ def plot_stacked_mqq(objects,
         vcfs = [None for i in range(n_plot)]
     elif mode=="mqq":
         n_plot = len(objects)
-#
-        fig_args["figsize"] = [10,subplot_height*n_plot]
+        if "figsize" not in fig_args.keys():
+            fig_args["figsize"] = [10,subplot_height*n_plot]
         fig, axes = plt.subplots(n_plot, 2, sharex=True, 
                              gridspec_kw={'height_ratios': [1 for i in range(n_plot-1)],
                                           'width_ratios':[mqqratio,1]},

{gwaslab-3.4.48.dist-info → gwaslab-3.4.49.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gwaslab
-Version: 3.4.48
+Version: 3.4.49
 Summary: A collection of handy tools for GWAS SumStats
 Author-email: Yunye <yunye@gwaslab.com>
 Project-URL: Homepage, https://cloufield.github.io/gwaslab/

{gwaslab-3.4.48.dist-info → gwaslab-3.4.49.dist-info}/RECORD

@@ -1,25 +1,25 @@
 gwaslab/__init__.py,sha256=7TKJaODdpeuQKibL7gIEa4MtyQ0pmrU-vIHQ-Et27lQ,2433
-gwaslab/bd_common_data.py,sha256=qr6OMbBaTH2Smfu8347SO9NmF410tn8dq8pRGF5-OpY,13751
+gwaslab/bd_common_data.py,sha256=2voBqMrIsII1TN5T6uvyDax90fWcJK1Stmo1ZHNGGsE,13898
 gwaslab/bd_config.py,sha256=TP-r-DPhJD3XnRYZbw9bQHXaDIkiRgK8bG9HCt-UaLc,580
 gwaslab/bd_download.py,sha256=cDDk2C5IvjeAzvPvVYGTkI4Ss33DUtEDjGo8eAbQRvY,15663
 gwaslab/bd_get_hapmap3.py,sha256=asNjQYeGfQi8u3jnfenRvDdKMs5ptql5wpcUzqMlwUI,3937
 gwaslab/cache_manager.py,sha256=HOTnSkCOyGEPLRl90WT8D_6pAdI8d8AzenMIDGuCeWc,28113
 gwaslab/g_Log.py,sha256=C3Zv-_6c3C9ms8bgQ-ytplz22sjk7euqXYkWr9zNeAs,1573
 gwaslab/g_Phenotypes.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-gwaslab/g_Sumstats.py,sha256=U5la-fJCSTN8FNxXjcULcPH8MV9D6YgMaVFNesMTa_o,35854
+gwaslab/g_Sumstats.py,sha256=qWBh14-XheS88KLm2vr0OKRAhU9VmbUvy90T4L52zAo,36265
 gwaslab/g_SumstatsPair.py,sha256=20snPb4SlI6ftMGVjgxAuyxsxYRQF-GzzlBSnoB-3Lo,8851
 gwaslab/g_SumstatsT.py,sha256=u_DighLMnMxwTLnqm-B58pA0G6WXRj6pudPyKMVKjSU,2133
 gwaslab/g_Sumstats_summary.py,sha256=FECvvFXJVKaCX5dggBvvk9YvJ6AbdbcLfjltysX7wEE,6380
 gwaslab/g_meta.py,sha256=htWlgURWclm9R6UqFcX1a93WN27xny7lGUeyJZOtszQ,2583
-gwaslab/g_vchange_status.py,sha256=jLoVzMJFhB5k_cJKzHuBNc2HZGBWydAunCNa0n_d54g,1923
-gwaslab/g_version.py,sha256=kB6LiBYcRZElabjDUUEBhF5c5SVUq8vHO0pctDfBqkw,1886
+gwaslab/g_vchange_status.py,sha256=w3zsYYOcCaI3PTeboonvkQjudzUAfVIgATzRdiPViZs,1939
+gwaslab/g_version.py,sha256=dhN-3rbJKALlFhTZACsxBZL_U8j4PQmcVuCuWx9dapo,1886
 gwaslab/hm_casting.py,sha256=FqP4EQl83Q2OKLw004OgLIvUH795TVCGwziLk5jsHqY,11368
-gwaslab/hm_harmonize_sumstats.py,sha256=1hjUdle2DSKHGBp2BktfFqf-QHU_q2xWl_mPhiYc_ZA,78616
+gwaslab/hm_harmonize_sumstats.py,sha256=_sZ8soikAxDokw-dcr_CLguBB8OmTmPPS04MfmsJc_Q,79509
 gwaslab/hm_rsid_to_chrpos.py,sha256=ODWREO0jPN0RAfNzL5fRzSRANfhiksOvUVPuEsFZQqA,6552
-gwaslab/io_preformat_input.py,sha256=AZ43WGqVTzbo3XtClWhjRjsj6pBR9stw6JBL_TZ461U,20673
+gwaslab/io_preformat_input.py,sha256=J8Ny4OPMaLVdo2nP8lTM-c5A8LSdqphSrp9G4i9JjDQ,24097
 gwaslab/io_read_ldsc.py,sha256=wsYXpH50IchBKd2dhYloSqc4YgnDkiwMsAweaCoN5Eo,12471
 gwaslab/io_read_tabular.py,sha256=EG-C6KhCutt4J4LlOMgXnqzJvU-EZXzVhMvaDFnHrMM,2380
-gwaslab/io_to_formats.py,sha256=m57dGoqmHzAE1E27j9YxYKVCA12_lKd1qCnZtp0WZLw,29401
+gwaslab/io_to_formats.py,sha256=8FmbQjWUIsz_V1Lb80TuwRIXKBgs5t42j25Znougk1Y,29401
 gwaslab/io_to_pickle.py,sha256=HhePU0VcaGni0HTNU0BqoRaOnrr0NOxotgY6ISdx3Ck,1833
 gwaslab/ldsc_irwls.py,sha256=83JbAMAhD0KOfpv4IJa6LgUDfQjp4XSJveTjnhCBJYQ,6142
 gwaslab/ldsc_jackknife.py,sha256=XrWHoKS_Xn9StG1I83S2vUMTertsb-GH-_gOFYUhLeU,17715
@@ -28,7 +28,7 @@ gwaslab/ldsc_parse.py,sha256=MBnfgcWlV4oHp9MoDRh1mpilaHhAR15Af77hMFn4-5k,10564
 gwaslab/ldsc_regressions.py,sha256=yzbGjgNV7u-SWXNPsh9S8y9mK97Bim_Nmad9G9V18ZU,30078
 gwaslab/ldsc_sumstats.py,sha256=O0olsDxKlh1MJ1gAuEN1t40rxhajOEwOQ20ak7xoDrI,26245
 gwaslab/qc_check_datatype.py,sha256=kW68uk4dTLOU2b1dHoVat6n0loundDysAjIqxsXW28Q,3379
-gwaslab/qc_fix_sumstats.py,sha256=aTX5jf7MVpp6XTiP-rbEyaLAQFsn1gyH9vEH4SLH-GY,93199
+gwaslab/qc_fix_sumstats.py,sha256=rb1Iv05o0tYGGTiXXXJphkx-xVQNKAtlwkDfY-D0yPs,97979
 gwaslab/run_script.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 gwaslab/util_abf_finemapping.py,sha256=LRcopjtkT-iXtKPAJIzR4qjPdhD7nrS_BGit4EW89FM,3054
 gwaslab/util_ex_calculate_ldmatrix.py,sha256=Z_spxbq6SHDS0v84I59YTTF40iyLQIOZbt0dmEcNJjw,15417
@@ -47,7 +47,7 @@ gwaslab/util_in_calculate_gc.py,sha256=MWOXVzJv7SZx4i2_ncRiqsiEOADc7EfghaUzgGy4j
 gwaslab/util_in_calculate_power.py,sha256=JfHJFg3tNF0f4NHgWlzVW2mSxCiP07mAHIyEfVfxTak,10360
 gwaslab/util_in_convert_h2.py,sha256=a8Cbudt3xn9WP2bPc-7ysuowB-LYub8j8GeDXl7Lk7Q,6483
 gwaslab/util_in_correct_winnerscurse.py,sha256=Gp--yAQ8MMzdkWIvXP9C1BHVjZc-YzqHfYWhAj19w9w,2110
-gwaslab/util_in_fill_data.py,sha256=gdTwYA6FvBMnrtxAeL0lEj_Z0aGIoRNPScWDlJvZWeQ,14021
+gwaslab/util_in_fill_data.py,sha256=iVq5WLWwFI03v9HyvBanu5si3j2p-oyPFTl8jsX69xM,14693
 gwaslab/util_in_filter_value.py,sha256=dY4X66N9A4MHCRHjPqLYFufMM91ggLRwUBf_nJYh8Lg,23605
 gwaslab/util_in_get_density.py,sha256=kpKXH69acMkeYVG5vs-VbJC3COhmuLBfYco-wuOxgjc,3934
 gwaslab/util_in_get_sig.py,sha256=9kq1GXacknO2YnVmsTli1GlPA728ASweTZ3UKm3Wszo,38783
@@ -62,25 +62,25 @@ gwaslab/viz_plot_compare_af.py,sha256=qtXW45-Sq_ugK8ZfqBYMpmf58SKi3lB3YyHnzn_akc
 gwaslab/viz_plot_compare_effect.py,sha256=lV-4KSQatTSsYqzEGGTkBu17I2VlVyVh1uJr-kCq630,49585
 gwaslab/viz_plot_forestplot.py,sha256=xgOnefh737CgdQxu5naVyRNBX1NQXPFKzf51fbh6afs,6771
 gwaslab/viz_plot_miamiplot.py,sha256=rCFEp7VNuVqeBBG3WRkmFAtFklbF79BvIQQYiSY70VY,31238
-gwaslab/viz_plot_miamiplot2.py,sha256=Jg5j9qsr8gGydU8AOKU8xRZTNH_3QM8Nxv61Vh12aG0,15732
+gwaslab/viz_plot_miamiplot2.py,sha256=xiFCgFX8hEySmCJORpEurMVER9eEXQyy_Ik7mLkbi9g,16015
 gwaslab/viz_plot_mqqplot.py,sha256=q_c_ZDRdirEcdHHnb030h7dTUWAFy9OVuzwbdCzZ3gU,64580
 gwaslab/viz_plot_qqplot.py,sha256=psQgVpP29686CEZkzQz0iRbApzqy7aE3GGiBcazVvNw,7247
 gwaslab/viz_plot_regional2.py,sha256=tBoGox-4ngL5o_twdIjk_VW6Iam3JDyrPKuttm6_4Sg,36862
 gwaslab/viz_plot_regionalplot.py,sha256=8u-5-yfy-UaXhaxVVz3Y5k2kBAoqzczUw1hyyD450iI,37983
 gwaslab/viz_plot_rg_heatmap.py,sha256=PidUsgOiEVt6MfBPCF3_yDhOEytZ-I1q-ZD6_0pFrV4,13713
-gwaslab/viz_plot_stackedregional.py,sha256=AoGMer73P3y-21A2qityrDX-_sIJFRgKzkm8i9zVxKA,16420
+gwaslab/viz_plot_stackedregional.py,sha256=IRugpwLjqKdwuSVTQtCM36Rk4Gpv6MxdsGIWVj6vk8w,16651
 gwaslab/viz_plot_trumpetplot.py,sha256=ZHdc6WcVx0-oKoj88yglRkmB4bS9pOiEMcuwKW35Yvo,42672
 gwaslab/data/formatbook.json,sha256=N2nJs80HH98Rsu9FxaSvIQO9J5yIV97WEtAKjRqYwiY,38207
-gwaslab/data/reference.json,sha256=k8AvvgDsuLxzv-NCJHWvTUZ5q_DLAFxs1Th3jtL313k,11441
+gwaslab/data/reference.json,sha256=IrjwFnXjrpVUp3zYfcYClpibJE9Y-94gtrC1Aw8sXxg,12332
 gwaslab/data/chrx_par/chrx_par_hg19.bed.gz,sha256=LocZg_ozhZjQiIpgWCO4EYCW9xgkEKpRy1m-YdIpzQs,83
 gwaslab/data/chrx_par/chrx_par_hg38.bed.gz,sha256=VFW11MnQVC-Iu-ZGvUDcEhVpb-HVRsVTg-W-GNJyxP4,82
 gwaslab/data/hapmap3_SNPs/hapmap3_db150_hg19.snplist.gz,sha256=qD9RsC5S2h6l-OdpW09DWXHIi2kcPebctMnhxt8mzfU,10282886
 gwaslab/data/hapmap3_SNPs/hapmap3_db151_hg38.snplist.gz,sha256=Y8ZT2FIAhbhlgCJdE9qQVAiwnV_fcsPt72usBa7RSBM,10225828
 gwaslab/data/high_ld/high_ld_hla_hg19.bed.gz,sha256=R7IkssKu0L4WwkU9SrS84xCMdrkkKL0gnTNO_OKbG0Y,219
 gwaslab/data/high_ld/high_ld_hla_hg38.bed.gz,sha256=76CIU0pibDJ72Y6UY-TbIKE9gEPwTELAaIbCXyjm80Q,470
-gwaslab-3.4.48.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-gwaslab-3.4.48.dist-info/LICENSE_before_v3.4.39,sha256=GhLOU_1UDEKeOacYhsRN_m9u-eIuVTazSndZPeNcTZA,1066
-gwaslab-3.4.48.dist-info/METADATA,sha256=4Cqofc_AlsSBNtwKwiLBFiDtQsh8UUvWiM_XhBYksxw,7752
-gwaslab-3.4.48.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
-gwaslab-3.4.48.dist-info/top_level.txt,sha256=PyY6hWtrALpv2MAN3kjkIAzJNmmBTH5a2risz9KwH08,8
-gwaslab-3.4.48.dist-info/RECORD,,
+gwaslab-3.4.49.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+gwaslab-3.4.49.dist-info/LICENSE_before_v3.4.39,sha256=GhLOU_1UDEKeOacYhsRN_m9u-eIuVTazSndZPeNcTZA,1066
+gwaslab-3.4.49.dist-info/METADATA,sha256=bA0qeWgxt5mOkt_aL4aW_v4csyNSJCsbY6wrKMLyqY8,7752
+gwaslab-3.4.49.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+gwaslab-3.4.49.dist-info/top_level.txt,sha256=PyY6hWtrALpv2MAN3kjkIAzJNmmBTH5a2risz9KwH08,8
+gwaslab-3.4.49.dist-info/RECORD,,

{gwaslab-3.4.48.dist-info → gwaslab-3.4.49.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (73.0.1)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any