gwaslab 3.5.7__py3-none-any.whl → 3.5.8__py3-none-any.whl

gwaslab/g_Sumstats_polars.py

@@ -0,0 +1,245 @@
+import pandas as pd
+import numpy as np
+import time
+import copy
+from gwaslab.g_Sumstats_summary import summarize
+from gwaslab.g_Sumstats_summary import lookupstatus
+from gwaslab.io_preformat_input_polars import preformatp
+from gwaslab.io_to_formats import _to_format
+from gwaslab.g_Log import Log
+from gwaslab.qc_fix_sumstats import fixID
+from gwaslab.qc_fix_sumstats import flipSNPID
+from gwaslab.qc_fix_sumstats import stripSNPID
+from gwaslab.qc_fix_sumstats import removedup
+from gwaslab.qc_fix_sumstats import fixchr
+from gwaslab.qc_fix_sumstats import fixpos
+from gwaslab.qc_fix_sumstats import fixallele
+from gwaslab.qc_fix_sumstats import parallelnormalizeallele
+from gwaslab.qc_fix_sumstats import sanitycheckstats
+from gwaslab.qc_fix_sumstats import parallelizeliftovervariant
+from gwaslab.qc_fix_sumstats import flipallelestats
+from gwaslab.qc_fix_sumstats import sortcoordinate
+from gwaslab.qc_fix_sumstats import sortcolumn
+from gwaslab.qc_fix_sumstats import _set_build
+from gwaslab.qc_fix_sumstats import _process_build
+from gwaslab.hm_harmonize_sumstats import parallelecheckaf
+from gwaslab.hm_harmonize_sumstats import paralleleinferaf
+from gwaslab.hm_harmonize_sumstats import checkref
+from gwaslab.hm_harmonize_sumstats import oldcheckref
+from gwaslab.hm_harmonize_sumstats import rsidtochrpos
+from gwaslab.hm_harmonize_sumstats import parallelizeassignrsid
+from gwaslab.hm_harmonize_sumstats import parallelinferstrand
+from gwaslab.hm_harmonize_sumstats import parallelrsidtochrpos
+from gwaslab.hm_harmonize_sumstats import _paralleleinferafwithmaf
+from gwaslab.util_in_filter_value import filtervalues
+from gwaslab.util_in_filter_value import filterout
+from gwaslab.util_in_filter_value import filterin
+from gwaslab.util_in_filter_value import filterregionin
+from gwaslab.util_in_filter_value import filterregionout
+from gwaslab.util_in_filter_value import _filter_indel
+from gwaslab.util_in_filter_value import _filter_palindromic
+from gwaslab.util_in_filter_value import _filter_snp
+from gwaslab.util_in_filter_value import _filter_region
+from gwaslab.util_in_filter_value import _exclude_hla
+from gwaslab.util_in_filter_value import _search_variants
+from gwaslab.util_in_filter_value import inferbuild
+from gwaslab.util_in_filter_value import sampling
+from gwaslab.util_in_filter_value import _get_flanking
+from gwaslab.util_in_filter_value import _get_flanking_by_chrpos
+from gwaslab.util_in_filter_value import _get_flanking_by_id
+from gwaslab.util_in_calculate_gc import lambdaGC
+from gwaslab.util_in_convert_h2 import _get_per_snp_r2
+from gwaslab.util_in_get_sig import getsig
+from gwaslab.util_in_get_density import getsignaldensity
+from gwaslab.util_in_get_density import assigndensity
+from gwaslab.util_in_get_sig import annogene
+from gwaslab.util_in_get_sig import getnovel
+from gwaslab.util_in_get_sig import _check_cis
+from gwaslab.util_in_get_sig import _check_novel_set
+from gwaslab.util_in_fill_data import filldata
+from gwaslab.bd_get_hapmap3 import gethapmap3
+from gwaslab.bd_common_data import get_chr_list
+from gwaslab.bd_common_data import get_number_to_chr
+from gwaslab.bd_common_data import get_chr_to_number
+from gwaslab.bd_common_data import get_high_ld
+from gwaslab.bd_common_data import get_format_dict
+from gwaslab.bd_common_data import get_formats_list
+from gwaslab.g_version import _show_version
+from gwaslab.g_version import gwaslab_info
+from gwaslab.g_meta import _init_meta
+from gwaslab.g_meta import _append_meta_record
+from gwaslab.g_meta_update import _update_meta
+from gwaslab.util_ex_run_clumping import _clump
+from gwaslab.util_ex_calculate_ldmatrix import tofinemapping
+from gwaslab.io_load_ld import tofinemapping_using_ld
+from gwaslab.util_ex_calculate_prs import _calculate_prs
+from gwaslab.viz_plot_mqqplot import mqqplot
+from gwaslab.viz_plot_trumpetplot import plottrumpet
+from gwaslab.viz_plot_compare_af import plotdaf
+from gwaslab.util_ex_run_susie import _run_susie_rss
+from gwaslab.util_ex_run_susie import _get_cs_lead
+from gwaslab.qc_fix_sumstats import _check_data_consistency
+from gwaslab.util_ex_ldsc import _estimate_h2_by_ldsc
+from gwaslab.util_ex_ldsc import _estimate_rg_by_ldsc
+from gwaslab.util_ex_ldsc import _estimate_h2_cts_by_ldsc
+from gwaslab.util_ex_ldsc import _estimate_partitioned_h2_by_ldsc 
+from gwaslab.util_ex_ldproxyfinder import _extract_ld_proxy
+from gwaslab.bd_get_hapmap3 import gethapmap3
+from gwaslab.util_abf_finemapping import abf_finemapping
+from gwaslab.util_abf_finemapping import make_cs
+from gwaslab.io_read_pipcs import _read_pipcs
+from gwaslab.util_in_estimate_ess import _get_ess
+from gwaslab.viz_plot_credible_sets import _plot_cs
+from gwaslab.hm_casting import _align_with_mold
+from gwaslab.hm_casting  import _merge_mold_with_sumstats_by_chrpos
+import gc
+from gwaslab.viz_plot_phe_heatmap import _gwheatmap
+from gwaslab.util_ex_run_prscs import _run_prscs
+
+#20220309
+class Sumstatsp():
+    def __init__(self,
+             sumstats,
+             fmt=None,
+             tab_fmt="tsv",
+             snpid=None,
+             rsid=None,
+             chrom=None,
+             pos=None,
+             ea=None,
+             nea=None,
+             ref=None,
+             alt=None,
+             eaf=None,
+             neaf=None,
+             maf=None,
+             n=None,
+             beta=None,
+             se=None,
+             chisq=None,
+             z=None,
+             f=None,
+             t=None,
+             p=None,
+             q=None,
+             mlog10p=None,
+             test=None,
+             info=None,
+             OR=None,
+             OR_95L=None,
+             OR_95U=None,
+             beta_95L=None,
+             beta_95U=None,
+             HR=None,
+             HR_95L=None,
+             HR_95U=None,
+             ncase=None,
+             ncontrol=None,
+             neff=None,
+             i2=None,
+             phet=None,
+             dof=None,
+             snpr2=None,
+             status=None,
+             other=[],
+             chrom_pat=None,
+             snpid_pat=None,
+             usekeys=None,
+             direction=None,
+             verbose=True,
+             study="Study_1",
+             trait="Trait_1",
+             build="99",
+             species="homo sapiens",
+             build_infer=False,
+             **readargs):
+
+        # basic attributes
+        self.data = pd.DataFrame()
+        self.log = Log()
+        self.ldsc_h2 = None
+        self.ldsc_h2_results = None
+        self.ldsc_rg = pd.DataFrame()
+        self.ldsc_h2_cts = None
+        self.ldsc_partitioned_h2_summary = None
+        self.ldsc_partitioned_h2_results = None
+        # meta information
+        self.meta = _init_meta() 
+        self.build = build
+        self.meta["gwaslab"]["study_name"] =  study
+        self.meta["gwaslab"]["species"] = species
+        
+        # initialize attributes for clumping and finmapping
+        #self.to_finemapping_file_path = ""
+        #self.to_finemapping_file  = pd.DataFrame()
+        #self.plink_log = ""
+
+        # path / file / plink_log
+        self.finemapping = dict()
+
+        # clumps / clumps_raw / plink_log
+        self.clumps = dict()
+        
+        #
+        self.pipcs = pd.DataFrame()
+
+        # print gwaslab version information
+        _show_version(self.log, verbose=verbose)
+
+        #preformat the data
+        self.data  = preformatp(
+          sumstats=sumstats,
+          fmt=fmt,
+          tab_fmt = tab_fmt,
+          snpid=snpid,
+          rsid=rsid,
+          chrom=chrom,
+          pos=pos,
+          ea=ea,
+          nea=nea,
+          ref=ref,
+          alt=alt,
+          eaf=eaf,
+          neaf=neaf,
+          maf=maf,
+          n=n,
+          beta=beta,
+          se=se,
+          chisq=chisq,
+          z=z,
+          f=f,
+          t=t,
+          p=p,
+          q=q,
+          mlog10p=mlog10p,
+          test=test,
+          info=info,
+          OR=OR,
+          OR_95L=OR_95L,
+          OR_95U=OR_95U,
+          beta_95L=beta_95L,
+          beta_95U=beta_95U,
+          HR=HR,
+          HR_95L=HR_95L,
+          HR_95U=HR_95U,
+          i2=i2,
+          phet=phet,
+          dof=dof,
+          snpr2=snpr2,
+          ncase=ncase,
+          ncontrol=ncontrol,
+          neff=neff,
+          direction=direction,
+          study=study,
+          build=build,
+          trait=trait,
+          status=status,
+          other=other,
+          usekeys=usekeys,
+          chrom_pat=chrom_pat,
+          snpid_pat=snpid_pat,
+          verbose=verbose,
+          readargs=readargs,
+          log=self.log)
+
+        gc.collect()   

gwaslab/g_headers.py

@@ -33,9 +33,10 @@ dtype_dic={
  'SNPR2'             : 'float64'  ,
  'DOF'               : 'Int64'    ,
  'P_HET'             : 'float64'  ,
- 'I2_HET'            : 'float64'  ,
+ 'I2'            : 'float64'  ,
  'DENSITY'           : 'Int64'    ,
  'N'                 : 'Int64'    ,
+ 'N_EFF'             : 'float64'  ,
  'N_CASE'            : 'Int64'    ,
  'N_CONTROL'         : 'Int64'    ,
  'GENENAME'          : 'string'   ,
@@ -92,9 +93,10 @@ description_dic={
  'SNPR2'             :' per variant R2                                 ',
  'DOF'               :' degree of freedom                              ',
  'P_HET'             :' heterogeneity test P value                     ',
- 'I2_HET'            :' heterogeneity I2                               ',
+ 'I2'            :' heterogeneity I2                               ',
  'DENSITY'           :' signal density                                 ',
  'N'                 :' total sample size                              ',
+ 'N_EFF'             :' Effective sample size                          ',
  'N_CASE'            :' number of cases                                ',
  'N_CONTROL'         :' number of controls                             ',
  'GENENAME'          :' nearest gene symbol                            ',
@@ -117,7 +119,14 @@ def _get_headers(mode="all"):
     if mode=="info":
         return ["SNPID","rsID","CHR","POS","EA","NEA","STATUS"]
     elif mode=="stats":
-        return ["BETA","SE","P","MLOG10P","N","N_CASE","N_CONTROL","Z","T","F","OR","OR_95L","OR_95U","HR","HR_95L","HR_95U","MAF","EAF","BETA_95L","BETA_95U"]
+        return ["BETA","SE","P","MLOG10P",
+                "N","N_CASE","N_CONTROL","N_EFF",
+                "Z","T","F",
+                "OR","OR_95L","OR_95U",
+                "HR","HR_95L","HR_95U",
+                "MAF","EAF",
+                "BETA_95L","BETA_95U",
+                "P_HET","I2"]
     else:
         return description_dic.keys()
 

gwaslab/g_meta.py

@@ -1,54 +1,130 @@
 from gwaslab.g_version import gwaslab_info
 
-def _init_meta():
-    metadata = {"gwaslab":{
-                        "gwaslab_version": gwaslab_info()["version"],
-                        "study_name":"Sumstats_1",
-                        "study_type":"Unknown",
-                        "species":"homo sapiens",
-                        "genome_build":"99",
-                        "variants":{
-                            "variant_number":"Unknown",
-                            "min_P":"Unknown",
-                            "number_of_chromosomes":"Unknown",
-                        },
+def _init_meta(object="Sumstats"):
+    metadata_ssf ={
+                        "genotyping_technology":"Unknown", 
+                        "gwas_id":"Unknown", 
                         "samples":{
-                            "sample_size":"Unknown",
-                            "sample_size_case":"Unknown",
-                            "sample_size_control":"Unknown",
-                            "sample_size_median":"Unknown",
-                            "sample_size_min":"Unknown",
-                        },
-                        "references":{
-                            "ref_rsid_tsv":"Unknown",
-                            "ref_rsid_vcf":"Unknown",
-                            "ref_seq":"Unknown",
-                            "ref_infer":"Unknown",
-                            "ref_infer_af":"Unknown",
-                            "ref_infer_daf":"Unknown",
-                            "ref_rsid_to_chrpos_tsv":"Unknown",
-                            "ref_rsid_to_chrpos_vcf":"Unknown"
-                        }
+                            "sample_size":"Unknown", 
+                            "sample_ancestry":"European", 
+                            "ancestry_method":"self-reported|genetically determined", 
+                        } ,
+                        "trait_description":"Unknown", 
+                        "minor_allele_freq_lower_limit":"Unknown", 
+                        "data_file_name":"Unknown", 
+                        "file_type":"Unknown", 
+                        "data_file_md5sum":"Unknown", 
+                        "is_harmonised":"Unchecked", 
+                        "is_sorted":"Unchecked", 
+                        "genome_assembly":"Unknown",
+                        "date_last_modified":"Unknown", 
+                        "coordinate_system":"1-based",
+                        "sex": "M|F|combined"
+                    }
+    metadata_multi ={
+                        "genome_assembly":"Unknown",
+                        "date_last_modified":"Unknown", 
+                        "coordinate_system":"1-based"
+                    }
+   
+    # Sumstats
+    if object=="Sumstats":
+        metadata = {"gwaslab":{
+                    "gwaslab_version": gwaslab_info()["version"],
+                    "gwaslab_object":"gwaslab.Sumstats",
+                    "study_name":"Sumstats1",
+                    "study_type":"Unknown",
+                    "species":"homo sapiens",
+                    "genome_build":"99",
+                    "sample_prevalence":"Unknown",
+                    "population_prevalence":"Unknown",
+                    "variants":{
+                        "variant_number":"Unknown",
+                        "min_P":"Unknown",
+                        "number_of_chromosomes":"Unknown",
                     },
-                     "genotyping_technology":"Unknown", 
-                     "gwas_id":"Unknown", 
-                     "samples":{
-                          "sample_size":"Unknown", 
-                          "sample_ancestry":"European", 
-                          "ancestry_method":"self-reported|genetically determined", 
-                     } ,
-                     "trait_description":"Unknown", 
-                     "minor_allele_freq_lower_limit":"Unknown", 
-                     "data_file_name":"Unknown", 
-                     "file_type":"Unknown", 
-                     "data_file_md5sum":"Unknown", 
-                     "is_harmonised":"Unchecked", 
-                     "is_sorted":"Unchecked", 
-                     "genome_assembly":"Unknown",
-                     "date_last_modified":"Unknown", 
-                     "coordinate_system":"1-based",
-                     "sex": "M|F|combined"
-                     }
+                    "samples":{
+                        "sample_size":"Unknown",
+                        "sample_size_case":"Unknown",
+                        "sample_size_control":"Unknown",
+                        "sample_size_median":"Unknown",
+                        "sample_size_min":"Unknown",
+                    },
+                    "references":{
+                        "ref_rsid_tsv":"Unknown",
+                        "ref_rsid_vcf":"Unknown",
+                        "ref_seq":"Unknown",
+                        "ref_infer":"Unknown",
+                        "ref_infer_af":"Unknown",
+                        "ref_infer_daf":"Unknown",
+                        "ref_rsid_to_chrpos_tsv":"Unknown",
+                        "ref_rsid_to_chrpos_vcf":"Unknown"
+                    }}}
+        metadata |= metadata_ssf
+    
+    # SumstatsPair
+    elif object=="SumstatsPair":
+        metadata = {"gwaslab":{
+                                    "gwaslab_version": gwaslab_info()["version"],
+                                    "gwaslab_object":"gwaslab.SumstatsPair",
+                                    "group_name":"Group1",
+                                    "species":"homo sapiens",
+                                    "genome_build":"99",
+                                    "variants":{
+                                        "variant_number":"Unknown",
+                                        "min_P":"Unknown",
+                                        "number_of_chromosomes":"Unknown",
+                                    },
+                                    "samples":{
+                                        "sample_size":"Unknown",
+                                        "sample_size_case":"Unknown",
+                                        "sample_size_control":"Unknown",
+                                        "sample_size_median":"Unknown",
+                                        "sample_size_min":"Unknown",
+                                    },
+                                    "references":{
+                                        "ref_rsid_tsv":"Unknown",
+                                        "ref_rsid_vcf":"Unknown",
+                                        "ref_seq":"Unknown",
+                                        "ref_infer":"Unknown",
+                                        "ref_infer_af":"Unknown",
+                                        "ref_infer_daf":"Unknown",
+                                        "ref_rsid_to_chrpos_tsv":"Unknown",
+                                        "ref_rsid_to_chrpos_vcf":"Unknown"
+                                    }}}
+        metadata |= metadata_multi
+    
+    # SumstatsMulti
+    elif object=="SumstatsMulti":
+        metadata = {"gwaslab":{
+                            "gwaslab_version": gwaslab_info()["version"],
+                            "gwaslab_object":"gwaslab.SumstatsMulti",
+                            "group_name":"Group1",
+                            "species":"homo sapiens",
+                            "genome_build":"99",
+                            "variants":{
+                                "variant_number":"Unknown",
+                                "min_P":"Unknown",
+                                "number_of_chromosomes":"Unknown",
+                            },
+                            "samples":{
+                                "sample_size":"Unknown",
+                                "sample_size_case":"Unknown",
+                                "sample_size_control":"Unknown",
+                                "sample_size_median":"Unknown",
+                                "sample_size_min":"Unknown",
+                            },
+                            "references":{
+                                "ref_rsid_tsv":"Unknown",
+                                "ref_rsid_vcf":"Unknown",
+                                "ref_seq":"Unknown",
+                                "ref_infer":"Unknown",
+                                "ref_infer_af":"Unknown",
+                                "ref_infer_daf":"Unknown",
+                                "ref_rsid_to_chrpos_tsv":"Unknown",
+                                "ref_rsid_to_chrpos_vcf":"Unknown"
+                            }}}
+        metadata |= metadata_multi
     return metadata.copy()
 
 def _append_meta_record(old, new):

gwaslab/g_meta_update.py

@@ -0,0 +1,48 @@
+import numpy as np
+from gwaslab.util_in_filter_value import inferbuild
+from gwaslab.g_Log import Log
+import time
+
+def _update_meta(meta, sumstats, object="Sumstats",log=Log(), verbose=True):  
+    
+    meta["gwaslab"]["variants"]["variant_number"] = len(sumstats)
+    
+    if "CHR" in sumstats.columns:
+        meta["gwaslab"]["variants"]["number_of_chromosomes"] = len(sumstats["CHR"].unique())
+    
+    if  meta["gwaslab"]["gwaslab_object"]=="gwaslab.Sumstats":      
+        if "P" in sumstats.columns:
+            meta["gwaslab"]["variants"]["min_P"]=np.nanmin(sumstats["P"])
+        if "EAF" in sumstats.columns:
+            meta["gwaslab"]["variants"]["min_minor_allele_freq"]=min (np.min(sumstats["EAF"]) , 1- np.max(sumstats["EAF"]))
+        if "N" in sumstats.columns:
+            meta["gwaslab"]["samples"]["sample_size"] = int(sumstats["N"].max())
+            meta["gwaslab"]["samples"]["sample_size_median"] = sumstats["N"].median()
+            meta["gwaslab"]["samples"]["sample_size_min"] = int(sumstats["N"].min())
+    
+    if  meta["gwaslab"]["gwaslab_object"]=="gwaslab.SumstatsMulti" or meta["gwaslab"]["gwaslab_object"]=="gwaslab.SumstatsPair":   
+        nstudy = meta["gwaslab"]['number_of_studies']
+        for i in range(nstudy):
+            i_form_1 = i + 1
+            meta["gwaslab"]["variants"][i_form_1]=dict()
+            meta["gwaslab"]["samples"][i_form_1] =dict()
+
+            if "P_{}".format(i_form_1) in sumstats.columns:
+                p = "P_{}".format(i_form_1)
+                
+                meta["gwaslab"]["variants"][i_form_1]["min_P"]= np.nanmin(sumstats[p])
+            if "N_{}".format(i_form_1) in sumstats.columns:
+                n = "N_{}".format(i_form_1)
+                meta["gwaslab"]["samples"][i_form_1]["sample_size"] = int(sumstats[n].max())
+                meta["gwaslab"]["samples"][i_form_1]["sample_size_median"] = sumstats[n].median()
+                meta["gwaslab"]["samples"][i_form_1]["sample_size_min"] = int(sumstats[n].min())
+            if "EAF_{}".format(i_form_1) in sumstats.columns:
+                eaf="EAF_{}".format(i_form_1)
+                meta["gwaslab"]["variants"][i_form_1]["min_minor_allele_freq"]=min (np.min(sumstats[eaf]) , 1- np.max(sumstats[eaf]))
+
+    if meta["gwaslab"]["genome_build"] == "99":
+        _, meta["gwaslab"]["genome_build"] = inferbuild(sumstats, change_status=False, log=log, verbose=verbose)
+    
+    meta["date_last_modified"] = str(time.strftime('%Y/%m/%d'))
+    
+    return meta

gwaslab/g_vchange_status_polars.py

@@ -0,0 +1,44 @@
+import pandas as pd
+import polars as pl
+
+def vchange_statusp(sumstats, matched_index, status, digit, before, after):
+    dic={}
+    for i in range(len(before)):
+        dic[before[i]]=after[i]
+    
+    sumstats = sumstats.with_columns(pl.col(status).cast(pl.String).alias(status))
+
+    if digit>1:
+        sumstats = sumstats.with_columns(
+            pl.when( matched_index )  
+                .then(   pl.col(status).str.slice(0,digit-1) +   pl.col(status).str.slice(digit-1,1).str.replace_many(dic)  + pl.col(status).str.slice(digit))  
+                .otherwise( pl.col(status) )
+                .alias(status)
+        )
+    else:
+        sumstats = sumstats.with_columns(
+            pl.when( matched_index )  
+                .then( pl.col(status).str.slice(0,1).str.replace_many(dic)  + pl.col(status).str.slice(digit)  )  
+                .otherwise( pl.col(status) )
+                .alias(status)
+        )
+    return sumstats
+
+def copy_statusp(sumstats, matched_index, from_status, to_status, digit):
+    sumstats = sumstats.with_columns(pl.col(from_status).cast(pl.String).alias(from_status))
+    sumstats = sumstats.with_columns(pl.col(to_status).cast(pl.String).alias(to_status))
+    if digit>1:
+        sumstats = sumstats.with_columns(
+            pl.when( matched_index )  
+                .then(   pl.col(from_status).str.slice(0,digit-1) +   pl.col(to_status).str.slice(digit-1,1)  + pl.col(from_status).str.slice(digit))  
+                .otherwise( pl.col(to_status) )
+                .alias(to_status)
+        )
+    else:
+        sumstats = sumstats.with_columns(
+            pl.when( matched_index )  
+                .then( pl.col(from_status).str.slice(0,1)  + pl.col(to_status).str.slice(digit)  )  
+                .otherwise( pl.col(to_status) )
+                .alias(to_status)
+        )
+    return sumstats

gwaslab/g_version.py

@@ -15,8 +15,8 @@ def _get_version():
 def gwaslab_info():
     # version meta information
     dic={
-   "version":"3.5.7",
-   "release_date":"20250307"
+   "version":"3.5.8",
+   "release_date":"20250424"
     }
     return dic