gwaslab 3.5.7__py3-none-any.whl → 3.6.0__py3-none-any.whl

gwaslab/__init__.py

@@ -1,6 +1,8 @@
 from gwaslab.g_Sumstats import Sumstats
+from gwaslab.g_Sumstats_polars import Sumstatsp
 from gwaslab.g_SumstatsT import SumstatsT
 from gwaslab.g_SumstatsPair import SumstatsPair
+from gwaslab.g_SumstatsMulti import SumstatsMulti
 from gwaslab.util_in_convert_h2 import h2_obs_to_liab
 from gwaslab.util_in_convert_h2 import _get_per_snp_r2
 from gwaslab.util_in_convert_h2 import h2_se_to_p

gwaslab/bd_common_data.py

@@ -337,6 +337,7 @@ def _maketrans(complement_mapping):
     """
     keys = "".join(complement_mapping.keys()).encode("ASCII")
     values = "".join(complement_mapping.values()).encode("ASCII")
+
     return bytes.maketrans(keys + keys.lower(), values + values.lower())
         
 ####################################################################################################################   

gwaslab/bd_get_hapmap3.py

@@ -43,7 +43,6 @@ def gethapmap3(sumstats,rsid="rsID",chrom="CHR", pos="POS", ea="EA", nea="NEA",b
     else:
         additional_cols=[]
     hapmap3_ref = pd.read_csv(data_path,sep="\s+",usecols=["#CHROM","POS","rsid"]+additional_cols, dtype={"#CHROM":"string","POS":"string"})
-
     #rsid    A1      A2      #CHROM  POS
     #rs3094315       G       A       1       752566
     

gwaslab/data/formatbook.json

@@ -237,6 +237,16 @@
             "TotalSampleSize": "N",
             "Nsample": "N",
             "num_samples": "N",
+            "Neff": "N_EFF",
+            "N_EFF": "N_EFF",
+            "N_CASE": "N_CASE",
+            "Ncase": "N_CASE",
+            "ncase": "N_CASE",
+            "n_case": "N_CASE",
+            "Ncontrol": "N_CONTROL",
+            "N_control": "N_CONTROL",
+            "N_Control": "N_CONTROL",
+            "NCONTROL": "N_CONTROL",
             "beta": "BETA",
             "BETA": "BETA",
             "Beta": "BETA",
@@ -461,6 +471,31 @@
             "Direction": "DIRECTION"
         }
     },
+    "genomicsem": {
+        "meta_data": {
+            "format_name": "genomicSEM",
+            "format_source": "https://github.com/GenomicSEM/GenomicSEM/wiki/4.-Common-Factor-GWAS",
+            "format_source2": "https://github.com/GenomicSEM/GenomicSEM/wiki/5.-Multivariate-GWAS",
+            "format_version": 20241210
+        },
+        "format_dict": {
+            "SNP": "rsID",
+            "A2": "NEA",
+            "A1": "EA",
+            "Frq": "EAF",
+            "MAF": "MAF",
+            "N": "N",
+            "est": "BETA",
+            "se_c": "SE",
+            "Pval_Estimate": "P",
+            "Z_Estimate": "Z",
+            "Q": "Q",
+            "Q_df": "DOF",
+            "Q_pval": "P_HET",
+            "CHR": "CHR",
+            "BP": "POS"
+        }
+    },
     "plink_fam": {
         "meta_data": {
             "format_name": "plink_fam",
@@ -880,6 +915,21 @@
             "POS": "POS"
         }
     },
+    "mesusie": {
+        "meta_data": {
+            "format_name": "MESuSiE",
+            "format_source": "https://borangao.github.io/meSuSie_Analysis/installation.html",
+            "format_version": 20221109
+        },
+        "format_dict": {
+            "SNP": "SNPID",
+            "N": "N",
+            "Beta": "BETA",
+            "Se": "SE",
+            "Z": "Z",
+            "POS": "POS"
+        }
+    },
     "plink2_linear": {
         "meta_data": {
             "format_name": "PLINK2 .glm.linear",
@@ -1032,6 +1082,34 @@
             "ci_upper": "OR_95U"
         }
     },
+    "ccgwas": {
+        "meta_data": {
+            "format_name": "CCGWAS",
+            "format_source": "https://github.com/wouterpeyrot/CCGWAS",
+            "format_version": "20220901",
+            "last_check_date": "20250416",
+            "format_separator": "\t",
+            "format_citation": "Peyrot, W. J., & Price, A. L. (2021). Identifying loci with different allele frequencies among cases of eight psychiatric disorders using CC-GWAS. Nature genetics, 53(4), 445-454.",
+            "format_comment": null,
+            "format_na": null,
+            "format_other_cols": [
+                "Exact_beta",
+                "Exact_se",
+                "Exact_pval",
+                "CCGWAS_signif"
+            ]
+        },
+        "format_dict": {
+            "SNP": "SNPID",
+            "CHR": "CHR",
+            "BP": "POS",
+            "A2": "NEA",
+            "A1": "EA",
+            "OLS_beta": "BETA",
+            "OLS_se": "SE",
+            "OLS_pval": "P"
+        }
+    },
     "fastgwa": {
         "meta_data": {
             "format_name": "fastgwa",

gwaslab/data/reference.json

@@ -103,7 +103,9 @@
     "13to19":"https://s3-us-west-2.amazonaws.com/human-pangenomics/T2T/CHM13/assemblies/chain/v1_nflo/chm13v2-hg19.chain",
     "13to38":"https://s3-us-west-2.amazonaws.com/human-pangenomics/T2T/CHM13/assemblies/chain/v1_nflo/chm13v2-grch38.chain",
     "18to19":"https://hgdownload.soe.ucsc.edu/goldenPath/hg18/liftOver/hg18ToHg19.over.chain.gz",
-    "18to38":"https://hgdownload.soe.ucsc.edu/goldenPath/hg18/liftOver/hg18ToHg38.over.chain.gz"
+    "18to38":"https://hgdownload.soe.ucsc.edu/goldenPath/hg18/liftOver/hg18ToHg38.over.chain.gz",
+    "1kg_hm3_hg38_eaf":"https://www.dropbox.com/scl/fi/ymkqfsaec6mwjzlvxsm45/PAN.hapmap3.hg38.EAF.tsv.gz?rlkey=p1auef5y1kk7ui41k6j3s8b0z&dl=1",
+    "1kg_hm3_hg19_eaf":"https://www.dropbox.com/scl/fi/dmv9wtfchv6ahim86d49r/PAN.hapmap3.hg19.EAF.tsv.gz?rlkey=ywne2gj1rlm2nj42q9lt2d99n&dl=1"
 }
 
 

gwaslab/g_Sumstats.py

@@ -39,6 +39,7 @@ from gwaslab.util_in_filter_value import filterregionout
 from gwaslab.util_in_filter_value import _filter_indel
 from gwaslab.util_in_filter_value import _filter_palindromic
 from gwaslab.util_in_filter_value import _filter_snp
+from gwaslab.util_in_filter_value import _filter_region
 from gwaslab.util_in_filter_value import _exclude_hla
 from gwaslab.util_in_filter_value import _search_variants
 from gwaslab.util_in_filter_value import inferbuild
@@ -67,26 +68,35 @@ from gwaslab.g_version import _show_version
 from gwaslab.g_version import gwaslab_info
 from gwaslab.g_meta import _init_meta
 from gwaslab.g_meta import _append_meta_record
+from gwaslab.g_meta_update import _update_meta
 from gwaslab.util_ex_run_clumping import _clump
 from gwaslab.util_ex_calculate_ldmatrix import tofinemapping
+from gwaslab.io_load_ld import tofinemapping_using_ld
 from gwaslab.util_ex_calculate_prs import _calculate_prs
 from gwaslab.viz_plot_mqqplot import mqqplot
 from gwaslab.viz_plot_trumpetplot import plottrumpet
 from gwaslab.viz_plot_compare_af import plotdaf
 from gwaslab.util_ex_run_susie import _run_susie_rss
+from gwaslab.util_ex_run_susie import _get_cs_lead
 from gwaslab.qc_fix_sumstats import _check_data_consistency
 from gwaslab.util_ex_ldsc import _estimate_h2_by_ldsc
 from gwaslab.util_ex_ldsc import _estimate_rg_by_ldsc
 from gwaslab.util_ex_ldsc import _estimate_h2_cts_by_ldsc
 from gwaslab.util_ex_ldsc import _estimate_partitioned_h2_by_ldsc 
 from gwaslab.util_ex_ldproxyfinder import _extract_ld_proxy
+from gwaslab.util_ex_run_magma import _run_magma
+from gwaslab.util_ex_infer_ancestry import _infer_ancestry
 from gwaslab.bd_get_hapmap3 import gethapmap3
 from gwaslab.util_abf_finemapping import abf_finemapping
 from gwaslab.util_abf_finemapping import make_cs
 from gwaslab.io_read_pipcs import _read_pipcs
+from gwaslab.util_in_estimate_ess import _get_ess
 from gwaslab.viz_plot_credible_sets import _plot_cs
+from gwaslab.hm_casting import _align_with_mold
+from gwaslab.hm_casting  import _merge_mold_with_sumstats_by_chrpos
 import gc
 from gwaslab.viz_plot_phe_heatmap import _gwheatmap
+from gwaslab.util_ex_run_prscs import _run_prscs
 
 #20220309
 class Sumstats():
@@ -113,6 +123,7 @@ class Sumstats():
              f=None,
              t=None,
              p=None,
+             q=None,
              mlog10p=None,
              test=None,
              info=None,
@@ -126,6 +137,7 @@ class Sumstats():
              HR_95U=None,
              ncase=None,
              ncontrol=None,
+             neff=None,
              i2=None,
              phet=None,
              dof=None,
@@ -149,7 +161,7 @@ class Sumstats():
         self.log = Log()
         self.ldsc_h2 = None
         self.ldsc_h2_results = None
-        self.ldsc_rg = None
+        self.ldsc_rg = pd.DataFrame()
         self.ldsc_h2_cts = None
         self.ldsc_partitioned_h2_summary = None
         self.ldsc_partitioned_h2_results = None
@@ -200,6 +212,7 @@ class Sumstats():
           f=f,
           t=t,
           p=p,
+          q=q,
           mlog10p=mlog10p,
           test=test,
           info=info,
@@ -217,6 +230,7 @@ class Sumstats():
           snpr2=snpr2,
           ncase=ncase,
           ncontrol=ncontrol,
+          neff=neff,
           direction=direction,
           study=study,
           build=build,
@@ -243,18 +257,8 @@ class Sumstats():
         gc.collect()   
 
 #### healper #################################################################################
-    def update_meta(self):
-        self.meta["gwaslab"]["variants"]["variant_number"]=len(self.data)
-        if "CHR" in self.data.columns:
-            self.meta["gwaslab"]["variants"]["number_of_chromosomes"]=len(self.data["CHR"].unique())
-        if "P" in self.data.columns:
-            self.meta["gwaslab"]["variants"]["min_P"]=np.min(self.data["P"])
-        if "EAF" in self.data.columns:
-            self.meta["gwaslab"]["variants"]["min_minor_allele_freq"]=min (np.min(self.data["EAF"]) , 1- np.max(self.data["EAF"]))
-        if "N" in self.data.columns:
-            self.meta["gwaslab"]["samples"]["sample_size"] = int(self.data["N"].max())
-            self.meta["gwaslab"]["samples"]["sample_size_median"] = self.data["N"].median()
-            self.meta["gwaslab"]["samples"]["sample_size_min"] = int(self.data["N"].min())
+    def update_meta(self, **kwargs):
+        self.meta = _update_meta(self.meta, self.data,log = self.log, **kwargs)
 
     def summary(self):
         return summarize(self.data)
@@ -363,7 +367,8 @@ class Sumstats():
             self.data = parallelnormalizeallele(self.data,log=self.log,n_cores=n_cores,**normalizeallele_args)
             
             self.data = sortcolumn(self.data,log=self.log)
-            
+
+            self.data = sortcoordinate(self.data,log=self.log)
             gc.collect()
         
         #####################################################
@@ -380,6 +385,7 @@ class Sumstats():
         #   3.2 infer strand for palindromic SNP (target build)
         #####################################################
         if ref_seq is not None:
+
             if ref_seq_mode=="v":
                 self.data = checkref(self.data,ref_seq,log=self.log,**checkref_args)
             elif ref_seq_mode=="s":
@@ -441,6 +447,20 @@ class Sumstats():
         self.meta["is_sorted"] = True
         self.meta["is_harmonised"] = True
         return self
+    
+    def align_with_template(self, template, **kwargs):
+        ## merge
+        molded_sumstats, sumstats1 = _merge_mold_with_sumstats_by_chrpos(mold=template, 
+                                            sumstats=self.data,
+                                            log=self.log,
+                                            suffixes=("_MOLD",""),
+                                            return_not_matched_mold = True)
+        ## align
+        aligned_data = _align_with_mold(molded_sumstats)
+
+        ## flip
+        self.data =flipallelestats(aligned_data, log=self.log)
+        
     ############################################################################################################
     #customizable API to build your own QC pipeline
     def fix_id(self,**kwargs):
@@ -507,6 +527,15 @@ class Sumstats():
 
 # utilities ############################################################################################################
     # filter series ######################################################################
+    
+    def filter_region(self, inplace=False,**kwargs):
+        if inplace is False:
+            new_Sumstats_object = copy.deepcopy(self)
+            new_Sumstats_object.data = _filter_region(new_Sumstats_object.data, **kwargs)
+            return new_Sumstats_object
+        else:
+            self.data = _filter_region(self.data, **kwargs)
+
     def filter_flanking(self, inplace=False,**kwargs):
         if inplace is False:
             new_Sumstats_object = copy.deepcopy(self)
@@ -647,6 +676,9 @@ class Sumstats():
         fig,outliers = plotdaf(self.data, **kwargs)
         return fig, outliers
     
+    def infer_ancestry(self, **kwargs):
+        self.meta["gwaslab"]["inferred_ancestry"] = _infer_ancestry(self.data, **kwargs)
+
     def plot_gwheatmap(self, **kwargs):
         fig = _gwheatmap(self.data, **kwargs)
         return fig
@@ -749,6 +781,7 @@ class Sumstats():
                            chrom="CHR",
                            pos="POS",
                            p="P",
+                           build=self.meta["gwaslab"]["genome_build"],
                            log=self.log,
                            **kwargs)
         # return sumstats object    
@@ -789,7 +822,22 @@ class Sumstats():
                            **kwargs)
         # return sumstats object    
         return output
-    
+
+    def check_cs_overlap(self, **kwargs):
+        if "SNPID" in self.pipcs.columns:
+            id_to_use = "SNPID"
+        else:
+            id_to_use = "rsID"
+        output = _check_novel_set(self.pipcs,
+                           id=id_to_use,
+                           chrom="CHR",
+                           pos="POS",
+                           p="P",
+                           log=self.log,
+                           **kwargs)
+        # return sumstats object    
+        return output
+
     def anno_gene(self, **kwargs):
         if "SNPID" in self.data.columns:
             id_to_use = "SNPID"
@@ -806,6 +854,9 @@ class Sumstats():
     def get_per_snp_r2(self,**kwargs):
         self.data = _get_per_snp_r2(self.data, beta="BETA", af="EAF", n="N", log=self.log, **kwargs)
         #add data inplace
+        
+    def get_ess(self, **kwargs):
+        self.data = _get_ess(self.data, log=self.log, **kwargs)
 
     def get_gc(self, mode=None, **kwargs):
         if mode is None:
@@ -831,40 +882,73 @@ class Sumstats():
         credible_sets = make_cs(region_data,threshold=0.95,log=self.log)
         return region_data, credible_sets
     
-
+######################################################################################################
+    def run_prscs(self, build=None, verbose=True, match_allele=True, how="inner", **kwargs):
+        if build is None:
+            build = self.meta["gwaslab"]["genome_build"]
+        insumstats = gethapmap3(self.data.copy(), build=build, verbose=verbose , match_allele=match_allele, how=how )
+        _run_prscs(sst_file = insumstats[["rsID","CHR","POS","EA","NEA","BETA","SE"]], 
+                   log=self.log, 
+                   **kwargs)
+    
+    def run_magma(self, build=None, verbose=True, **kwargs):
+        _run_magma(self.data, 
+                   study=self.meta["gwaslab"]["study_name"], 
+                   build=build, verbose=verbose, log=self.log, **kwargs)
 ## LDSC ##############################################################################################
     def estimate_h2_by_ldsc(self, build=None, verbose=True, match_allele=True, how="right", **kwargs):
         if build is None:
             build = self.meta["gwaslab"]["genome_build"]
         insumstats = gethapmap3(self.data.copy(), build=build, verbose=verbose , match_allele=match_allele, how=how )
-        self.ldsc_h2, self.ldsc_h2_results = _estimate_h2_by_ldsc(insumstats=insumstats, log=self.log, verbose=verbose, **kwargs)
+        self.ldsc_h2, self.ldsc_h2_results = _estimate_h2_by_ldsc(insumstats=insumstats, 
+                                                                  meta=self.meta,
+                                                                  log=self.log, 
+                                                                  verbose=verbose, 
+                                                                  **kwargs)
     
     def estimate_rg_by_ldsc(self, build=None, verbose=True, match_allele=True, how="right",**kwargs):
         if build is None:
             build = self.meta["gwaslab"]["genome_build"]
         insumstats = gethapmap3(self.data.copy(), build=build, verbose=verbose , match_allele=match_allele, how=how )
-        self.ldsc_rg = _estimate_rg_by_ldsc(insumstats=insumstats, log=self.log, verbose=verbose, **kwargs)
+        ldsc_rg = _estimate_rg_by_ldsc(insumstats=insumstats,
+                                             meta=self.meta,
+                                             log=self.log, 
+                                             verbose=verbose, 
+                                             **kwargs)
+        self.ldsc_rg = pd.concat([self.ldsc_rg, ldsc_rg],ignore_index=True)
 
     def estimate_h2_cts_by_ldsc(self, build=None, verbose=True, match_allele=True, how="right",**kwargs):
         if build is None:
             build = self.meta["gwaslab"]["genome_build"]
         insumstats = gethapmap3(self.data.copy(), build=build, verbose=verbose , match_allele=match_allele, how=how )
-        self.ldsc_h2_cts  = _estimate_h2_cts_by_ldsc(insumstats=insumstats, log=self.log, verbose=verbose, **kwargs)
+        self.ldsc_h2_cts  = _estimate_h2_cts_by_ldsc(insumstats=insumstats, 
+                                                     log=self.log,
+                                                       verbose=verbose, 
+                                                       **kwargs)
     
     def estimate_partitioned_h2_by_ldsc(self, build=None, verbose=True, match_allele=True, how="right",**kwargs):
         if build is None:
             build = self.meta["gwaslab"]["genome_build"]
         insumstats = gethapmap3(self.data.copy(), build=build, verbose=verbose , match_allele=match_allele, how=how )
-        self.ldsc_partitioned_h2_summary, self.ldsc_partitioned_h2_results  = _estimate_partitioned_h2_by_ldsc(insumstats=insumstats, log=self.log, verbose=verbose, **kwargs)
+        self.ldsc_partitioned_h2_summary, self.ldsc_partitioned_h2_results  = _estimate_partitioned_h2_by_ldsc(insumstats=insumstats, 
+                                                                                                               meta=self.meta,
+                                                                                                               log=self.log, 
+                                                                                                               verbose=verbose, 
+                                                                                                               **kwargs)
 # external ################################################################################################
     
     def calculate_ld_matrix(self,**kwargs):
         self.finemapping["path"],self.finemapping["file"],self.finemapping["plink_log"]= tofinemapping(self.data,study = self.meta["gwaslab"]["study_name"],**kwargs)
         #self.to_finemapping_file_path, self.to_finemapping_file, self.plink_log  = tofinemapping(self.data,study = self.meta["gwaslab"]["study_name"],**kwargs)
-    
+    def extract_ld_matrix(self,**kwargs):
+        self.finemapping["path"],self.finemapping["file"],self.finemapping["plink_log"]= tofinemapping_using_ld(self.data,study = self.meta["gwaslab"]["study_name"],**kwargs)
+
     def run_susie_rss(self,**kwargs):
-        self.pipcs=_run_susie_rss(self.finemapping["path"],**kwargs)
+        self.pipcs=_run_susie_rss(self.finemapping["path"], main_sumstats = self.data[["SNPID","CHR","POS"]], **kwargs)
+        self.finemapping["pipcs"] = self.pipcs
         #self.pipcs=_run_susie_rss(self.to_finemapping_file_path,**kwargs)
+    def get_cs_lead(self,**kwargs):
+        return _get_cs_lead(self.pipcs,**kwargs)
     
     def clump(self,**kwargs):
         self.clumps["clumps"], self.clumps["clumps_raw"], self.clumps["plink_log"] = _clump(self.data, log=self.log, study = self.meta["gwaslab"]["study_name"], **kwargs)
@@ -875,10 +959,11 @@ class Sumstats():
 
 # loading aux data
     def read_pipcs(self,prefix,**kwargs):
-        self.pipcs = _read_pipcs(self.data[["SNPID","CHR","POS"]],prefix, **kwargs)
+        self.pipcs = _read_pipcs(self.data[["SNPID","CHR","POS"]],prefix, study= self.meta["gwaslab"]["study_name"], **kwargs)
+        self.finemapping["pipcs"] = self.pipcs
 
-    def plot_pipcs(self, region,**kwargs):
-        _plot_cs(self.pipcs, region, **kwargs)
+    def plot_pipcs(self, region=None, locus=None, **kwargs):
+        _plot_cs(self.pipcs, region=region,locus=locus, **kwargs)
 # to_format ###############################################################################################       
 
     def to_format(self, path, build=None, verbose=True, **kwargs):