gwaslab 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

gwaslab/{ldsc_irwls.py → extension/ldsc/ldsc_irwls.py}

@@ -6,7 +6,7 @@ Iterativey re-weighted least squares.
 '''
 from __future__ import division
 import numpy as np
-import gwaslab.ldsc_jackknife as jk
+import gwaslab.extension.ldsc.ldsc_jackknife as jk
 
 def xrange(*args):
     return range(*args)

gwaslab/{ldsc_regressions.py → extension/ldsc/ldsc_regressions.py}

@@ -11,8 +11,8 @@ from __future__ import division
 import numpy as np
 import pandas as pd
 from scipy.stats import norm, chi2
-import gwaslab.ldsc_jackknife as jk
-from gwaslab.ldsc_irwls import IRWLS
+import gwaslab.extension.ldsc.ldsc_jackknife as jk
+from gwaslab.extension.ldsc.ldsc_irwls import IRWLS
 from scipy.stats import t as tdist
 from collections import namedtuple
 np.seterr(divide='raise', invalid='raise')

gwaslab/{ldsc_sumstats.py → extension/ldsc/ldsc_sumstats.py}

@@ -10,8 +10,8 @@ import numpy as np
 import pandas as pd
 from scipy import stats
 import itertools as it
-import gwaslab.ldsc_parse as ps
-import gwaslab.ldsc_regressions as reg
+import gwaslab.extension.ldsc.ldsc_parse as ps
+import gwaslab.extension.ldsc.ldsc_regressions as reg
 import sys
 import traceback
 import copy

gwaslab/{prscs_mcmc_gtb.py → extension/prscs/prscs_mcmc_gtb.py}

@@ -7,7 +7,7 @@ Markov Chain Monte Carlo (MCMC) sampler for polygenic prediction with continuous
 import numpy as np
 from scipy import linalg 
 from numpy import random
-from gwaslab.prscs_gigrnd import gigrnd
+from gwaslab.extension.prscs.prscs_gigrnd import gigrnd
 from gwaslab.g_Log import Log
 import time
 def mcmc(a, b, phi, sst_dict, n, ld_blk, blk_size, n_iter, n_burnin, thin, chrom, out_dir, beta_std, write_psi, write_pst, seed, log):

gwaslab/g_Sumstats.py

@@ -5,99 +5,110 @@ import copy
 import gc
 from gwaslab.g_Sumstats_summary import summarize
 from gwaslab.g_Sumstats_summary import lookupstatus
-from gwaslab.io_preformat_input import preformat
-from gwaslab.io_to_formats import _to_format
-from gwaslab.g_Log import Log
-from gwaslab.qc_fix_sumstats import fixID
-from gwaslab.qc_fix_sumstats import flipSNPID
-from gwaslab.qc_fix_sumstats import stripSNPID
-from gwaslab.qc_fix_sumstats import removedup
-from gwaslab.qc_fix_sumstats import fixchr
-from gwaslab.qc_fix_sumstats import fixpos
-from gwaslab.qc_fix_sumstats import fixallele
-from gwaslab.qc_fix_sumstats import parallelnormalizeallele
-from gwaslab.qc_fix_sumstats import sanitycheckstats
-from gwaslab.qc_fix_sumstats import parallelizeliftovervariant
-from gwaslab.qc_fix_sumstats import flipallelestats
-from gwaslab.qc_fix_sumstats import sortcoordinate
-from gwaslab.qc_fix_sumstats import sortcolumn
-from gwaslab.qc_fix_sumstats import _set_build
-from gwaslab.qc_fix_sumstats import _process_build
-from gwaslab.hm_harmonize_sumstats import parallelecheckaf
-from gwaslab.hm_harmonize_sumstats import paralleleinferaf
-from gwaslab.hm_harmonize_sumstats import checkref
-from gwaslab.hm_harmonize_sumstats import oldcheckref
-from gwaslab.hm_harmonize_sumstats import rsidtochrpos
-from gwaslab.hm_harmonize_sumstats import parallelizeassignrsid
-from gwaslab.hm_harmonize_sumstats import parallelinferstrand
-from gwaslab.hm_harmonize_sumstats import parallelrsidtochrpos
-from gwaslab.hm_harmonize_sumstats import _paralleleinferafwithmaf
-from gwaslab.util_in_filter_value import filtervalues
-from gwaslab.util_in_filter_value import filterout
-from gwaslab.util_in_filter_value import filterin
-from gwaslab.util_in_filter_value import filterregionin
-from gwaslab.util_in_filter_value import filterregionout
-from gwaslab.util_in_filter_value import _filter_indel
-from gwaslab.util_in_filter_value import _filter_palindromic
-from gwaslab.util_in_filter_value import _filter_snp
-from gwaslab.util_in_filter_value import _filter_region
-from gwaslab.util_in_filter_value import _exclude_hla
-from gwaslab.util_in_filter_value import _search_variants
-from gwaslab.util_in_filter_value import inferbuild
-from gwaslab.util_in_filter_value import sampling
-from gwaslab.util_in_filter_value import _get_flanking
-from gwaslab.util_in_filter_value import _get_flanking_by_chrpos
-from gwaslab.util_in_filter_value import _get_flanking_by_id
-from gwaslab.util_in_calculate_gc import lambdaGC
-from gwaslab.util_in_convert_h2 import _get_per_snp_r2
-from gwaslab.util_in_get_sig import getsig
-from gwaslab.util_in_get_density import getsignaldensity
-from gwaslab.util_in_get_density import assigndensity
-from gwaslab.util_in_get_sig import annogene
-from gwaslab.util_in_get_sig import getnovel
-from gwaslab.util_in_get_sig import _check_cis
-from gwaslab.util_in_get_sig import _check_novel_set
-from gwaslab.util_in_fill_data import filldata
-from gwaslab.bd_get_hapmap3 import gethapmap3
-from gwaslab.bd_common_data import get_chr_list
-from gwaslab.bd_common_data import get_number_to_chr
-from gwaslab.bd_common_data import get_chr_to_number
-from gwaslab.bd_common_data import get_high_ld
-from gwaslab.bd_common_data import get_format_dict
-from gwaslab.bd_common_data import get_formats_list
 from gwaslab.g_version import _show_version
 from gwaslab.g_version import gwaslab_info
 from gwaslab.g_meta import _init_meta
 from gwaslab.g_meta import _append_meta_record
 from gwaslab.g_meta_update import _update_meta
-from gwaslab.io_load_ld import tofinemapping_using_ld
-from gwaslab.qc_fix_sumstats import _check_data_consistency
-from gwaslab.bd_get_hapmap3 import gethapmap3
-from gwaslab.util_abf_finemapping import abf_finemapping
-from gwaslab.util_abf_finemapping import make_cs
-from gwaslab.io_read_pipcs import _read_pipcs
-from gwaslab.util_in_estimate_ess import _get_ess
-from gwaslab.hm_casting import _align_with_mold
-from gwaslab.hm_casting  import _merge_mold_with_sumstats_by_chrpos
-from gwaslab.viz_plot_phe_heatmap import _gwheatmap
-from gwaslab.viz_plot_mqqplot import mqqplot
-from gwaslab.viz_plot_trumpetplot import plottrumpet
-from gwaslab.viz_plot_compare_af import plotdaf
-from gwaslab.viz_plot_credible_sets import _plot_cs
-from gwaslab.util_ex_run_clumping import _clump
-from gwaslab.util_ex_calculate_ldmatrix import tofinemapping
-from gwaslab.util_ex_run_susie import _run_susie_rss
-from gwaslab.util_ex_run_susie import _get_cs_lead
-from gwaslab.util_ex_ldsc import _estimate_h2_by_ldsc
-from gwaslab.util_ex_ldsc import _estimate_rg_by_ldsc
-from gwaslab.util_ex_ldsc import _estimate_h2_cts_by_ldsc
-from gwaslab.util_ex_ldsc import _estimate_partitioned_h2_by_ldsc 
-from gwaslab.util_ex_ldproxyfinder import _extract_ld_proxy
-from gwaslab.util_ex_run_magma import _run_magma
-from gwaslab.util_ex_infer_ancestry import _infer_ancestry
-from gwaslab.util_ex_run_prscs import _run_prscs
-from gwaslab.util_ex_calculate_prs import _calculate_prs
-from gwaslab.util_ex_run_scdrs import _run_scdrs
+from gwaslab.g_Log import Log
+
+from gwaslab.qc.qc_fix_sumstats import fixID
+from gwaslab.qc.qc_fix_sumstats import flipSNPID
+from gwaslab.qc.qc_fix_sumstats import stripSNPID
+from gwaslab.qc.qc_fix_sumstats import removedup
+from gwaslab.qc.qc_fix_sumstats import fixchr
+from gwaslab.qc.qc_fix_sumstats import fixpos
+from gwaslab.qc.qc_fix_sumstats import fixallele
+from gwaslab.qc.qc_fix_sumstats import parallelnormalizeallele
+from gwaslab.qc.qc_fix_sumstats import sanitycheckstats
+from gwaslab.qc.qc_fix_sumstats import parallelizeliftovervariant
+from gwaslab.qc.qc_fix_sumstats import flipallelestats
+from gwaslab.qc.qc_fix_sumstats import sortcoordinate
+from gwaslab.qc.qc_fix_sumstats import sortcolumn
+from gwaslab.qc.qc_fix_sumstats import _set_build
+from gwaslab.qc.qc_fix_sumstats import _process_build
+from gwaslab.qc.qc_fix_sumstats import _check_data_consistency
+
+from gwaslab.hm.hm_harmonize_sumstats import parallelecheckaf
+from gwaslab.hm.hm_harmonize_sumstats import paralleleinferaf
+from gwaslab.hm.hm_harmonize_sumstats import checkref
+from gwaslab.hm.hm_harmonize_sumstats import oldcheckref
+from gwaslab.hm.hm_harmonize_sumstats import rsidtochrpos
+from gwaslab.hm.hm_harmonize_sumstats import parallelizeassignrsid
+from gwaslab.hm.hm_harmonize_sumstats import parallelinferstrand
+from gwaslab.hm.hm_harmonize_sumstats import parallelrsidtochrpos
+from gwaslab.hm.hm_harmonize_sumstats import _paralleleinferafwithmaf
+
+from gwaslab.util.util_in_filter_value import filtervalues
+from gwaslab.util.util_in_filter_value import filterout
+from gwaslab.util.util_in_filter_value import filterin
+from gwaslab.util.util_in_filter_value import filterregionin
+from gwaslab.util.util_in_filter_value import filterregionout
+from gwaslab.util.util_in_filter_value import _filter_indel
+from gwaslab.util.util_in_filter_value import _filter_palindromic
+from gwaslab.util.util_in_filter_value import _filter_snp
+from gwaslab.util.util_in_filter_value import _filter_region
+from gwaslab.util.util_in_filter_value import _exclude_hla
+from gwaslab.util.util_in_filter_value import _search_variants
+from gwaslab.util.util_in_filter_value import inferbuild
+from gwaslab.util.util_in_filter_value import sampling
+from gwaslab.util.util_in_filter_value import _get_flanking
+from gwaslab.util.util_in_filter_value import _get_flanking_by_chrpos
+from gwaslab.util.util_in_filter_value import _get_flanking_by_id
+from gwaslab.util.util_in_calculate_gc import lambdaGC
+from gwaslab.util.util_in_convert_h2 import _get_per_snp_r2
+from gwaslab.util.util_in_get_sig import getsig
+from gwaslab.util.util_in_get_density import getsignaldensity
+from gwaslab.util.util_in_get_density import assigndensity
+from gwaslab.util.util_in_get_sig import annogene
+from gwaslab.util.util_in_get_sig import getnovel
+from gwaslab.util.util_in_get_sig import _check_cis
+from gwaslab.util.util_in_get_sig import _check_novel_set
+from gwaslab.util.util_ex_phewwas import _extract_associations
+from gwaslab.util.util_in_fill_data import filldata
+from gwaslab.util.util_ex_run_clumping import _clump
+from gwaslab.util.util_ex_calculate_ldmatrix import tofinemapping
+from gwaslab.util.util_ex_run_susie import _run_susie_rss
+from gwaslab.util.util_ex_run_susie import _get_cs_lead
+from gwaslab.util.util_ex_ldsc import _estimate_h2_by_ldsc
+from gwaslab.util.util_ex_ldsc import _estimate_rg_by_ldsc
+from gwaslab.util.util_ex_ldsc import _estimate_h2_cts_by_ldsc
+from gwaslab.util.util_ex_ldsc import _estimate_partitioned_h2_by_ldsc 
+from gwaslab.util.util_ex_ldproxyfinder import _extract_ld_proxy
+from gwaslab.util.util_ex_run_magma import _run_magma
+from gwaslab.util.util_ex_infer_ancestry import _infer_ancestry
+from gwaslab.util.util_ex_run_prscs import _run_prscs
+from gwaslab.util.util_ex_calculate_prs import _calculate_prs
+from gwaslab.util.util_ex_run_scdrs import _run_scdrs
+from gwaslab.util.util_in_estimate_ess import _get_ess
+from gwaslab.util.util_abf_finemapping import abf_finemapping
+from gwaslab.util.util_abf_finemapping import make_cs
+
+from gwaslab.bd.bd_get_hapmap3 import gethapmap3
+from gwaslab.bd.bd_common_data import get_chr_list
+from gwaslab.bd.bd_common_data import get_number_to_chr
+from gwaslab.bd.bd_common_data import get_chr_to_number
+from gwaslab.bd.bd_common_data import get_high_ld
+from gwaslab.bd.bd_common_data import get_format_dict
+from gwaslab.bd.bd_common_data import get_formats_list
+from gwaslab.bd.bd_path_manager import _path
+from gwaslab.bd.bd_get_hapmap3 import gethapmap3
+
+from gwaslab.hm.hm_casting import _align_with_mold
+from gwaslab.hm.hm_casting  import _merge_mold_with_sumstats_by_chrpos
+
+from gwaslab.viz.viz_plot_phe_heatmap import _gwheatmap
+from gwaslab.viz.viz_plot_mqqplot import mqqplot
+from gwaslab.viz.viz_plot_trumpetplot import plottrumpet
+from gwaslab.viz.viz_plot_compare_af import plotdaf
+from gwaslab.viz.viz_plot_credible_sets import _plot_cs
+
+from gwaslab.io.io_read_pipcs import _read_pipcs
+from gwaslab.io.io_load_ld import tofinemapping_using_ld
+from gwaslab.io.io_preformat_input import preformat
+from gwaslab.io.io_to_formats import _to_format
+from gwaslab.io.io_to_pickle import _offload
+from gwaslab.io.io_to_pickle import _reload
 
 #20220309
 class Sumstats():
@@ -173,6 +184,8 @@ class Sumstats():
         self.meta["gwaslab"]["study_name"] =  study
         self.meta["gwaslab"]["species"] = species
         
+
+        
         # initialize attributes for clumping and finmapping
         #self.to_finemapping_file_path = ""
         #self.to_finemapping_file  = pd.DataFrame()
@@ -246,7 +259,6 @@ class Sumstats():
           verbose=verbose,
           readargs=readargs,
           log=self.log)
-        
 
         # checking genome build
         self.meta["gwaslab"]["genome_build"] = _process_build(build, log=self.log, verbose=False)
@@ -258,6 +270,9 @@ class Sumstats():
             except:
                 pass
         gc.collect()   
+        
+        self.id = id(self)
+        self.tmp_path = _path(pid=self.id, log = self.log, verbose=verbose)
 
 #### healper #################################################################################
     def update_meta(self, **kwargs):
@@ -790,6 +805,13 @@ class Sumstats():
         # return sumstats object    
         return output
     
+    def get_associations(self, **kwargs):
+        associations_full,associations_summary  = _extract_associations(self.data,**kwargs)
+        
+        self.associations = associations_full
+        
+        return associations_summary
+
     def check_cis(self, gls=False, **kwargs):
         if "SNPID" in self.data.columns:
             id_to_use = "SNPID"
@@ -895,14 +917,15 @@ class Sumstats():
                    **kwargs)
     
     def run_magma(self, build=None, verbose=True, **kwargs):
-        _run_magma(self.data, 
+        _run_magma(self, 
                    study=self.meta["gwaslab"]["study_name"], 
                    build=build, verbose=verbose, log=self.log, **kwargs)
         
     def run_scdrs(self, build=None, verbose=True, **kwargs):
-        _run_scdrs(
+        _run_scdrs(self,
                    study=self.meta["gwaslab"]["study_name"], 
-                   build=build, verbose=verbose, log=self.log, **kwargs)
+                   build=build, 
+                   verbose=verbose, log=self.log, **kwargs)
 ## LDSC ##############################################################################################
     def estimate_h2_by_ldsc(self, build=None, verbose=True, match_allele=True, how="right", **kwargs):
         if build is None:
@@ -949,20 +972,24 @@ class Sumstats():
 # external ################################################################################################
     
     def calculate_ld_matrix(self,**kwargs):
-        self.finemapping["path"],self.finemapping["file"],self.finemapping["plink_log"]= tofinemapping(self.data,study = self.meta["gwaslab"]["study_name"],**kwargs)
+        self.finemapping["path"],self.finemapping["file"],self.finemapping["plink_log"]= tofinemapping(self, study = self.meta["gwaslab"]["study_name"],**kwargs)
         #self.to_finemapping_file_path, self.to_finemapping_file, self.plink_log  = tofinemapping(self.data,study = self.meta["gwaslab"]["study_name"],**kwargs)
     def extract_ld_matrix(self,**kwargs):
         self.finemapping["path"],self.finemapping["file"],self.finemapping["plink_log"]= tofinemapping_using_ld(self.data,study = self.meta["gwaslab"]["study_name"],**kwargs)
 
     def run_susie_rss(self,**kwargs):
-        self.pipcs=_run_susie_rss(self.finemapping["path"], main_sumstats = self.data[["SNPID","CHR","POS"]], **kwargs)
+        self.pipcs=_run_susie_rss(self, self.finemapping["path"], **kwargs)
         self.finemapping["pipcs"] = self.pipcs
         #self.pipcs=_run_susie_rss(self.to_finemapping_file_path,**kwargs)
+    
     def get_cs_lead(self,**kwargs):
         return _get_cs_lead(self.pipcs,**kwargs)
     
     def clump(self,**kwargs):
-        self.clumps["clumps"], self.clumps["clumps_raw"], self.clumps["plink_log"] = _clump(self.data, log=self.log, study = self.meta["gwaslab"]["study_name"], **kwargs)
+        self.clumps["clumps"], self.clumps["clumps_raw"], self.clumps["plink_log"] = _clump(self, 
+                                                                                            log=self.log, 
+                                                                                            study = self.meta["gwaslab"]["study_name"], 
+                                                                                            **kwargs)
 
     def calculate_prs(self,**kwargs):
         combined_results_summary = _calculate_prs(self.data, log=self.log, study = self.meta["gwaslab"]["study_name"], **kwargs)
@@ -982,4 +1009,11 @@ class Sumstats():
             build = self.meta["gwaslab"]["genome_build"]
         _to_format(self.data, path, log=self.log, verbose=verbose, meta=self.meta, build=build, **kwargs)
 
- 
+######################################################################################
+    def offload(self):
+        _offload(self.data, self.tmp_path, self.log)
+        del self.data
+        gc.collect()
+
+    def reload(self):
+         self.data = _reload(self.tmp_path, self.log)

gwaslab/g_SumstatsMulti.py

@@ -2,49 +2,60 @@ import pandas as pd
 import numpy as np
 import copy
 import gc
-from gwaslab.util_in_filter_value import filtervalues
+
 from gwaslab.g_Log import Log
 from math import floor
 from gwaslab.g_Sumstats import Sumstats
 from gwaslab.g_Sumstats_polars import Sumstatsp
-from gwaslab.hm_casting import _merge_mold_with_sumstats_by_chrpos
-from gwaslab.hm_casting import _align_with_mold
-from gwaslab.hm_casting import _fill_missing_columns
-from gwaslab.hm_casting import _check_daf
-from gwaslab.hm_casting import _assign_warning_code
-from gwaslab.hm_casting import _renaming_cols
-from gwaslab.hm_casting import _sort_pair_cols
-
-from gwaslab.hm_casting_polars import _merge_mold_with_sumstats_by_chrposp
-from gwaslab.hm_casting_polars import _align_with_moldp
-from gwaslab.hm_casting_polars import _fill_missing_columnsp
-from gwaslab.hm_casting_polars import _renaming_colsp
-from gwaslab.hm_casting_polars import _sort_pair_colsp
-
-from gwaslab.qc_fix_sumstats import flipallelestats
-from gwaslab.qc_fix_sumstats_polars import flipallelestatsp
-from gwaslab.qc_check_datatype_polars import check_datatype
-from gwaslab.qc_check_datatype_polars import check_dataframe_shape
-from gwaslab.util_ex_calculate_ldmatrix import tofinemapping
-from gwaslab.util_ex_run_coloc import _run_coloc_susie
-from gwaslab.viz_plot_miamiplot2 import plot_miami2
-from gwaslab.viz_plot_compare_af import  plotdaf
-from gwaslab.util_ex_run_2samplemr import _run_two_sample_mr
-from gwaslab.util_ex_run_clumping import _clump
-from gwaslab.util_ex_ldproxyfinder import _extract_with_ld_proxy
+
+from gwaslab.bd.bd_path_manager import _path
+
+from gwaslab.hm.hm_casting import _merge_mold_with_sumstats_by_chrpos
+from gwaslab.hm.hm_casting import _align_with_mold
+from gwaslab.hm.hm_casting import _fill_missing_columns
+from gwaslab.hm.hm_casting import _check_daf
+from gwaslab.hm.hm_casting import _assign_warning_code
+from gwaslab.hm.hm_casting import _renaming_cols
+from gwaslab.hm.hm_casting import _sort_pair_cols
+from gwaslab.hm.hm_casting_polars import _merge_mold_with_sumstats_by_chrposp
+from gwaslab.hm.hm_casting_polars import _align_with_moldp
+from gwaslab.hm.hm_casting_polars import _fill_missing_columnsp
+from gwaslab.hm.hm_casting_polars import _renaming_colsp
+from gwaslab.hm.hm_casting_polars import _sort_pair_colsp
+
+from gwaslab.qc.qc_fix_sumstats import flipallelestats
+from gwaslab.qc.qc_fix_sumstats_polars import flipallelestatsp
+
+from gwaslab.qc.qc_check_datatype_polars import check_datatype_polars
+from gwaslab.qc.qc_check_datatype_polars import check_dataframe_shape_polars
+
+from gwaslab.qc.qc_check_datatype import check_datatype
+from gwaslab.qc.qc_check_datatype import check_dataframe_shape
+from gwaslab.qc.qc_fix_sumstats import _process_build
+
+from gwaslab.util.util_ex_calculate_ldmatrix import tofinemapping
+from gwaslab.util.util_ex_run_coloc import _run_coloc_susie
+from gwaslab.util.util_in_filter_value import filtervalues
+from gwaslab.util.util_ex_run_2samplemr import _run_two_sample_mr
+from gwaslab.util.util_ex_run_clumping import _clump
+from gwaslab.util.util_ex_ldproxyfinder import _extract_with_ld_proxy
+from gwaslab.util.util_ex_match_ldmatrix import tofinemapping_m
+from gwaslab.util.util_ex_run_mesusie import _run_mesusie
+from gwaslab.util.util_in_meta import meta_analyze_multi
+from gwaslab.util.util_ex_run_hyprcoloc import _run_hyprcoloc
+from gwaslab.util.util_in_get_sig import getsig
+from gwaslab.util.util_in_fill_data import _get_multi_min
+from gwaslab.util.util_ex_run_mtag import _run_mtag
+
+from gwaslab.viz.viz_plot_miamiplot2 import plot_miami2
+from gwaslab.viz.viz_plot_compare_af import  plotdaf
+
 from gwaslab.g_headers import _get_headers
-from gwaslab.util_ex_match_ldmatrix import tofinemapping_m
-from gwaslab.util_ex_run_mesusie import _run_mesusie
-from gwaslab.util_in_meta import meta_analyze_multi
-from gwaslab.util_ex_run_hyprcoloc import _run_hyprcoloc
-from gwaslab.util_in_get_sig import getsig
-from gwaslab.util_in_fill_data import _get_multi_min
 from gwaslab.g_meta import _init_meta
 from gwaslab.g_meta_update import _update_meta
-from gwaslab.qc_fix_sumstats import _process_build
-from gwaslab.util_ex_run_mtag import _run_mtag
-
 
+from gwaslab.io.io_to_pickle import _offload
+from gwaslab.io.io_to_pickle import _reload
 
 class SumstatsMulti( ):
     def __init__(self, 
@@ -63,6 +74,10 @@ class SumstatsMulti( ):
         
         self.log = Log()
         self.meta = _init_meta(object="SumstatsMulti") 
+        self.id = id(self)
+        self.tmp_path = _path(pid=self.id, 
+                              log = self.log, 
+                              verbose=verbose)
         
         if engine=="polars":
             import polars as pl
@@ -94,10 +109,17 @@ class SumstatsMulti( ):
 
         for i,sumstatsObject in enumerate(sumstatsObjects):
             self.log.write( " -Checking sumstats Object #{}...".format(i+1), verbose=verbose)
-            check_datatype(sumstatsObject.data, log=self.log, verbose=verbose)
-            check_dataframe_shape(sumstats=sumstatsObject.data, 
-                            log=self.log, 
-                            verbose=verbose)
+
+            if engine=="polars":
+                check_datatype_polars(sumstatsObject.data, log=self.log, verbose=verbose)
+                check_dataframe_shape_polars(sumstats=sumstatsObject.data, 
+                                log=self.log, 
+                                verbose=verbose)
+            else:
+                check_datatype(sumstatsObject.data, log=self.log, verbose=verbose)
+                check_dataframe_shape(sumstats=sumstatsObject.data, 
+                                log=self.log, 
+                                verbose=verbose)
             
             if sumstatsObject.meta["gwaslab"]["study_name"] in self.names:
                 new_study_name = "{}_{}".format(sumstatsObject.meta["gwaslab"]["study_name"],i+1)
@@ -249,7 +271,7 @@ class SumstatsMulti( ):
         self.hyprcoloc = hyprcoloc_res_combined
 
     def run_mtag(self,**kwargs):
-        _run_mtag(     self.data,
+        _run_mtag(     self,
                        nstudy = self.meta["gwaslab"]["number_of_studies"], 
                        study= self.meta["gwaslab"]["group_name"], 
                        traits=self.names, 
@@ -288,3 +310,10 @@ class SumstatsMulti( ):
         
         return output
     
+    def offload(self):
+        _offload(self.data, self.tmp_path, self.log)
+        del self.data
+        gc.collect()
+
+    def reload(self):
+         self.data = _reload(self.tmp_path, self.log)

gwaslab/g_SumstatsPair.py

@@ -2,34 +2,42 @@ import pandas as pd
 import numpy as np
 import copy
 import gc
-from gwaslab.util_in_filter_value import filtervalues
-from gwaslab.g_Log import Log
 from math import floor
-from gwaslab.g_Sumstats import Sumstats
-from gwaslab.hm_casting import _merge_mold_with_sumstats_by_chrpos
-from gwaslab.hm_casting import _align_with_mold
-from gwaslab.hm_casting import _fill_missing_columns
-from gwaslab.hm_casting import _check_daf
-from gwaslab.hm_casting import _assign_warning_code
-from gwaslab.qc_fix_sumstats import flipallelestats
-from gwaslab.qc_check_datatype import check_datatype
-from gwaslab.qc_check_datatype import check_dataframe_shape
-from gwaslab.hm_casting import _renaming_cols
-from gwaslab.hm_casting import _sort_pair_cols
-from gwaslab.util_ex_calculate_ldmatrix import tofinemapping
-from gwaslab.util_ex_run_coloc import _run_coloc_susie
-from gwaslab.viz_plot_miamiplot2 import plot_miami2
-from gwaslab.viz_plot_compare_af import  plotdaf
-from gwaslab.util_ex_run_2samplemr import _run_two_sample_mr
-from gwaslab.util_ex_run_clumping import _clump
-from gwaslab.util_ex_ldproxyfinder import _extract_with_ld_proxy
-from gwaslab.g_headers import _get_headers
-from gwaslab.util_ex_match_ldmatrix import tofinemapping_m
-from gwaslab.util_ex_run_mesusie import _run_mesusie
-from gwaslab.io_read_pipcs import _read_pipcs
+
 from gwaslab.g_meta import _init_meta
-from gwaslab.viz_plot_stackedregional import plot_stacked_mqq
-from gwaslab.util_ex_run_ccgwas import _run_ccgwas
+from gwaslab.g_headers import _get_headers
+from gwaslab.g_Log import Log
+from gwaslab.g_Sumstats import Sumstats
+
+from gwaslab.hm.hm_casting import _merge_mold_with_sumstats_by_chrpos
+from gwaslab.hm.hm_casting import _align_with_mold
+from gwaslab.hm.hm_casting import _fill_missing_columns
+from gwaslab.hm.hm_casting import _check_daf
+from gwaslab.hm.hm_casting import _assign_warning_code
+from gwaslab.hm.hm_casting import _renaming_cols
+from gwaslab.hm.hm_casting import _sort_pair_cols
+
+from gwaslab.qc.qc_fix_sumstats import flipallelestats
+from gwaslab.qc.qc_check_datatype import check_datatype
+from gwaslab.qc.qc_check_datatype import check_dataframe_shape
+
+from gwaslab.util.util_ex_calculate_ldmatrix import tofinemapping
+from gwaslab.util.util_ex_run_coloc import _run_coloc_susie
+from gwaslab.util.util_ex_run_ccgwas import _run_ccgwas
+from gwaslab.util.util_ex_run_2samplemr import _run_two_sample_mr
+from gwaslab.util.util_ex_run_clumping import _clump
+from gwaslab.util.util_ex_ldproxyfinder import _extract_with_ld_proxy
+from gwaslab.util.util_ex_match_ldmatrix import tofinemapping_m
+from gwaslab.util.util_ex_run_mesusie import _run_mesusie
+from gwaslab.util.util_in_filter_value import filtervalues
+
+from gwaslab.viz.viz_plot_stackedregional import plot_stacked_mqq
+from gwaslab.viz.viz_plot_miamiplot2 import plot_miami2
+from gwaslab.viz.viz_plot_compare_af import  plotdaf
+
+from gwaslab.io.io_to_pickle import _offload
+from gwaslab.io.io_to_pickle import _reload
+from gwaslab.io.io_read_pipcs import _read_pipcs
 
 class SumstatsPair( ):
     def __init__(self, sumstatsObject1, sumstatsObject2, study=None, suffixes = ("_1","_2") ,verbose=True ):
@@ -40,6 +48,8 @@ class SumstatsPair( ):
             raise ValueError("Please provide GWASLab Sumstats Object #2.")
         
         self.meta = _init_meta(object="SumstatsPair") 
+        self.id = id(self)
+        self.tmp_path = f"./{self.id}"
 
         if sumstatsObject1.meta["gwaslab"]["study_name"]!=sumstatsObject2.meta["gwaslab"]["study_name"]:
             self.study_name = "{}_{}".format(sumstatsObject1.meta["gwaslab"]["study_name"], sumstatsObject2.meta["gwaslab"]["study_name"])
@@ -186,18 +196,18 @@ class SumstatsPair( ):
 
 
     def clump(self,**kwargs):
-        self.clumps["clumps"],self.clumps["clumps_raw"],self.clumps["plink_log"] = _clump(self.data, log=self.log, p="P_1",mlog10p="MLOG10P_1", study = self.meta["gwaslab"]["group_name"], **kwargs)
+        self.clumps["clumps"],self.clumps["clumps_raw"],self.clumps["plink_log"] = _clump(self, log=self.log, p="P_1",mlog10p="MLOG10P_1", study = self.meta["gwaslab"]["group_name"], **kwargs)
 
     def to_coloc(self,**kwargs):
-        self.coloc["path"],self.coloc["file"],self.coloc["plink_log"] = tofinemapping(self.data,study=self.meta["gwaslab"]["group_name"],suffixes=self.suffixes,log=self.log,**kwargs)
+        self.coloc["path"],self.coloc["file"],self.coloc["plink_log"] = tofinemapping(self,study=self.meta["gwaslab"]["group_name"],suffixes=self.suffixes,log=self.log,**kwargs)
 
     def to_mesusie(self,**kwargs):
         self.mesusie["path"],self.mesusie["file"],self.mesusie["plink_log"] = tofinemapping_m(self.data,
-                                                                                                             studies = self.study_names,
-                                                                                                             group = self.meta["gwaslab"]["group_name"],
-                                                                                                             suffixes=self.suffixes,
-                                                                                                             log=self.log,
-                                                                                                             **kwargs)
+                                                                                              studies = self.study_names,
+                                                                                              group = self.meta["gwaslab"]["group_name"],
+                                                                                              suffixes=self.suffixes,
+                                                                                              log=self.log,
+                                                                                              **kwargs)
         
     def run_mesusie(self,**kwargs):
         prefix = _run_mesusie(self.mesusie["path"],log=self.log,ncols=self.ns,**kwargs)
@@ -224,11 +234,11 @@ class SumstatsPair( ):
                                    **kwargs)
          
     def run_coloc_susie(self,**kwargs):
-        self.coloc_susie_res = _run_coloc_susie(self.coloc["path"],log=self.log,ncols=self.ns,**kwargs)
+        self.coloc_susie_res = _run_coloc_susie(self, self.coloc["path"],log=self.log,ncols=self.ns,**kwargs)
 
     def run_two_sample_mr(self, clump=False, **kwargs):
-        exposure1 = self.meta["gwaslab"]["group_name"].split("_")[0]
-        outcome2 = self.meta["gwaslab"]["group_name"].split("_")[1]
+        exposure1 = self.study_names[0]
+        outcome2 = self.study_names[1]
         _run_two_sample_mr(self,exposure1=exposure1,outcome2=outcome2, clump=clump,**kwargs)
 
     def extract_with_ld_proxy(self,**arg):
@@ -266,4 +276,11 @@ class SumstatsPair( ):
                      xlabel="Effect Allele Frequency in Sumstats 1",
                      ylabel="Effect Allele Frequency in Sumstats 2",
                      **kwargs)
-                     
+
+    def offload(self):
+        _offload(self.data, self.tmp_path, self.log)
+        del self.data
+        gc.collect()
+
+    def reload(self):
+         self.data = _reload(self.tmp_path, self.log)