gwaslab 3.6.5__py3-none-any.whl → 3.6.7__py3-none-any.whl

gwaslab/{qc_fix_sumstats.py → qc/qc_fix_sumstats.py}

@@ -7,22 +7,29 @@ from multiprocessing import  Pool
 from liftover import get_lifter
 from liftover import ChainFile
 from functools import partial
+
 from gwaslab.g_vchange_status import vchange_status
 from gwaslab.g_vchange_status import status_match
 from gwaslab.g_vchange_status import change_status
 from gwaslab.g_Log import Log
-from gwaslab.bd_common_data import get_chr_to_number
-from gwaslab.bd_common_data import get_number_to_chr
-from gwaslab.bd_common_data import get_chr_list
-from gwaslab.qc_check_datatype import check_datatype
-from gwaslab.qc_check_datatype import check_dataframe_shape
-from gwaslab.qc_build import _process_build
-from gwaslab.qc_build import _set_build
 from gwaslab.g_version import _get_version
-from gwaslab.util_in_fill_data import _convert_betase_to_mlog10p
-from gwaslab.util_in_fill_data import _convert_betase_to_p
-from gwaslab.util_in_fill_data import _convert_mlog10p_to_p
-from gwaslab.bd_common_data import get_chain
+from gwaslab.g_vchange_status import STATUS_CATEGORIES
+
+from gwaslab.bd.bd_common_data import get_chr_to_number
+from gwaslab.bd.bd_common_data import get_number_to_chr
+from gwaslab.bd.bd_common_data import get_chr_list
+from gwaslab.bd.bd_common_data import get_chain
+from gwaslab.bd.bd_common_data import NA_STRINGS
+
+from gwaslab.qc.qc_check_datatype import check_datatype
+from gwaslab.qc.qc_check_datatype import check_dataframe_shape
+from gwaslab.qc.qc_build import _process_build
+from gwaslab.qc.qc_build import _set_build
+
+from gwaslab.util.util_in_fill_data import _convert_betase_to_mlog10p
+from gwaslab.util.util_in_fill_data import _convert_betase_to_p
+from gwaslab.util.util_in_fill_data import _convert_mlog10p_to_p
+
 #process build
 #setbuild
 #fixID
@@ -69,7 +76,7 @@ from gwaslab.bd_common_data import get_chain
 
 def fixID(sumstats,
        snpid="SNPID",rsid="rsID",chrom="CHR",pos="POS",nea="NEA",ea="EA",status="STATUS",fixprefix=False,
-       fixchrpos=False,fixid=False,fixeanea=False,fixeanea_flip=False,fixsep=False,
+       fixchrpos=False,fixid=False,fixeanea=False,fixeanea_flip=False,fixsep=False, reversea=False,
        overwrite=False,verbose=True,forcefixid=False,log=Log()):  
     '''
     1. fx SNPid
@@ -120,7 +127,21 @@ def fixID(sumstats,
         except:
             log.write(" -Force converting SNPID to pd.string data type...",verbose=verbose)
             sumstats[snpid] = sumstats[snpid].astype("string")
+    ############################  checking string NA ###################################################
+    log.write(" -Checking NA strings :{}".format(",".join(NA_STRINGS)),verbose=verbose)  
+    if snpid in sumstats.columns:  
+        log.write(" -Checking if SNPID contains NA strings...",verbose=verbose)
+        is_snpid_string_na = sumstats[snpid].isin(NA_STRINGS)
+        if sum(is_snpid_string_na) >0:
+            log.write("  -Converting {} NA strings in SNPID to pd.NA...".format(sum(is_snpid_string_na)),verbose=verbose)
+            sumstats.loc[is_snpid_string_na ,snpid] = pd.NA
 
+    if rsid in sumstats.columns: 
+        log.write(" -Checking if rsID contains NA strings...",verbose=verbose)
+        is_rsid_string_na = sumstats[rsid].isin(NA_STRINGS)
+        if sum(is_rsid_string_na) >0:
+            log.write("  -Converting {} NA strings in rsID to pd.NA...".format(sum(is_rsid_string_na)),verbose=verbose)
+            sumstats.loc[is_rsid_string_na ,rsid] = pd.NA
     ############################  checking ###################################################  
     if snpid in sumstats.columns:  
         log.write(" -Checking if SNPID is CHR:POS:NEA:EA...(separator: - ,: , _)",verbose=verbose)
@@ -148,7 +169,15 @@ def fixID(sumstats,
         log.write(" -A look at the unrecognized rsID :",set(sumstats.loc[(~is_rsid)&(~is_rs_chrpos),rsid].head()),"...", verbose=verbose) 
       
     ############################  fixing chr pos###################################################  
-    
+    if reversea == True:
+        if snpid in sumstats.columns: 
+            log.write(" -Reversing Alleles in SNPID...", verbose=verbose)
+            to_fix = is_chrposrefalt
+            to_fix_num = sum(to_fix)
+            if to_fix_num>0 and verbose: log.write(" -Number of variants could be reversed: "+str(to_fix_num)+" ...")
+            extracted = sumstats.loc[to_fix, snpid].str.extract(r'(^\w+[:_-]\d+[:_-])([ATCG]+)([:_-])([ATCG]+$)', flags=re.IGNORECASE)
+            sumstats.loc[to_fix, snpid] = extracted[0] + extracted[3] + extracted[2] + extracted[1]
+            
     if fixchrpos == True:
     # from snpid or rsid, extract CHR:POS to fix CHR and POS    
         if snpid in sumstats.columns: 
@@ -537,24 +566,24 @@ def removedup(sumstats,mode="dm",chrom="CHR",pos="POS",snpid="SNPID",ea="EA",nea
         pre_number =len(sumstats) 
         specified_columns = []
         if "d" in mode:
-            specified_columns.append(rsid)
-            specified_columns.append(snpid)
-            specified_columns.append(chrom)
-            specified_columns.append(pos)
-            specified_columns.append(ea)
-            specified_columns.append(nea)
+            if rsid in sumstats.columns: specified_columns.append(rsid)
+            if snpid in sumstats.columns: specified_columns.append(snpid)
+            if chrom in sumstats.columns: specified_columns.append(chrom)
+            if pos in sumstats.columns: specified_columns.append(pos)
+            if ea in sumstats.columns: specified_columns.append(ea)
+            if nea in sumstats.columns: specified_columns.append(nea)
         if "r" in mode:
-            specified_columns.append(rsid)
+            if rsid in sumstats.columns:specified_columns.append(rsid)
         if "s" in mode:
-            specified_columns.append(snpid)
+            if snpid in sumstats.columns:specified_columns.append(snpid)
         if "m" in mode:
-            specified_columns.append(chrom)
-            specified_columns.append(pos)
+            if chrom in sumstats.columns:specified_columns.append(chrom)
+            if pos in sumstats.columns:specified_columns.append(pos)
         if "c" in mode:
-            specified_columns.append(chrom)
-            specified_columns.append(pos)
-            specified_columns.append(ea)
-            specified_columns.append(nea)
+            if chrom in sumstats.columns:specified_columns.append(chrom)
+            if pos in sumstats.columns:specified_columns.append(pos)
+            if ea in sumstats.columns:specified_columns.append(ea)
+            if nea in sumstats.columns:specified_columns.append(nea)
         sumstats = sumstats.loc[~sumstats[specified_columns].isna().any(axis=1),:]
         after_number=len(sumstats) 
         log.write(" -Removed ",pre_number -after_number," variants with NA values in {} .".format(set(specified_columns)), verbose=verbose)  
@@ -1123,19 +1152,17 @@ def check_range(sumstats, var_range, header, coltocheck, cols_to_check, log, ver
         cols_to_check.append(header)
         if header=="STATUS": 
             log.write(" -Checking STATUS and converting STATUS to categories....", verbose=verbose) 
-            categories = {str(j+i) for j in [1300000,1800000,1900000,3800000,9700000,9800000,9900000] for i in range(0,100000)}
-            sumstats[header] = pd.Categorical(sumstats[header],categories=categories)
+            sumstats[header] = pd.Categorical(sumstats[header],categories=STATUS_CATEGORIES)
             return sumstats
         
         if dtype in ["Int64","Int32","int","int32","in64"]:
             log.write(" -Checking if {} <= {} <= {} ...".format( var_range[0] ,header, var_range[1]), verbose=verbose) 
             sumstats[header] = np.floor(pd.to_numeric(sumstats[header], errors='coerce')).astype(dtype)
-        
+            is_valid = (sumstats[header]>=var_range[0]) & (sumstats[header]<=var_range[1])
         elif dtype in ["Float64","Float32","float","float64","float32"]:
             log.write(" -Checking if {} < {} < {} ...".format( var_range[0] ,header, var_range[1]),verbose=verbose) 
             sumstats[header] = pd.to_numeric(sumstats[header], errors='coerce').astype(dtype)
-        
-        is_valid = (sumstats[header]>=var_range[0]) & (sumstats[header]<=var_range[1])
+            is_valid = (sumstats[header]>var_range[0]) & (sumstats[header]<var_range[1])
         is_valid = is_valid.fillna(False)
 
         if header=="P":
@@ -1835,4 +1862,4 @@ def check_col(df_col_names, verbose=True, log=Log(), cols=None, function=None):
 ###############################################################################################################
 def _df_split(dataframe, n):
     k, m = divmod(len(dataframe), n)
-    return [dataframe.iloc[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n)]
+    return [dataframe.iloc[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n)]

gwaslab/{qc_fix_sumstats_polars.py → qc/qc_fix_sumstats_polars.py}

@@ -11,18 +11,22 @@ from gwaslab.g_vchange_status_polars import vchange_statusp
 from gwaslab.g_vchange_status import status_match
 from gwaslab.g_vchange_status import change_status
 from gwaslab.g_Log import Log
-from gwaslab.bd_common_data import get_chr_to_number
-from gwaslab.bd_common_data import get_number_to_chr
-from gwaslab.bd_common_data import get_chr_list
-from gwaslab.qc_check_datatype import check_datatype
-from gwaslab.qc_check_datatype import check_dataframe_shape
-from gwaslab.qc_build import _process_build
-from gwaslab.qc_build import _set_build
 from gwaslab.g_version import _get_version
-from gwaslab.util_in_fill_data import _convert_betase_to_mlog10p
-from gwaslab.util_in_fill_data import _convert_betase_to_p
-from gwaslab.util_in_fill_data import _convert_mlog10p_to_p
-from gwaslab.bd_common_data import get_chain
+
+from gwaslab.bd.bd_common_data import get_chr_to_number
+from gwaslab.bd.bd_common_data import get_number_to_chr
+from gwaslab.bd.bd_common_data import get_chr_list
+from gwaslab.bd.bd_common_data import get_chain
+
+from gwaslab.qc.qc_check_datatype import check_datatype
+from gwaslab.qc.qc_check_datatype import check_dataframe_shape
+from gwaslab.qc.qc_build import _process_build
+from gwaslab.qc.qc_build import _set_build
+
+from gwaslab.util.util_in_fill_data import _convert_betase_to_mlog10p
+from gwaslab.util.util_in_fill_data import _convert_betase_to_p
+from gwaslab.util.util_in_fill_data import _convert_mlog10p_to_p
+
 import polars as pl
 ###############################################################################################################
 # 20220426

gwaslab/{util_abf_finemapping.py → util/util_abf_finemapping.py}

@@ -1,8 +1,8 @@
 import pandas as pd
 import numpy as np
 from gwaslab.g_Log import Log
-from gwaslab.util_in_filter_value import _get_flanking_by_chrpos
-from gwaslab.util_in_filter_value import _get_flanking_by_id
+from gwaslab.util.util_in_filter_value import _get_flanking_by_chrpos
+from gwaslab.util.util_in_filter_value import _get_flanking_by_id
 
 # Calculate PIP based on approximate Bayesian factor (ABF)
 # Wakefield, J. A bayesian measure of the probability of false discovery in genetic epidemiology studies. Am J Hum Genet 81, 208–227 (2007).

gwaslab/{util_ex_calculate_ldmatrix.py → util/util_ex_calculate_ldmatrix.py}

@@ -4,14 +4,14 @@ import gc
 import pandas as pd
 import numpy as np
 from gwaslab.g_Log import Log
-from gwaslab.qc_fix_sumstats import start_to
-from gwaslab.qc_fix_sumstats import finished
-from gwaslab.util_in_get_sig import getsig
-from gwaslab.util_ex_process_ref import _process_plink_input_files
+from gwaslab.qc.qc_fix_sumstats import start_to
+from gwaslab.qc.qc_fix_sumstats import finished
+from gwaslab.util.util_in_get_sig import getsig
+from gwaslab.util.util_ex_process_ref import _process_plink_input_files
+from gwaslab.util.util_in_filter_value import _exclude_hla
 from gwaslab.g_version import _checking_plink_version
-from gwaslab.util_in_filter_value import _exclude_hla
 
-def tofinemapping(sumstats, 
+def tofinemapping(gls, 
                   study=None, 
                   bfile=None, 
                   vcf=None, 
@@ -39,6 +39,9 @@ def tofinemapping(sumstats,
     _start_cols =["SNPID","CHR","POS","EA","NEA"]
     _start_function = ".calculate_ld_matrix()"
     _must_args ={}
+    
+    sumstats = gls.data
+    gls.offload()
 
     is_enough_info = start_to(sumstats=sumstats,
                             log=log,
@@ -114,7 +117,8 @@ def tofinemapping(sumstats,
                                                     locus_sumstats=locus_sumstats, 
                                                     ref_bim=ref_bim[0],
                                                     log=log,suffixes=suffixes)
-        
+        del locus_sumstats
+        gc.collect()
         #########################################################################################################
         # create matched snp list
         matched_snp_list_path,matched_sumstats_path=_export_snplist_and_locus_sumstats(matched_sumstats=matched_sumstats, 
@@ -144,7 +148,8 @@ def tofinemapping(sumstats,
                                                             extra_plink_option=extra_plink_option,
                                                             ref_allele_path = matched_sumstats_path,
                                                             verbose=verbose)
-    
+        del matched_sumstats
+        gc.collect()
     
         # print file list
         row_dict={}
@@ -166,7 +171,12 @@ def tofinemapping(sumstats,
         output_file_list_path=None
         log.write(" -No avaialable lead variants.",verbose=verbose)
         log.write(" -Stopped LD matrix calculation.",verbose=verbose)
+    
+    del sumstats
+
     finished(log=log, verbose=verbose, end_line=_end_line)
+    gls.reload()
+
     return output_file_list_path, output_file_list, plink_log
 
 

gwaslab/{util_ex_calculate_prs.py → util/util_ex_calculate_prs.py}

@@ -4,8 +4,8 @@ import gc
 import pandas as pd
 import numpy as np
 from gwaslab.g_Log import Log
-from gwaslab.util_in_get_sig import getsig
-from gwaslab.util_ex_process_ref import _process_plink_input_files
+from gwaslab.util.util_in_get_sig import getsig
+from gwaslab.util.util_ex_process_ref import _process_plink_input_files
 from gwaslab.g_version import _checking_plink_version
 
 def _calculate_prs(sumstats, 

gwaslab/{util_ex_ldproxyfinder.py → util/util_ex_ldproxyfinder.py}

@@ -17,12 +17,12 @@ from mpl_toolkits.axes_grid1.inset_locator import mark_inset
 from adjustText import adjust_text
 from gtfparse import read_gtf
 from gwaslab.g_Log import Log
-from gwaslab.bd_common_data import get_chr_to_number
-from gwaslab.bd_common_data import get_number_to_chr
-from gwaslab.bd_common_data import get_recombination_rate
-from gwaslab.bd_common_data import get_gtf
-from gwaslab.util_in_filter_value import _get_flanking
-from gwaslab.hm_harmonize_sumstats import auto_check_vcf_chr_dict
+from gwaslab.bd.bd_common_data import get_chr_to_number
+from gwaslab.bd.bd_common_data import get_number_to_chr
+from gwaslab.bd.bd_common_data import get_recombination_rate
+from gwaslab.bd.bd_common_data import get_gtf
+from gwaslab.util.util_in_filter_value import _get_flanking
+from gwaslab.hm.hm_harmonize_sumstats import auto_check_vcf_chr_dict
 # unmatched SNP list 1 
 
 # for each SNP in unmatched SNP list 1:

gwaslab/{util_ex_ldsc.py → util/util_ex_ldsc.py}

@@ -1,17 +1,22 @@
-from gwaslab.ldsc_sumstats import estimate_h2
-from gwaslab.ldsc_sumstats import estimate_rg
-from gwaslab.ldsc_sumstats import cell_type_specific
-from gwaslab.g_Log import Log
-from gwaslab.qc_fix_sumstats import start_to
-from gwaslab.qc_fix_sumstats import finished
-from gwaslab.qc_fix_sumstats import skipped
-from gwaslab.io_read_ldsc import parse_ldsc_summary
-from gwaslab.io_read_ldsc import parse_partitioned_ldsc_summary
-from gwaslab.util_in_filter_value import filtervalues
-from gwaslab.util_in_filter_value import _filter_palindromic
-from gwaslab.util_in_filter_value import _exclude_hla
-from gwaslab.util_in_filter_value import _exclude_sexchr
 import copy
+from gwaslab.g_Log import Log
+
+from gwaslab.extension.ldsc.ldsc_sumstats import estimate_h2
+from gwaslab.extension.ldsc.ldsc_sumstats import estimate_rg
+from gwaslab.extension.ldsc.ldsc_sumstats import cell_type_specific
+
+from gwaslab.qc.qc_fix_sumstats import start_to
+from gwaslab.qc.qc_fix_sumstats import finished
+from gwaslab.qc.qc_fix_sumstats import skipped
+
+from gwaslab.io.io_read_ldsc import parse_ldsc_summary
+from gwaslab.io.io_read_ldsc import parse_partitioned_ldsc_summary
+
+from gwaslab.util.util_in_filter_value import filtervalues
+from gwaslab.util.util_in_filter_value import _filter_palindromic
+from gwaslab.util.util_in_filter_value import _exclude_hla
+from gwaslab.util.util_in_filter_value import _exclude_sexchr
+
 
 class ARGS():
     def __init__(self, kwargs=None):

gwaslab/{util_ex_match_ldmatrix.py → util/util_ex_match_ldmatrix.py}

@@ -1,7 +1,7 @@
 import scipy.sparse as sparse
 import numpy as np
 import pandas as pd
-from gwaslab.hm_casting import _merge_mold_with_sumstats_by_chrpos
+
 import subprocess
 import os
 import re
@@ -9,16 +9,17 @@ import gc
 import pandas as pd
 import numpy as np
 from gwaslab.g_Log import Log
-from gwaslab.qc_fix_sumstats import start_to
-from gwaslab.qc_fix_sumstats import finished
-from gwaslab.util_in_get_sig import getsig
-from gwaslab.util_ex_process_ref import _process_plink_input_files
 from gwaslab.g_version import _checking_plink_version
-from gwaslab.util_in_filter_value import _exclude_hla
-from gwaslab.util_ex_calculate_ldmatrix import _extract_variants_in_locus
 
+from gwaslab.hm.hm_casting import _merge_mold_with_sumstats_by_chrpos
 
+from gwaslab.qc.qc_fix_sumstats import start_to
+from gwaslab.qc.qc_fix_sumstats import finished
 
+from gwaslab.util.util_in_get_sig import getsig
+from gwaslab.util.util_ex_process_ref import _process_plink_input_files
+from gwaslab.util.util_in_filter_value import _exclude_hla
+from gwaslab.util.util_ex_calculate_ldmatrix import _extract_variants_in_locus
 
 def tofinemapping_m(sumstats, 
                     studies=None, 

gwaslab/util/util_ex_phewwas.py

@@ -0,0 +1,117 @@
+import pandas as pd
+from gwaslab.g_Log import Log
+import pandas as pd
+import numpy as np
+
+def _extract_associations(sumstats, rsid="rsID", log = Log(), verbose=True):
+
+    assoc, traits, studies, variants = get_associations_from_gwascatalog(sumstats, rsid=rsid, log=log, verbose=verbose)
+    
+    assoc = _fix_beta(assoc)
+
+    traits_agg = traits.groupby("associationId")[["trait","shortForm"]].agg(lambda x: ",".join(x)).reset_index()
+
+    assoc_traits_agg= pd.merge(assoc, traits_agg, on ="associationId",how="left")
+    
+    assoc_traits_agg= pd.merge(assoc_traits_agg, studies, on ="associationId", how="left")
+
+    assoc_traits_agg= pd.merge(assoc_traits_agg, variants, on ="associationId",how="left")
+    
+    assoc_traits_agg = assoc_traits_agg.rename(columns={"trait":"GWASCATALOG_TRAIT",
+                                                        "riskFrequency":"RAF",
+                                                        "betaNum":"Beta",
+                                                        "pvalue":"P-value"
+                                                        })
+    
+    summary_columns=['GWASCATALOG_TRAIT','associationId', 'rsID', "geneName", 
+                     'RA', 'RAF','Beta', 'P-value','cohort','initialSampleSize','publicationInfo.pubmedId',
+                     "functionalClass","gene.geneName"]
+    
+    assoc_traits_agg_summary = assoc_traits_agg[summary_columns]
+
+    return assoc_traits_agg, assoc_traits_agg_summary
+
+def get_associations_from_gwascatalog(sumstats, rsid="rsID", log=Log(), verbose=True):
+    from pandasgwas import get_associations
+    from pandasgwas import get_traits
+    from pandasgwas import get_studies
+    from pandasgwas import get_variants
+
+    association = pd.DataFrame()
+    strongest_risk_alleles=pd.DataFrame()
+    author_reported_genes = pd.DataFrame()
+    unique_sumstats = sumstats.dropna(subset=[rsid]).drop_duplicates(subset=[rsid])
+    
+    for index,row in unique_sumstats.iterrows():
+        log.write(f"Getting associations from GWAS Catalog for {row[rsid]}...",verbose=verbose)
+
+        df = get_associations(variant_id = row[rsid])
+        
+        empty=[]
+        if len(df.associations)>0:
+            df.associations[rsid] = row[rsid]
+            association = pd.concat([association, df.associations],ignore_index=True)
+        
+            df.strongest_risk_alleles[rsid] = row[rsid]
+            strongest_risk_alleles = pd.concat([strongest_risk_alleles, df.strongest_risk_alleles],ignore_index=True)
+            
+            try:
+                author_reported_genes = pd.concat([author_reported_genes, df.author_reported_genes],ignore_index=True)
+            except:
+                pass
+            log.write("", show_time=False, verbose=verbose)
+        else:
+            empty.append(row[rsid])
+
+    log.write(f"No associations: {empty}", verbose=verbose)
+
+    if len(strongest_risk_alleles)>0:
+        strongest_risk_alleles["RA"] = strongest_risk_alleles["riskAlleleName"].str.split("-").str[-1]
+    
+    if len(association)>0:
+        association = pd.merge(association, strongest_risk_alleles[["associationId","RA"]],on="associationId",how="left")
+
+        author_reported_genes = author_reported_genes.groupby("associationId")["geneName"].agg(lambda x: ",".join(x))
+        association = pd.merge(association, author_reported_genes,on="associationId",how="left")
+
+    log.write(f"Retrieved {len(association)} associations from GWAS Catalog...", verbose=verbose)
+
+    traits = pd.DataFrame()
+    studies = pd.DataFrame()
+    variants = pd.DataFrame()
+
+    for index,row in association.drop_duplicates(subset=["associationId"]).iterrows():
+        log.write(f'Getting traits/studies/variants from GWAS Catalog for associationId: {row["associationId"]}...',verbose=verbose)
+
+        df = get_traits(association_id = row["associationId"])
+        df.efo_traits["associationId"] = row["associationId"]
+        traits = pd.concat([traits, df.efo_traits],ignore_index=True)
+        
+        df = get_studies(association_id = row["associationId"])
+        df.studies["associationId"] = row["associationId"]
+        studies = pd.concat([studies, df.studies],ignore_index=True)
+
+        df = get_variants(association_id = row["associationId"])
+        df.variants["associationId"] = row["associationId"]
+        min_distance = df.genomic_contexts["distance"].min()
+        df.genomic_contexts = df.genomic_contexts.loc[df.genomic_contexts["distance"]==min_distance,:].drop_duplicates("gene.geneName").groupby("rsId")["gene.geneName"].agg(lambda x: ",".join(x))
+        df.variants = pd.merge(df.variants[["rsId","functionalClass","associationId"]],df.genomic_contexts, on="rsId")
+        variants = pd.concat([variants, df.variants[["associationId","functionalClass","gene.geneName"]]],ignore_index=True)
+
+    return association, traits, studies, variants
+
+def _fix_beta(association):
+
+    is_or_available = (association["betaNum"].isna()) & (~association["orPerCopyNum"].isna())
+    is_range_available = (association["betaNum"].isna()) & (association["orPerCopyNum"].isna()) & (~association["range"].isna())
+
+    association.loc[is_or_available ,"betaNum"] = np.log(association.loc[is_or_available,"orPerCopyNum"]) 
+    association.loc[is_range_available ,"betaNum"] = association.loc[is_range_available,"range"].apply(lambda x: parse_range(x))
+    return association
+
+def parse_range(x):
+    range_list = x.strip("[|]").split("-")
+    high = np.log(range_list[1])
+    low = np.log(range_list[0])
+    beta = (high + low)/2
+    return beta

gwaslab/{util_ex_process_h5.py → util/util_ex_process_h5.py}

@@ -2,8 +2,8 @@ import pandas as pd
 import os
 import numpy as np
 from gwaslab.g_Log import Log
-from gwaslab.qc_fix_sumstats import start_to
-from gwaslab.qc_fix_sumstats import finished
+from gwaslab.qc.qc_fix_sumstats import start_to
+from gwaslab.qc.qc_fix_sumstats import finished
 
 def process_vcf_to_hfd5(vcf, 
                     directory=None, 

gwaslab/{util_ex_process_ref.py → util/util_ex_process_ref.py}

@@ -4,8 +4,8 @@ import subprocess
 from gwaslab.g_Log import Log
 import os
 from gwaslab.g_version import _checking_plink_version
-from gwaslab.qc_fix_sumstats import start_to
-from gwaslab.qc_fix_sumstats import finished
+from gwaslab.qc.qc_fix_sumstats import start_to
+from gwaslab.qc.qc_fix_sumstats import finished
 
 def _process_plink_input_files(chrlist, 
                                bfile=None, 

gwaslab/{util_ex_run_2samplemr.py → util/util_ex_run_2samplemr.py}

@@ -7,13 +7,14 @@ import numpy as np
 from gwaslab.g_Log import Log
 from gwaslab.g_version import _checking_r_version
 from gwaslab.g_version import _check_susie_version
-from gwaslab.util_in_convert_h2 import _get_per_snp_r2
-from gwaslab.qc_fix_sumstats import start_to
-from gwaslab.qc_fix_sumstats import finished
+from gwaslab.util.util_in_convert_h2 import _get_per_snp_r2
+from gwaslab.qc.qc_fix_sumstats import start_to
+from gwaslab.qc.qc_fix_sumstats import finished
 
 
 def _run_two_sample_mr(sumstatspair_object,
                        r,
+                       out="./",
                        clump=False,
                        f_check=10,
                        exposure1="Trait1",
@@ -77,9 +78,15 @@ def _run_two_sample_mr(sumstatspair_object,
     # Clumping       
 
     prefix = "{exposure}_{outcome}_{memory_id}".format(exposure = exposure1, outcome= outcome2, memory_id = id(sumstatspair))
-    temp_sumstats_path = "twosample_mr_{exposure}_{outcome}_{memory_id}.csv.gz".format(exposure = exposure1, outcome= outcome2, memory_id = id(sumstatspair))
-    sumstatspair.to_csv(temp_sumstats_path ,index=None)
-    
+    prefix = "{}{}".format(out.rstrip('/') + "/",prefix)
+    temp_sumstats_path = "{out}twosample_mr_{exposure}_{outcome}_{memory_id}.csv.gz".format(out=out.rstrip('/') + "/",
+                                                                                               exposure = exposure1, 
+                                                                                               outcome= outcome2, 
+                                                                                               memory_id = id(sumstatspair))
+    if len(sumstatspair)>0:
+        sumstatspair.to_csv(temp_sumstats_path ,index=None)
+    else:
+        return 0
     ###
     calculate_r_script = ""
     
@@ -173,7 +180,10 @@ def _run_two_sample_mr(sumstatspair_object,
         directionality_test = directionality_test_script
     )
         
-    temp_r_script_path = "_{}_{}_{}_gwaslab_2smr_temp.R".format(exposure1,outcome2,id(sumstatspair))
+    temp_r_script_path = "{}_{}_{}_{}_gwaslab_2smr_temp.R".format(out.rstrip('/') + "/",
+                                                                  exposure1,
+                                                                  outcome2,
+                                                                  id(sumstatspair))
     with open(temp_r_script_path,"w") as file:
             file.write(rscript)
 
@@ -200,6 +210,7 @@ def _run_two_sample_mr(sumstatspair_object,
         log.write(rscript)
         log.write(e.output)
         os.remove(temp_r_script_path)
+    log.write(" Finished running MR using twosampleMR from command line.")
 
 
 

gwaslab/{util_ex_run_ccgwas.py → util/util_ex_run_ccgwas.py}

@@ -6,10 +6,10 @@ import numpy as np
 from gwaslab.g_Log import Log
 from gwaslab.g_version import _checking_r_version
 from gwaslab.g_version import _check_susie_version
-from gwaslab.qc_fix_sumstats import start_to
-from gwaslab.qc_fix_sumstats import finished
-from gwaslab.util_ex_calculate_ldmatrix import _extract_variants_in_locus
-from gwaslab.util_in_get_sig import getsig
+from gwaslab.qc.qc_fix_sumstats import start_to
+from gwaslab.qc.qc_fix_sumstats import finished
+from gwaslab.util.util_ex_calculate_ldmatrix import _extract_variants_in_locus
+from gwaslab.util.util_in_get_sig import getsig
 
 def _run_ccgwas(    sumstats_pair, 
                     r="Rscript",