genal-python 1.3.1__tar.gz → 1.3.3__tar.gz

{genal_python-1.3.1 → genal_python-1.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: genal-python
-Version: 1.3.1
+Version: 1.3.3
 Summary: A python toolkit for polygenic risk scoring and mendelian randomization.
 Author-email: Cyprien Rivier <riviercyprien@gmail.com>
 Requires-Python: >=3.8

{genal_python-1.3.1 → genal_python-1.3.3}/genal/Geno.py

@@ -1,7 +1,7 @@
 import pandas as pd
 import numpy as np
 import warnings
-import os, subprocess
+import os
 import copy
 import psutil
 import uuid
@@ -30,7 +30,8 @@ from .geno_tools import (
     fill_se_p,
     check_allele_column,
     check_snp_column,
-    remove_na
+    remove_na,
+    filter_by_gene_func
 )
 from .association import set_phenotype_func, association_test_func_plink2
 from .extract_prs import extract_snps_func, prs_func
@@ -117,9 +118,22 @@ class Geno:
 
         Attributes:
             name (str): Randomly generated ID for the Geno object.
-            outcome (list): List of outcomes (initialized as empty).
             cpus (int): Number of CPUs to be used.
             ram (int): Amount of RAM to be used in MBs.
+            checks (dict): Dictionary of checks performed on the main DataFrame.
+            reference_panel (pd.DataFrame): Reference population SNP data used for SNP info
+                adjustments. Initialized when first needed.
+            reference_panel_name (str): string to identify the reference_panel (path or population string)
+            phenotype (pd.DataFrame, str): Tuple with a DataFrame of individual-level phenotype
+                data and a string representing the phenotype trait column. Initialized after
+                running the 'set_phenotype' method.
+            MR_data (pd.DataFrame, pd.DataFrame, str): Tuple containing DataFrames for associations
+                with exposure and outcome, and a string for the outcome name. Initialized after
+                running the 'query_outcome' method.
+            MR_results (pd.DataFrame, pd.DataFrame, str, str): Contains an MR results dataframe, a dataframe of harmonized SNPs, an exposure name, an outcome name. 
+                Assigned after calling the MR method and used for plotting with the MR_plot method.
+            MRpresso_subset_data (pd.DataFrame, pd.DataFrame, str, str): Contains a dataframe of subsetted harmonized SNPs without outliers. 
+                Assigned after calling the MRpresso method.
         """
 
         # Validate df type
@@ -399,10 +413,7 @@ class Geno:
 
         # If clumped data is successfully generated, assign it to the object's attribute
         if clumped_data is not None:
-            Clumped = Geno(clumped_data, keep_columns=True)
-            Clumped.checks = self.checks.copy()
-            if hasattr(self, "phenotype"):
-                Clumped.phenotype = self.phenotype
+            Clumped = self.copy(clumped_data)
             return Clumped
         return None
 
@@ -644,7 +655,8 @@ class Geno:
                     kb=kb,
                     r2=r2,
                     window_snps=window_snps,
-                    threads=self.cpus
+                    threads=self.cpus,
+                    name=self.name
                 )
                 # Apply proxies if found
                 if isinstance(ld, pd.DataFrame) and not ld.empty:
@@ -700,6 +712,8 @@ class Geno:
         # Assign the processed data and inferred phenotype type to the .phenotype attribute
         self.phenotype = (processed_data, inferred_pheno_type, PHENO)
 
+        return
+
     def association_test(self, path=None, covar=[], standardize=True):
         """
         Conduct single-SNP association tests against a phenotype.
@@ -1201,23 +1215,77 @@ class Geno:
 
         return mod_table, GlobalTest, OutlierTest, BiasTest
 
+    def filter_by_gene(self, gene_id, id_type="symbol", window_size=1000000, build="37", replace=False):
+        """
+        Filter the data to include only variants that are within a specified distance of a specific gene.
+
+        Args:
+            gene_id (str): Identifier for the gene/protein to filter variants around.
+            id_type (str, optional): Type of identifier provided. Options are:
+                - "symbol": Gene symbol (e.g., "APOE")
+                - "HGNC": HGNC ID (e.g., "HGNC:613")
+                - "name": Full gene name (e.g., "apolipoprotein E")
+                - "Ensembl": Ensembl gene ID (e.g., "ENSG00000130203")
+                - "NCBI": NCBI gene ID (e.g., "348")
+                - "UCSC": UCSC gene ID (e.g., "uc001hbu.2")
+                - "Vega": Vega gene ID (e.g., "OTTHUMG00000019505")
+                Default is "symbol".
+            window_size (int, optional): Size of the window around the gene in base pairs. Default is 1,000,000 (1Mb).
+            build (str, optional): Genome build of the data. Default is "37".
+            replace (bool, optional): If True, replace the existing data attribute with the filtered data. Default is True.
+        Returns:
+            if replace is True:
+                pd.DataFrame: Filtered DataFrame containing only variants within the specified window 
+                    around the gene, with additional column 'Distance'.
+            if replace is False:
+                genal.Geno: A new Geno object with the filtered data.
+
+        Raises:
+            ValueError: If required columns are missing, gene information cannot be found, or invalid id_type is provided.
+            
+        Notes:
+            - Distance is calculated from the nearest gene boundary (start or end position)
+            - Null distances indicate the variant is within the gene
+        """
+        # Check required columns
+        for col in ["CHR", "POS"]:
+            if col not in self.data.columns:
+                raise ValueError(f"Column {col} must be present in the input data!")
+            
+        # Do the appropriate preprocessing on CHR and POS columns if not already done
+        if not self.checks.get("CHR"):
+            check_int_column(self.data, "CHR")
+            self.checks["CHR"] = True
+        if not self.checks.get("POS"):
+            check_int_column(self.data, "POS")
+            self.checks["POS"] = True
+
+        filtered = filter_by_gene_func(self.data, gene_id, id_type, window_size, build)
+        
+        if replace:
+            self.data = filtered
+        else:
+            Geno_filtered = self.copy(filtered)
+            return Geno_filtered
+
     def colocalize(self, outcome, method="abf", trait1_type=None, trait2_type=None,
-                   sdY1=None, sdY2=None, n1=None, n2=None, p1=1e-4, p2=1e-4, p12=1e-5):
+                   sdY1=None, sdY2=None, n1=None, n2=None, p1=1e-4, p2=1e-4, p12=1e-5, merge_on_snp=False):
         """
         Perform colocalization analysis between two GWAS datasets.
         
         Args:
             outcome: Another Geno object containing the outcome dataset
             method: Method to use for colocalization (default: "abf")
-            trait1_type: Type of exposure trait ("quant" or "cc")
-            trait2_type: Type of outcome trait ("quant" or "cc")
-            sdY1: Standard deviation of exposure trait (required for quantitative traits)
-            sdY2: Standard deviation of outcome trait (required for quantitative traits)
-            n1: Sample size for exposure (used to estimate sdY1 if not provided)
-            n2: Sample size for outcome (used to estimate sdY2 if not provided)
+            trait1_type: Type of exposure trait ("quant" for quantitative traits or "cc" for case-control traits)
+            trait2_type: Type of outcome trait ("quant" for quantitative traits or "cc" for case-control traits)
+            sdY1: Standard deviation of exposure trait (required for quantitative traits, but can be estimated from EAF and sample size)
+            sdY2: Standard deviation of outcome trait (required for quantitative traits, but can be estimated from EAF and sample size)
+            n1: Sample size for exposure (used to estimate sdY1 if sdY1 is not provided)
+            n2: Sample size for outcome (used to estimate sdY2 if sdY2 is not provided)
             p1: Prior probability SNP associated with exposure
             p2: Prior probability SNP associated with outcome
             p12: Prior probability SNP associated with both traits
+            merge_on_snp: If True, merge the datasets on SNP column. If False, first attempt to merge on CHR and POS columns.
         """
         # Ensure required columns exist in both datasets
         required_cols = ['BETA', 'SE']
@@ -1237,56 +1305,10 @@ class Geno:
         # Make copies of the data to avoid modifying the original data
         data1 = self.data.copy()
         data2 = outcome.data.copy()
-        
-        # Ensure that the BETA columns are preprocessed
-        check_beta_column(data1, 'BETA', 'Fill')
-        check_beta_column(data2, 'BETA', 'Fill')
-
-        # Adjust EAF column names before merging in case one of the datasets does not have it
-        if 'EAF' in data1.columns:
-            data1.rename(columns={'EAF': 'EAF_1'}, inplace=True)
-        if 'EAF' in data2.columns:
-            data2.rename(columns={'EAF': 'EAF_2'}, inplace=True)
-        
-        # Determine merge strategy based on available columns
-        if all(col in self.data.columns for col in ['CHR', 'POS']) and \
-           all(col in outcome.data.columns for col in ['CHR', 'POS']):
-            print("Merging datasets using CHR and POS")
-
-            #Ensure that the CHR, POS columns are preprocessed
-            check_int_column(data1, "CHR")
-            check_int_column(data1, "POS")
-            check_int_column(data2, "CHR")
-            check_int_column(data2, "POS")
-
-            # Merge the datasets
-            merged_data = pd.merge(data1, data2,
-                                 on=['CHR', 'POS'],
-                                 suffixes=('_1', '_2'))
-            
-        elif 'SNP' in self.data.columns and 'SNP' in outcome.data.columns:
-            print("Merging datasets using SNP IDs")
-
-            # Ensure that the SNP column is preprocessed
-            check_snp_column(data1)
-            check_snp_column(data2)
-
-            # Merge the datasets
-            merged_data = pd.merge(data1, data2,
-                                   on='SNP',
-                                   suffixes=('_1', '_2'))
-        else:
-            raise ValueError("Either CHR/POS or SNP columns must be present in both datasets for merging")
-        
-        # Drop any rows with missing values
-        merged_data = merged_data.dropna()
-        if merged_data.empty:
-            raise ValueError("No overlapping variants found between the datasets")
-        
-        print(f"Using {len(merged_data)} overlapping variants for colocalization analysis")
             
         # Call the implementation function
-        return coloc_abf_func(merged_data, 
+        return coloc_abf_func(data1, 
+                             data2,
                              trait1_type=trait1_type,
                              trait2_type=trait2_type,
                              sdY1=sdY1, 
@@ -1295,7 +1317,8 @@ class Geno:
                              n2=n2,
                              p1=p1,
                              p2=p2,
-                             p12=p12)
+                             p12=p12,
+                             merge_on_snp=merge_on_snp)
 
 
     def lift(
@@ -1480,14 +1503,24 @@ class Geno:
             self.data = self.data.groupby(by=["SNP"]).first().reset_index(drop=False)
         return
 
-    def copy(self):
+    def copy(self, data):
         """
-        Create a deep copy of the Geno instance.
+        Create another Geno instance with the updated data attribute.
+        The relevant attributes are copied as well (checks, phenotype, reference_panel, reference_panel_name).
+        Attributes that are not copied are MR_data, MR_results, MRpresso_subset_data, MRpresso_results.
 
         Returns:
             Geno: A deep copy of the instance.
         """
-        return copy.deepcopy(self)
+        Geno_copy = Geno(data, keep_columns=True)
+        Geno_copy.checks = self.checks.copy()
+        if hasattr(self, "phenotype"):
+            Geno_copy.phenotype = self.phenotype
+            if hasattr(self, "reference_panel"):
+                Geno_copy.reference_panel = self.reference_panel
+        if hasattr(self, "reference_panel_name"):
+            Geno_copy.reference_panel_name = self.reference_panel_name
+        return Geno_copy
 
     def save(self, path="", fmt="h5", sep="\t", header=True):
         """

{genal_python-1.3.1 → genal_python-1.3.3}/genal/__init__.py

@@ -1,10 +1,10 @@
 import os
 import json
 from .tools import default_config, write_config, set_plink, install_plink, delete_tmp, get_reference_panel_path, get_plink_path
-from .geno_tools import Combine_Geno
+from .geno_tools import Combine_Geno, filter_by_gene_func
 from .constants import CONFIG_DIR
 
-__version__ = "1.3.1"
+__version__ = "1.3.3"
 
 config_path = os.path.join(CONFIG_DIR, "config.json")
 

genal_python-1.3.3/genal/colocalization.py

@@ -0,0 +1,249 @@
+import numpy as np
+import pandas as pd
+from numpy import exp, log
+from genal.geno_tools import check_beta_column, check_allele_column, check_snp_column, check_int_column
+
+# Currently does not support multi-allelic SNPs
+
+def coloc_abf_func(data1, data2, trait1_type="quant", trait2_type="quant", 
+                   sdY1=None, sdY2=None, n1=None, n2=None,
+                   p1=1e-4, p2=1e-4, p12=1e-5, merge_on_snp=False):
+    """
+    Perform colocalization analysis between two GWAS datasets using approximate Bayes factors.
+    Corresponds to the :meth:`Geno.colocalize` method.
+    
+    Args:
+        data1: DataFrame containing GWAS results for trait 1
+        data2: DataFrame containing GWAS results for trait 2
+        trait1_type: Type of trait 1 ("quant" for quantitative traits or "cc" for case-control traits)
+        trait2_type: Type of trait 2 ("quant" for quantitative traits or "cc" for case-control traits)
+        sdY1: Standard deviation of trait 1 (required for quantitative traits)
+        sdY2: Standard deviation of trait 2 (required for quantitative traits)
+        n1: Sample size for trait 1 (used to estimate sdY if not provided)
+        n2: Sample size for trait 2 (used to estimate sdY if not provided)
+        p1: Prior probability SNP associated with trait 1
+        p2: Prior probability SNP associated with trait 2
+        p12: Prior probability SNP associated with both traits
+        merge_on_snp: If True, merge the datasets on SNP column. If False, first attempt to merge on CHR and POS columns.
+
+    """
+
+    # Ensure that the BETA columns are preprocessed
+    check_beta_column(data1, 'BETA', 'Fill')
+    check_beta_column(data2, 'BETA', 'Fill')
+
+    # Adjust EAF column names before merging in case one of the datasets does not have it
+    if 'EAF' in data1.columns:
+        data1.rename(columns={'EAF': 'EAF_1'}, inplace=True)
+    if 'EAF' in data2.columns:
+        data2.rename(columns={'EAF': 'EAF_2'}, inplace=True)
+
+    # First determine if we can merge on position, otherwise try SNP
+    if all(col in data1.columns for col in ['CHR', 'POS']) and \
+       all(col in data2.columns for col in ['CHR', 'POS']) and not merge_on_snp:
+        
+        print("Merging datasets using genomic positions (CHR, POS)")
+        
+        # Ensure that the CHR and POS columns are preprocessed
+        check_int_column(data1, "CHR")
+        check_int_column(data2, "CHR")
+        check_int_column(data1, "POS")
+        check_int_column(data2, "POS")
+        
+        # Merge using position
+        merged_data = pd.merge(
+            data1,
+            data2,
+            on=['CHR', 'POS'],
+            how='left',
+            suffixes=('_1', '_2')
+        )
+        
+    elif 'SNP' in data1.columns and 'SNP' in data2.columns:
+        print("Position columns (CHR, POS) not present in both datasets. Merging datasets using SNP IDs.")
+        
+        # Ensure that the SNP column is preprocessed
+        check_snp_column(data1)
+        check_snp_column(data2)
+        
+        # Merge using SNP
+        merged_data = pd.merge(
+            data1,
+            data2,
+            on='SNP',
+            suffixes=('_1', '_2')
+        )
+    
+    else:
+        raise ValueError("At least CHR/POS or SNP columns must be present in both datasets for colocalization analysis")
+
+    # After merging, check if we can align alleles
+    if all(col in merged_data.columns for col in ['EA_1', 'NEA_1', 'EA_2', 'NEA_2']):
+        print("Aligning effect alleles between datasets")
+        
+        # Ensure allele columns are preprocessed
+        check_allele_column(data1, "EA", keep_indel=False)
+        check_allele_column(data1, "NEA", keep_indel=False)
+        check_allele_column(data2, "EA", keep_indel=False)
+        check_allele_column(data2, "NEA", keep_indel=False)
+        
+        # Adjust BETA from trait 2 to correspond to the same effect allele as trait 1
+        conditions = [
+            merged_data["EA_1"] == merged_data["EA_2"],
+            merged_data["EA_1"] == merged_data["NEA_2"],
+            True,
+        ]
+        choices = [
+            merged_data["BETA_2"],
+            -merged_data["BETA_2"],
+            np.nan,
+        ]
+        merged_data["BETA_2"] = np.select(conditions, choices)
+    else:
+        print("Allele columns (EA, NEA) not present in both datasets. "
+              "This might lead to incorrect results if the effect estimates (BETA) were not obtained with the same reference allele in both datasets.")
+
+    # Clean up columns
+    merged_data.drop(columns=["EA_2", "NEA_2", "SNP_2", "CHR_2", "POS_2"], inplace=True, errors='ignore')
+    merged_data.rename(columns={"SNP_1": "SNP", "CHR_1": "CHR", "POS_1": "POS"}, inplace=True, errors='ignore')
+
+    # Drop any rows with duplicate values
+    if "SNP" in merged_data.columns:    
+        merged_data.drop_duplicates(subset=['SNP'], keep='first', inplace=True)
+    if "CHR" in merged_data.columns and "POS" in merged_data.columns:
+        merged_data.drop_duplicates(subset=["CHR", "POS"], keep='first', inplace=True)
+
+    # Drop any rows with missing values
+    merged_data = merged_data.dropna()
+    if merged_data.empty:
+        raise ValueError("No overlapping variants found between the datasets")
+    
+    print(f"Using {len(merged_data)} overlapping variants for colocalization analysis")
+    
+    # Estimate sdY if not provided for quantitative traits
+    if trait1_type == "quant" and sdY1 is None:
+        if 'EAF_1' not in merged_data.columns or n1 is None:
+            print("Neither sdY1 nor EAF and n1 are provided for trait 1. Assuming sdY1 = 1.")
+            sdY1 = 1
+        else:
+            sdY1 = sdY_est(merged_data['SE_1']**2, merged_data['EAF_1'], n1)
+            print(f"Using EAF and n1 to estimate sdY1: {sdY1:.2f}")
+        
+    if trait2_type == "quant" and sdY2 is None:
+        if 'EAF_2' not in merged_data.columns or n2 is None:
+            print("Neither sdY2 nor EAF and n2 are provided for trait 2. Assuming sdY2 = 1.")
+            sdY2 = 1
+        else:
+            sdY2 = sdY_est(merged_data['SE_2']**2, merged_data['EAF_2'], n2)
+            print(f"Using EAF and n2 to estimate sdY2: {sdY2:.2f}")
+    
+    # Calculate Bayes factors for each dataset
+    lABF_1 = approx_bf_estimates(merged_data['BETA_1'], merged_data['SE_1']**2, 
+                                trait_type=trait1_type, sdY=sdY1)
+    lABF_2 = approx_bf_estimates(merged_data['BETA_2'], merged_data['SE_2']**2, 
+                                trait_type=trait2_type, sdY=sdY2)
+    
+    # Adjust priors based on number of SNPs
+    n_snps = len(merged_data)
+    if n_snps * p1 >= 1:
+        p1 = 1 / (n_snps + 1)
+    if n_snps * p2 >= 1:
+        p2 = 1 / (n_snps + 1)
+    if n_snps * p12 >= 1:
+        p12 = 1 / (n_snps + 1)
+    
+    # Calculate posterior probabilities
+    pp = combine_abf(lABF_1, lABF_2, p1, p2, p12)
+    
+    # Add SNP-specific results
+    results_df = merged_data.copy()
+    results_df['lABF_1'] = lABF_1
+    results_df['lABF_2'] = lABF_2
+    results_df['internal.sum.lABF'] = lABF_1 + lABF_2
+    
+    # Calculate SNP-specific PP for H4
+    my_denom_log_abf = logsum(results_df['internal.sum.lABF'])
+    results_df['SNP.PP.H4'] = np.exp(results_df['internal.sum.lABF'] - my_denom_log_abf)
+    
+    return {
+            'nsnps': n_snps,
+            **pp
+        }
+
+def approx_bf_estimates(beta, varbeta, trait_type="quant", sdY=1, effect_priors={'quant': 0.15, 'cc': 0.2}):
+    """
+    Calculate approximate Bayes factors using regression estimates.
+    
+    Args:
+        beta: effect size estimate
+        varbeta: variance of the effect size estimate
+        trait_type: either "quant" for quantitative trait or "cc" for case-control
+        sdY: standard deviation of the trait (for quantitative traits)
+        effect_priors: dictionary with prior effect sizes for quantitative and case-control traits
+        
+    Returns:
+        array: log approximate Bayes factors
+    """
+    z = beta / np.sqrt(varbeta)
+    
+    # Set prior standard deviation based on trait type
+    if trait_type == "quant":
+        sd_prior = effect_priors['quant'] * sdY
+    else:  # case-control
+        sd_prior = effect_priors['cc']
+        
+    r = sd_prior**2 / (sd_prior**2 + varbeta)
+    lABF = 0.5 * (np.log(1 - r) + (r * z**2))
+    return lABF
+
+def logsum(x):
+    """Calculate log of sum of exponentials"""
+    my_max = np.max(x)
+    return my_max + np.log(np.sum(np.exp(x - my_max)))
+
+def logdiff(x, y):
+    """Calculate log of difference of exponentials"""
+    my_max = max(x, y)
+    return my_max + np.log(exp(x - my_max) - np.exp(y - my_max))
+
+def combine_abf(l1, l2, p1, p2, p12):
+    """Calculate posterior probabilities for different hypotheses"""
+    lsum = l1 + l2
+    
+    lH0_abf = 0
+    lH1_abf = np.log(p1) + logsum(l1)
+    lH2_abf = np.log(p2) + logsum(l2)
+    lH3_abf = np.log(p1) + np.log(p2) + logdiff(logsum(l1) + logsum(l2), logsum(lsum))
+    lH4_abf = np.log(p12) + logsum(lsum)
+    
+    all_abf = np.array([lH0_abf, lH1_abf, lH2_abf, lH3_abf, lH4_abf])
+    denom_log_abf = logsum(all_abf)
+    pp_abf = np.exp(all_abf - denom_log_abf)
+    
+    return {
+        'PP.H0.abf': pp_abf[0],
+        'PP.H1.abf': pp_abf[1],
+        'PP.H2.abf': pp_abf[2],
+        'PP.H3.abf': pp_abf[3],
+        'PP.H4.abf': pp_abf[4]
+    }
+
+def sdY_est(vbeta, maf, n):
+    """
+    Estimate trait standard deviation given vectors of variance of coefficients, MAF and sample size.
+    
+    Args:
+        vbeta: vector of variance of coefficients
+        maf: vector of MAF (same length as vbeta)
+        n: sample size
+        
+    Returns:
+        float: estimated standard deviation of Y
+    """
+    oneover = 1/vbeta
+    nvx = 2 * n * maf * (1-maf)
+    # Fit linear regression through origin
+    coef = np.sum(nvx * oneover) / np.sum(oneover**2)
+    if coef < 0:
+        raise ValueError("Estimated sdY is negative - this can happen with small datasets, or those with errors. A reasonable estimate of sdY is required to continue.")
+    return np.sqrt(coef)

{genal_python-1.3.1 → genal_python-1.3.3}/genal/constants.py

@@ -5,8 +5,9 @@ BUILDS = ["37", "38"]
 POPULATIONS = ["EUR", "AFR", "EAS", "AMR", "SAS"]
 REF_PANELS = [f"{pop}_{build}" for pop in POPULATIONS for build in BUILDS]
 REF_PANEL_COLUMNS = ["CHR", "SNP", "POS", "A1", "A2"]
-REF_PANELS_URL = "https://storage.googleapis.com/genal_files/{panel}.tar.gz"
-REF_PARQUET_URL = "https://storage.googleapis.com/genal_files/reference_variants_{build}.parquet"
+BUCKET_URL = "https://storage.googleapis.com/genal_files/"
+REF_PANELS_URL = BUCKET_URL + "{panel}.tar.gz"
+REF_PARQUET_URL = BUCKET_URL + "reference_variants_{build}.parquet"
 CONFIG_DIR = os.path.expanduser("~/.genal/")
 CHECKS_DICT = {
     "CHR": False,