gsMap 1.72.3__py3-none-any.whl → 1.73.0__py3-none-any.whl

gsMap/__init__.py

@@ -2,4 +2,4 @@
 Genetics-informed pathogenic spatial mapping
 """
 
-__version__ = "1.72.3"
+__version__ = "1.73.0"

gsMap/cauchy_combination_test.py

@@ -48,16 +48,16 @@ def acat_test(pvalues, weights=None):
     elif any(i < 0 for i in weights):
         raise Exception("All weights must be positive.")
     else:
-        weights = [i / len(weights) for i in weights]
+        weights = [i / np.sum(weights) for i in weights]
 
     pvalues = np.array(pvalues)
     weights = np.array(weights)
 
-    if not any(i < 1e-16 for i in pvalues):
+    if not any(i < 1e-15 for i in pvalues):
         cct_stat = sum(weights * np.tan((0.5 - pvalues) * np.pi))
     else:
-        is_small = [i < (1e-16) for i in pvalues]
-        is_large = [i >= (1e-16) for i in pvalues]
+        is_small = [i < (1e-15) for i in pvalues]
+        is_large = [i >= (1e-15) for i in pvalues]
         cct_stat = sum((weights[is_small] / pvalues[is_small]) / np.pi)
         cct_stat += sum(weights[is_large] * np.tan((0.5 - pvalues[is_large]) * np.pi))
 
@@ -118,7 +118,7 @@ def run_Cauchy_combination(config: CauchyCombinationConfig):
         n_removed = len(p_values) - len(p_values_filtered)
 
         # Remove outliers if the number is reasonable
-        if 0 < n_removed < 20:
+        if 0 < n_removed < max(len(p_values) * 0.01, 20):
             logger.info(f"Removed {n_removed}/{len(p_values)} outliers (median + 3IQR) for {ct}.")
             p_cauchy_temp = acat_test(p_values_filtered)
         else:

gsMap/config.py

@@ -1,7 +1,10 @@
 import argparse
 import dataclasses
 import logging
+import os
 import sys
+import threading
+import time
 from collections import OrderedDict, namedtuple
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -10,6 +13,7 @@ from pathlib import Path
 from pprint import pprint
 from typing import Literal
 
+import psutil
 import pyfiglet
 import yaml
 
@@ -34,9 +38,109 @@ def get_gsMap_logger(logger_name):
 logger = get_gsMap_logger("gsMap")
 
 
+def track_resource_usage(func):
+    """
+    Decorator to track resource usage during function execution.
+    Logs memory usage, CPU time, and wall clock time at the end of the function.
+    """
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        # Get the current process
+        process = psutil.Process(os.getpid())
+
+        # Initialize tracking variables
+        peak_memory = 0
+        cpu_percent_samples = []
+        stop_thread = False
+
+        # Function to monitor resource usage
+        def resource_monitor():
+            nonlocal peak_memory, cpu_percent_samples
+            while not stop_thread:
+                try:
+                    # Get current memory usage in MB
+                    current_memory = process.memory_info().rss / (1024 * 1024)
+                    peak_memory = max(peak_memory, current_memory)
+
+                    # Get CPU usage percentage
+                    cpu_percent = process.cpu_percent(interval=None)
+                    if cpu_percent > 0:  # Skip initial zero readings
+                        cpu_percent_samples.append(cpu_percent)
+
+                    time.sleep(0.5)
+                except Exception:  # Catching all exceptions here because... # noqa: BLE001
+                    pass
+
+        # Start resource monitoring in a separate thread
+        monitor_thread = threading.Thread(target=resource_monitor)
+        monitor_thread.daemon = True
+        monitor_thread.start()
+
+        # Get start times
+        start_wall_time = time.time()
+        start_cpu_time = process.cpu_times().user + process.cpu_times().system
+
+        try:
+            # Run the actual function
+            result = func(*args, **kwargs)
+            return result
+        finally:
+            # Stop the monitoring thread
+            stop_thread = True
+            monitor_thread.join(timeout=1.0)
+
+            # Calculate elapsed times
+            end_wall_time = time.time()
+            end_cpu_time = process.cpu_times().user + process.cpu_times().system
+
+            wall_time = end_wall_time - start_wall_time
+            cpu_time = end_cpu_time - start_cpu_time
+
+            # Calculate average CPU percentage
+            avg_cpu_percent = (
+                sum(cpu_percent_samples) / len(cpu_percent_samples) if cpu_percent_samples else 0
+            )
+
+            # Format memory for display
+            if peak_memory < 1024:
+                memory_str = f"{peak_memory:.2f} MB"
+            else:
+                memory_str = f"{peak_memory / 1024:.2f} GB"
+
+            # Format times for display
+            if wall_time < 60:
+                wall_time_str = f"{wall_time:.2f} seconds"
+            elif wall_time < 3600:
+                wall_time_str = f"{wall_time / 60:.2f} minutes"
+            else:
+                wall_time_str = f"{wall_time / 3600:.2f} hours"
+
+            if cpu_time < 60:
+                cpu_time_str = f"{cpu_time:.2f} seconds"
+            elif cpu_time < 3600:
+                cpu_time_str = f"{cpu_time / 60:.2f} minutes"
+            else:
+                cpu_time_str = f"{cpu_time / 3600:.2f} hours"
+
+            # Log the resource usage
+            import logging
+
+            logger = logging.getLogger("gsMap")
+            logger.info("Resource usage summary:")
+            logger.info(f"  • Wall clock time: {wall_time_str}")
+            logger.info(f"  • CPU time: {cpu_time_str}")
+            logger.info(f"  • Average CPU utilization: {avg_cpu_percent:.1f}%")
+            logger.info(f"  • Peak memory usage: {memory_str}")
+
+    return wrapper
+
+
 # Decorator to register functions for cli parsing
 def register_cli(name: str, description: str, add_args_function: Callable) -> Callable:
     def decorator(func: Callable) -> Callable:
+        @track_resource_usage  # Use enhanced resource tracking
+        @wraps(func)
         def wrapper(*args, **kwargs):
             name.replace("_", " ")
             gsMap_main_logo = pyfiglet.figlet_format(
@@ -50,8 +154,16 @@ def register_cli(name: str, description: str, add_args_function: Callable) -> Ca
             print(version_number.center(80), flush=True)
             print("=" * 80, flush=True)
             logger.info(f"Running {name}...")
+
+            # Record start time for the log message
+            start_time = time.strftime("%Y-%m-%d %H:%M:%S")
+            logger.info(f"Started at: {start_time}")
+
             func(*args, **kwargs)
-            logger.info(f"Finished running {name}.")
+
+            # Record end time for the log message
+            end_time = time.strftime("%Y-%m-%d %H:%M:%S")
+            logger.info(f"Finished running {name} at: {end_time}.")
 
         cli_function_registry[name] = subcommand(
             name=name, func=wrapper, add_args_function=add_args_function, description=description
@@ -61,6 +173,13 @@ def register_cli(name: str, description: str, add_args_function: Callable) -> Ca
     return decorator
 
 
+def str_or_float(value):
+    try:
+        return int(value)
+    except ValueError:
+        return value
+
+
 def add_shared_args(parser):
     parser.add_argument(
         "--workdir", type=str, required=True, help="Path to the working directory."
@@ -429,7 +548,7 @@ def add_format_sumstats_args(parser):
     parser.add_argument(
         "--n",
         default=None,
-        type=str,
+        type=str_or_float,
         help="Name of sample size column (if not a name that gsMap understands)",
     )
     parser.add_argument(
@@ -1037,7 +1156,7 @@ class CauchyCombinationConfig(ConfigWithAutoPaths):
 
     def __post_init__(self):
         if self.sample_name is not None:
-            if len(self.sample_name_list) > 0:
+            if self.sample_name_list and len(self.sample_name_list) > 0:
                 raise ValueError("Only one of sample_name and sample_name_list must be provided.")
             else:
                 self.sample_name_list = [self.sample_name]
@@ -1106,6 +1225,9 @@ class RunAllModeConfig(ConfigWithAutoPaths):
     annotation: str
     data_layer: str = "X"
 
+    # == Find Latent Representation PARAMETERS ==
+    n_comps: int = 300
+
     # == latent 2 Gene PARAMETERS ==
     gM_slices: str | None = None
     latent_representation: str = None
@@ -1124,9 +1246,7 @@ class RunAllModeConfig(ConfigWithAutoPaths):
 
     def __post_init__(self):
         super().__post_init__()
-        self.gtffile = (
-            f"{self.gsMap_resource_dir}/genome_annotation/gtf/gencode.v39lift37.annotation.gtf"
-        )
+        self.gtffile = f"{self.gsMap_resource_dir}/genome_annotation/gtf/gencode.v46lift37.basic.annotation.gtf"
         self.bfile_root = (
             f"{self.gsMap_resource_dir}/LD_Reference_Panel/1000G_EUR_Phase3_plink/1000G.EUR.QC"
         )
@@ -1191,7 +1311,7 @@ class FormatSumstatsConfig:
     se: str = None
     p: str = None
     frq: str = None
-    n: str = None
+    n: str | int = None
     z: str = None
     OR: str = None
     se_OR: str = None
@@ -1204,9 +1324,21 @@ class FormatSumstatsConfig:
     keep_chr_pos: bool = False
 
 
+@register_cli(
+    name="quick_mode",
+    description="Run the entire gsMap pipeline in quick mode, utilizing pre-computed weights for faster execution.",
+    add_args_function=add_run_all_mode_args,
+)
+def run_all_mode_from_cli(args: argparse.Namespace):
+    from gsMap.run_all_mode import run_pipeline
+
+    config = get_dataclass_from_parser(args, RunAllModeConfig)
+    run_pipeline(config)
+
+
 @register_cli(
     name="run_find_latent_representations",
-    description="Run Find_latent_representations \nFind the latent representations of each spot by running GNN-VAE",
+    description="Run Find_latent_representations \nFind the latent representations of each spot by running GNN",
     add_args_function=add_find_latent_representations_args,
 )
 def run_find_latent_representation_from_cli(args: argparse.Namespace):
@@ -1278,7 +1410,7 @@ def run_Report_from_cli(args: argparse.Namespace):
 
 @register_cli(
     name="format_sumstats",
-    description="Format gwas summary statistics",
+    description="Format GWAS summary statistics",
     add_args_function=add_format_sumstats_args,
 )
 def gwas_format_from_cli(args: argparse.Namespace):
@@ -1288,18 +1420,6 @@ def gwas_format_from_cli(args: argparse.Namespace):
     gwas_format(config)
 
 
-@register_cli(
-    name="quick_mode",
-    description="Run all the gsMap pipeline in quick mode",
-    add_args_function=add_run_all_mode_args,
-)
-def run_all_mode_from_cli(args: argparse.Namespace):
-    from gsMap.run_all_mode import run_pipeline
-
-    config = get_dataclass_from_parser(args, RunAllModeConfig)
-    run_pipeline(config)
-
-
 @register_cli(
     name="create_slice_mean",
     description="Create slice mean from multiple h5ad files",

gsMap/create_slice_mean.py

@@ -5,8 +5,9 @@ import anndata
 import numpy as np
 import pandas as pd
 import scanpy as sc
+import scipy
 import zarr
-from scipy.stats import rankdata
+from scipy.stats import gmean, rankdata
 from tqdm import tqdm
 
 from gsMap.config import CreateSliceMeanConfig
@@ -62,22 +63,27 @@ def calculate_one_slice_mean(
 
     adata = adata[:, common_genes].copy()
     n_cells = adata.shape[0]
-    log_ranks = np.zeros((n_cells, adata.n_vars), dtype=np.float32)
-    # Compute log of ranks to avoid overflow when computing geometric mean
-    for i in tqdm(range(n_cells), desc=f"Computing log ranks for {sample_name}"):
-        data = adata.X[i, :].toarray().flatten()
-        ranks = rankdata(data, method="average")
-        log_ranks[i, :] = np.log(ranks)  # Adding small value to avoid log(0)
 
-    # Calculate geometric mean via log trick: exp(mean(log(values)))
-    gmean = (np.exp(np.mean(log_ranks, axis=0))).reshape(-1, 1)
+    if not scipy.sparse.issparse(adata.X):
+        adata_X = scipy.sparse.csr_matrix(adata.X)
+    elif isinstance(adata.X, scipy.sparse.csr_matrix):
+        adata_X = adata.X  # Avoid copying if already CSR
+    else:
+        adata_X = adata.X.tocsr()
+
+    ranks = np.zeros((n_cells, adata.n_vars), dtype=np.float16)
+    for i in tqdm(range(n_cells), desc="Computing ranks per cell"):
+        data = adata_X[i, :].toarray().flatten()
+        ranks[i, :] = rankdata(data, method="average")
+
+    gM = gmean(ranks, axis=0).reshape(-1, 1)
 
     # Calculate the expression fractio
     adata_X_bool = adata.X.astype(bool)
     frac = (np.asarray(adata_X_bool.sum(axis=0)).flatten()).reshape(-1, 1)
 
     # Save to zarr group
-    gmean_frac = np.concatenate([gmean, frac], axis=1)
+    gmean_frac = np.concatenate([gM, frac], axis=1)
     s1_zarr = gmean_zarr_group.array(sample_name, data=gmean_frac, chunks=None, dtype="f4")
     s1_zarr.attrs["spot_number"] = adata.shape[0]
 
@@ -85,34 +91,42 @@ def calculate_one_slice_mean(
 def merge_zarr_means(zarr_group_path, output_file, common_genes):
     """
     Merge all Zarr arrays into a weighted geometric mean and save to a Parquet file.
-    Instead of calculating the mean, it sums the logs and applies the exponential.
     """
     gmean_zarr_group = zarr.open(zarr_group_path, mode="a")
-    log_sum = None
+
+    sample_gmeans = []
+    sample_weights = []
     frac_sum = None
     total_spot_number = 0
+
+    # Collect all geometric means and their weights (spot numbers)
     for key in tqdm(gmean_zarr_group.array_keys(), desc="Merging Zarr arrays"):
         s1 = gmean_zarr_group[key]
         s1_array_gmean = s1[:][:, 0]
         s1_array_frac = s1[:][:, 1]
         n = s1.attrs["spot_number"]
 
-        if log_sum is None:
-            log_sum = np.log(s1_array_gmean) * n
+        sample_gmeans.append(s1_array_gmean)
+        sample_weights.append(n)
+
+        if frac_sum is None:
             frac_sum = s1_array_frac
         else:
-            log_sum += np.log(s1_array_gmean) * n
             frac_sum += s1_array_frac
 
         total_spot_number += n
 
-    # Apply the geometric mean via exponentiation of the averaged logs
-    final_mean = np.exp(log_sum / total_spot_number)
+    # Convert to arrays
+    sample_gmeans = np.array(sample_gmeans)
+    sample_weights = np.array(sample_weights)
+
+    final_gmean = gmean(sample_gmeans, axis=0, weights=sample_weights[:, np.newaxis])
+
     final_frac = frac_sum / total_spot_number
 
     # Save the final mean to a Parquet file
     gene_names = common_genes
-    final_df = pd.DataFrame({"gene": gene_names, "G_Mean": final_mean, "frac": final_frac})
+    final_df = pd.DataFrame({"gene": gene_names, "G_Mean": final_gmean, "frac": final_frac})
     final_df.set_index("gene", inplace=True)
     final_df.to_parquet(output_file)
     return final_df

gsMap/find_latent_representation.py

@@ -38,7 +38,7 @@ def preprocess_data(adata, params):
 
     if params.data_layer in adata.layers.keys():
         logger.info(f"Using data layer: {params.data_layer}...")
-        adata.X = adata.layers[params.data_layer]
+        adata.X = adata.layers[params.data_layer].copy()
     elif params.data_layer == "X":
         logger.info(f"Using data layer: {params.data_layer}...")
         if adata.X.dtype == "float32" or adata.X.dtype == "float64":

gsMap/format_sumstats.py

@@ -409,6 +409,12 @@ def gwas_format(config: FormatSumstatsConfig):
         compression=compression_type,
         na_values=[".", "NA"],
     )
+
+    if isinstance(config.n, int | float):
+        logger.info(f"Set the sample size of gwas data as {config.n}.")
+        gwas["N"] = config.n
+        config.n = "N"
+
     logger.info(f"Read {len(gwas)} SNPs from {config.sumstats}.")
 
     # Check name and format

gsMap/generate_ldscore.py

@@ -10,7 +10,7 @@ from scipy.sparse import csr_matrix
 from tqdm import trange
 
 from gsMap.config import GenerateLDScoreConfig
-from gsMap.utils.generate_r2_matrix import ID_List_Factory, PlinkBEDFileWithR2Cache, getBlockLefts
+from gsMap.utils.generate_r2_matrix import getBlockLefts, load_bfile
 
 warnings.filterwarnings("ignore", category=FutureWarning)
 logger = logging.getLogger(__name__)
@@ -71,9 +71,6 @@ def load_marker_score(mk_score_file):
     return mk_score
 
 
-# %%
-# load mkscore get common gene
-# %%
 # load bim
 def load_bim(bfile_root, chrom):
     """
@@ -146,52 +143,23 @@ def get_snp_pass_maf(bfile_root, chrom, maf_min=0.05):
     """
     Get the dummy matrix of SNP-gene pairs.
     """
-    # Load the bim file
-    PlinkBIMFile = ID_List_Factory(
-        ["CHR", "SNP", "CM", "BP", "A1", "A2"], 1, ".bim", usecols=[0, 1, 2, 3, 4, 5]
-    )
-    PlinkFAMFile = ID_List_Factory(["IID"], 0, ".fam", usecols=[1])
+    array_snps, array_indivs, geno_array = load_bfile(bfile_chr_prefix=f"{bfile_root}.{chrom}")
 
-    bfile = f"{bfile_root}.{chrom}"
-    snp_file, snp_obj = bfile + ".bim", PlinkBIMFile
-    array_snps = snp_obj(snp_file)
-    # m = len(array_snps.IDList)
-
-    # Load fam
-    ind_file, ind_obj = bfile + ".fam", PlinkFAMFile
-    array_indivs = ind_obj(ind_file)
+    m = len(array_snps.IDList)
     n = len(array_indivs.IDList)
-    array_file, array_obj = bfile + ".bed", PlinkBEDFileWithR2Cache
-    geno_array = array_obj(
-        array_file, n, array_snps, keep_snps=None, keep_indivs=None, mafMin=None
+    logger.info(
+        f"Loading genotype data for {m} SNPs and {n} individuals from {bfile_root}.{chrom}"
     )
+
     ii = geno_array.maf > maf_min
     snp_pass_maf = array_snps.IDList[ii]
-    print(f"After filtering SNPs with MAF < {maf_min}, {len(snp_pass_maf)} SNPs remain.")
+    logger.info(f"After filtering SNPs with MAF < {maf_min}, {len(snp_pass_maf)} SNPs remain.")
     return snp_pass_maf.SNP.to_list()
 
 
 def get_ldscore(bfile_root, chrom, annot_matrix, ld_wind, ld_unit="CM"):
-    PlinkBIMFile = ID_List_Factory(
-        ["CHR", "SNP", "CM", "BP", "A1", "A2"], 1, ".bim", usecols=[0, 1, 2, 3, 4, 5]
-    )
-    PlinkFAMFile = ID_List_Factory(["IID"], 0, ".fam", usecols=[1])
-
-    bfile = f"{bfile_root}.{chrom}"
-    snp_file, snp_obj = bfile + ".bim", PlinkBIMFile
-    array_snps = snp_obj(snp_file)
-    m = len(array_snps.IDList)
-    print(f"Read list of {m} SNPs from {snp_file}")
+    array_snps, array_indivs, geno_array = load_bfile(bfile_chr_prefix=f"{bfile_root}.{chrom}")
 
-    # Load fam
-    ind_file, ind_obj = bfile + ".fam", PlinkFAMFile
-    array_indivs = ind_obj(ind_file)
-    n = len(array_indivs.IDList)
-    print(f"Read list of {n} individuals from {ind_file}")
-    array_file, array_obj = bfile + ".bed", PlinkBEDFileWithR2Cache
-    geno_array = array_obj(
-        array_file, n, array_snps, keep_snps=None, keep_indivs=None, mafMin=None
-    )
     # Load the annotations of the baseline
     if ld_unit == "SNP":
         max_dist = ld_wind

gsMap/latent_to_gene.py

@@ -147,22 +147,50 @@ def run_latent_to_gene(config: LatentToGeneConfig):
         )
 
     # Homologs transformation
-    if config.homolog_file is not None:
-        logger.info(f"------Transforming the {config.species} to HUMAN_GENE_SYM...")
-        homologs = pd.read_csv(config.homolog_file, sep="\t")
-        if homologs.shape[1] != 2:
-            raise ValueError(
-                "Homologs file must have two columns: one for the species and one for the human gene symbol."
+    if config.homolog_file is not None and config.species is not None:
+        species_col_name = f"{config.species}_homolog"
+
+        # Check if homolog conversion has already been performed
+        if species_col_name in adata.var.columns:
+            logger.warning(
+                f"Column '{species_col_name}' already exists in adata.var. "
+                f"It appears gene names have already been converted to human gene symbols. "
+                f"Skipping homolog transformation."
             )
+        else:
+            logger.info(f"------Transforming the {config.species} to HUMAN_GENE_SYM...")
+            homologs = pd.read_csv(config.homolog_file, sep="\t")
+            if homologs.shape[1] != 2:
+                raise ValueError(
+                    "Homologs file must have two columns: one for the species and one for the human gene symbol."
+                )
+
+            homologs.columns = [config.species, "HUMAN_GENE_SYM"]
+            homologs.set_index(config.species, inplace=True)
+
+            # original_gene_names = adata.var_names.copy()
+
+            # Filter genes present in homolog file
+            adata = adata[:, adata.var_names.isin(homologs.index)]
+            logger.info(f"{adata.shape[1]} genes retained after homolog transformation.")
+            if adata.shape[1] < 100:
+                raise ValueError("Too few genes retained in ST data (<100).")
+
+            # Create mapping table of original to human gene names
+            gene_mapping = pd.Series(
+                homologs.loc[adata.var_names, "HUMAN_GENE_SYM"].values, index=adata.var_names
+            )
+
+            # Store original species gene names in var dataframe with the suffixed column name
+            adata.var[species_col_name] = adata.var_names.values
+
+            # Convert var_names to human gene symbols
+            adata.var_names = gene_mapping.values
+            adata.var.index.name = "HUMAN_GENE_SYM"
 
-        homologs.columns = [config.species, "HUMAN_GENE_SYM"]
-        homologs.set_index(config.species, inplace=True)
-        adata = adata[:, adata.var_names.isin(homologs.index)]
-        logger.info(f"{adata.shape[1]} genes retained after homolog transformation.")
-        if adata.shape[1] < 100:
-            raise ValueError("Too few genes retained in ST data (<100).")
-        adata.var_names = homologs.loc[adata.var_names, "HUMAN_GENE_SYM"].values
-        adata = adata[:, ~adata.var_names.duplicated()]
+            # Remove duplicated genes after conversion
+            adata = adata[:, ~adata.var_names.duplicated()]
+            logger.info(f"{adata.shape[1]} genes retained after removing duplicates.")
 
     if config.annotation is not None:
         cell_annotations = adata.obs[config.annotation].values

gsMap/run_all_mode.py

@@ -67,6 +67,7 @@ def run_pipeline(config: RunAllModeConfig):
             sample_name=config.sample_name,
             annotation=config.annotation,
             data_layer=config.data_layer,
+            n_comps=config.n_comps,
         )
 
         # Step 1: Find latent representations