gsMap 1.72.3__py3-none-any.whl → 1.73.1__py3-none-any.whl

gsMap/latent_to_gene.py

@@ -104,6 +104,7 @@ def compute_regional_mkscore(
     ranks,
     frac_whole,
     adata_X_bool,
+    pearson_residuals,
 ):
     """
     Compute gmean ranks of a region.
@@ -129,7 +130,8 @@ def compute_regional_mkscore(
         # Simultaneously consider the ratio of expression fractions and ranks
         gene_ranks_region = gene_ranks_region * frac_region
 
-    mkscore = np.exp(gene_ranks_region**1.5) - 1
+    mkscore = np.exp(gene_ranks_region) - 1 if not pearson_residuals else gene_ranks_region
+
     return mkscore.astype(np.float16, copy=False)
 
 
@@ -147,22 +149,50 @@ def run_latent_to_gene(config: LatentToGeneConfig):
         )
 
     # Homologs transformation
-    if config.homolog_file is not None:
-        logger.info(f"------Transforming the {config.species} to HUMAN_GENE_SYM...")
-        homologs = pd.read_csv(config.homolog_file, sep="\t")
-        if homologs.shape[1] != 2:
-            raise ValueError(
-                "Homologs file must have two columns: one for the species and one for the human gene symbol."
+    if config.homolog_file is not None and config.species is not None:
+        species_col_name = f"{config.species}_homolog"
+
+        # Check if homolog conversion has already been performed
+        if species_col_name in adata.var.columns:
+            logger.warning(
+                f"Column '{species_col_name}' already exists in adata.var. "
+                f"It appears gene names have already been converted to human gene symbols. "
+                f"Skipping homolog transformation."
+            )
+        else:
+            logger.info(f"------Transforming the {config.species} to HUMAN_GENE_SYM...")
+            homologs = pd.read_csv(config.homolog_file, sep="\t")
+            if homologs.shape[1] != 2:
+                raise ValueError(
+                    "Homologs file must have two columns: one for the species and one for the human gene symbol."
+                )
+
+            homologs.columns = [config.species, "HUMAN_GENE_SYM"]
+            homologs.set_index(config.species, inplace=True)
+
+            # original_gene_names = adata.var_names.copy()
+
+            # Filter genes present in homolog file
+            adata = adata[:, adata.var_names.isin(homologs.index)]
+            logger.info(f"{adata.shape[1]} genes retained after homolog transformation.")
+            if adata.shape[1] < 100:
+                raise ValueError("Too few genes retained in ST data (<100).")
+
+            # Create mapping table of original to human gene names
+            gene_mapping = pd.Series(
+                homologs.loc[adata.var_names, "HUMAN_GENE_SYM"].values, index=adata.var_names
             )
 
-        homologs.columns = [config.species, "HUMAN_GENE_SYM"]
-        homologs.set_index(config.species, inplace=True)
-        adata = adata[:, adata.var_names.isin(homologs.index)]
-        logger.info(f"{adata.shape[1]} genes retained after homolog transformation.")
-        if adata.shape[1] < 100:
-            raise ValueError("Too few genes retained in ST data (<100).")
-        adata.var_names = homologs.loc[adata.var_names, "HUMAN_GENE_SYM"].values
-        adata = adata[:, ~adata.var_names.duplicated()]
+            # Store original species gene names in var dataframe with the suffixed column name
+            adata.var[species_col_name] = adata.var_names.values
+
+            # Convert var_names to human gene symbols
+            adata.var_names = gene_mapping.values
+            adata.var.index.name = "HUMAN_GENE_SYM"
+
+            # Remove duplicated genes after conversion
+            adata = adata[:, ~adata.var_names.duplicated()]
+            logger.info(f"{adata.shape[1]} genes retained after removing duplicates.")
 
     if config.annotation is not None:
         cell_annotations = adata.obs[config.annotation].values
@@ -218,11 +248,18 @@ def run_latent_to_gene(config: LatentToGeneConfig):
     # Create mappings
     n_cells = adata.n_obs
     n_genes = adata.n_vars
-
+    pearson_residuals = True if "pearson_residuals" in adata.layers else False
     ranks = np.zeros((n_cells, adata.n_vars), dtype=np.float16)
-    for i in tqdm(range(n_cells), desc="Computing ranks per cell"):
-        data = adata_X[i, :].toarray().flatten()
-        ranks[i, :] = rankdata(data, method="average")
+
+    if pearson_residuals:
+        logger.info("Using pearson residuals for ranking.")
+        data = adata.layers["pearson_residuals"]
+        for i in tqdm(range(n_cells), desc="Computing ranks per cell"):
+            ranks[i, :] = rankdata(data[i, :], method="average")
+    else:
+        for i in tqdm(range(n_cells), desc="Computing ranks per cell"):
+            data = adata_X[i, :].toarray().flatten()
+            ranks[i, :] = rankdata(data, method="average")
 
     if gM is None:
         gM = gmean(ranks, axis=0)
@@ -252,6 +289,7 @@ def run_latent_to_gene(config: LatentToGeneConfig):
             ranks,
             frac_whole,
             adata_X_bool,
+            pearson_residuals,
         )
 
     logger.info("------Computing marker scores...")

gsMap/run_all_mode.py

@@ -67,6 +67,8 @@ def run_pipeline(config: RunAllModeConfig):
             sample_name=config.sample_name,
             annotation=config.annotation,
             data_layer=config.data_layer,
+            n_comps=config.n_comps,
+            pearson_residuals=config.pearson_residuals,
         )
 
         # Step 1: Find latent representations

gsMap/utils/generate_r2_matrix.py

@@ -1,42 +1,7 @@
-from pathlib import Path
-
 import bitarray as ba
 import numpy as np
 import pandas as pd
-from scipy.sparse import csr_matrix, load_npz, save_npz
-from tqdm import tqdm, trange
-
-
-# Define the log class
-class Logger:
-    # -
-    def __init__(self, fh):
-        self.log_fh = open(fh, "w")
-
-    # -
-    def log(self, msg):
-        """
-        Print to log file and stdout.
-        """
-        print(msg, file=self.log_fh)
-        print(msg)
-
-    # -
-    def close(self):
-        self.log_fh.close()
-
-
-# Compute ld-score using cellular annotations
-def get_compression(fh):
-    """Which sort of compression should we use with read_csv?"""
-    if fh.endswith("gz"):
-        compression = "gzip"
-    elif fh.endswith("bz2"):
-        compression = "bz2"
-    else:
-        compression = None
-    # -
-    return compression
+from tqdm import tqdm
 
 
 # Define the reading functions
@@ -67,29 +32,17 @@ def ID_List_Factory(colnames, keepcol, fname_end, header=None, usecols=None):
             end = self.fname_end
             if end and not fname.endswith(end):
                 raise ValueError(f"{end} filename must end in {end}")
-            comp = get_compression(fname)
             self.df = pd.read_csv(
-                fname, header=self.header, usecols=self.usecols, sep=r"\s+", compression=comp
+                fname,
+                header=self.header,
+                usecols=self.usecols,
+                sep=r"\s+",
             )
             if self.colnames:
                 self.df.columns = self.colnames
             if self.keepcol is not None:
                 self.IDList = self.df.iloc[:, [self.keepcol]].astype("object")
 
-        # -
-        def loj(self, externalDf):
-            """
-            Perform a left outer join operation with the given external DataFrame.
-            """
-            r = externalDf.columns[0]
-            l = self.IDList.columns[0]
-            merge_df = externalDf.iloc[:, [0]]
-            merge_df["keep"] = True
-            z = pd.merge(self.IDList, merge_df, how="left", left_on=l, right_on=r, sort=False)
-            ii = z["keep"]
-            return np.nonzero(ii)[0]
-
-    # -
     return IDContainer
 
 
@@ -512,257 +465,25 @@ class PlinkBEDFile(GenotypeArrayInMemory):
         return Y
 
 
-class PlinkBEDFileWithR2Cache(PlinkBEDFile):
-    def compute_r2_cache(
-        self,
-        block_left,
-        output_cache_file_dir: Path,
-        chunk_size=500_000_000,
-        c=500,
-        r2_threshold=1e-4,
-        annot=None,
-    ):
-        func = np.square
-        snp_getter = self.nextSNPs
-        data, rows, cols = [], [], []
-
-        def add_rfuncAB(rfuncAB, l_A, l_B):
-            non_zero_indices = np.nonzero(rfuncAB > r2_threshold)
-            data.extend(rfuncAB[non_zero_indices])
-            rows.extend(l_A + non_zero_indices[0])
-            cols.extend(l_B + non_zero_indices[1])
-
-        # def add_rfuncAB(rfuncAB, l_A, l_B):
-        #     # not need select non zero indices
-        #     data.extend(rfuncAB.flatten())
-        #     rows.extend(l_A + np.repeat(np.arange(rfuncAB.shape[0]), rfuncAB.shape[1]))
-        #     cols.extend(l_B + np.tile(np.arange(rfuncAB.shape[1]), rfuncAB.shape[0]))
-
-        # def add_rfuncBB(rfuncBB, l_B):
-        #     non_zero_indices = np.nonzero(rfuncBB)
-        #     data.extend(rfuncBB[non_zero_indices])
-        #     rows.extend(l_B + non_zero_indices[0])
-        #     cols.extend(l_B + non_zero_indices[1])
-
-        def add_rfuncBB(rfuncBB, l_B):
-            non_zero_indices = np.nonzero(rfuncBB > r2_threshold)
-            data.extend(rfuncBB[non_zero_indices])
-            rows.extend(l_B + non_zero_indices[0])
-            cols.extend(l_B + non_zero_indices[1])
-            if len(data) > chunk_size:
-                # save the cache
-                print(f"Start saving the cache file: {output_cache_file_dir / f'{l_B}.npz'}")
-                r2_sparse_matrix = csr_matrix(
-                    (data, (rows, cols)), shape=(self.m, self.m), dtype="float16"
-                )
-                save_npz(output_cache_file_dir / f"{l_B}.npz", r2_sparse_matrix)
-                # reset the data
-                data.clear()
-                rows.clear()
-                cols.clear()
-
-        m, n = self.m, self.n
-        block_sizes = np.array(np.arange(m) - block_left)
-        block_sizes = np.ceil(block_sizes / c) * c
-        if annot is None:
-            annot = np.ones((m, 1))
-        else:
-            annot_m = annot.shape[0]
-            if annot_m != self.m:
-                raise ValueError("Incorrect number of SNPs in annot")
-        # -
-        # n_a = annot.shape[1]  # number of annotations
-        # cor_sum = np.zeros((m, n_a))
-        # b = index of first SNP for which SNP 0 is not included in LD Score
-        b = np.nonzero(block_left > 0)
-        if np.any(b):
-            b = b[0][0]
-        else:
-            b = m
-        b = int(np.ceil(b / c) * c)  # round up to a multiple of c
-        if b > m:
-            c = 1
-            b = m
-
-        l_A = 0  # l_A := index of leftmost SNP in matrix A
-        A = snp_getter(b)
-        rfuncAB = np.zeros((b, c))
-        rfuncBB = np.zeros((c, c))
-        # chunk inside of block
-        for l_B in np.arange(0, b, c):  # l_B := index of leftmost SNP in matrix B
-            B = A[:, l_B : l_B + c]
-            # ld matrix
-            np.dot(A.T, B / n, out=rfuncAB)
-            # ld matrix square
-            rfuncAB = func(rfuncAB)
-            add_rfuncAB(rfuncAB, l_A, l_B)
-            # cor_sum[l_A:l_A + b, :] += np.dot(rfuncAB, annot[l_B:l_B + c, :])
-
-        # chunk to right of block
-        b0 = b
-        md = int(c * np.floor(m / c))
-        end = md + 1 if md != m else md
-        for l_B in trange(b0, end, c, desc=f"Compute r2 cache for {output_cache_file_dir.name}"):
-            # check if the annot matrix is all zeros for this block + chunk
-            # this happens w/ sparse categories (i.e., pathways)
-            # update the block
-            old_b = b
-            b = int(block_sizes[l_B])
-            if l_B > b0 and b > 0:
-                # block_size can't increase more than c
-                # block_size can't be less than c unless it is zero
-                # both of these things make sense
-                A = np.hstack((A[:, old_b - b + c : old_b], B))
-                l_A += old_b - b + c
-            elif l_B == b0 and b > 0:
-                A = A[:, b0 - b : b0]
-                l_A = b0 - b
-            elif b == 0:  # no SNPs to left in window, e.g., after a sequence gap
-                A = np.array(()).reshape((n, 0))
-                l_A = l_B
-            if l_B == md:
-                c = m - md
-                rfuncAB = np.zeros((b, c))
-                rfuncBB = np.zeros((c, c))
-            if b != old_b:
-                rfuncAB = np.zeros((b, c))
-            # -
-            B = snp_getter(c)
-            p1 = np.all(annot[l_A : l_A + b, :] == 0)
-            p2 = np.all(annot[l_B : l_B + c, :] == 0)
-            if p1 and p2:
-                continue
-            # -
-            np.dot(A.T, B / n, out=rfuncAB)
-            rfuncAB = func(rfuncAB)
-            # cor_sum[l_A:l_A + b, :] += np.dot(rfuncAB, annot[l_B:l_B + c, :])
-            # cor_sum[l_B:l_B + c, :] += np.dot(annot[l_A:l_A + b, :].T, rfuncAB).T
-            add_rfuncAB(rfuncAB, l_A, l_B)
-            add_rfuncAB(rfuncAB.T, l_B, l_A)
-            np.dot(B.T, B / n, out=rfuncBB)
-            rfuncBB = func(rfuncBB)
-            # cor_sum[l_B:l_B + c, :] += np.dot(rfuncBB, annot[l_B:l_B + c, :])
-            add_rfuncBB(rfuncBB, l_B)
-        if len(data) > 0:
-            # save remaining data
-            # save the cache
-            print(f"Start saving the cache file: {output_cache_file_dir / f'{l_B}.npz'}")
-            r2_sparse_matrix = csr_matrix((data, (rows, cols)), shape=(m, m), dtype="float16")
-            save_npz(output_cache_file_dir / f"{l_B}.npz", r2_sparse_matrix)
-        # combine the cache files
-        print(f"Start combining the cache files in {output_cache_file_dir}")
-        cached_r2_matrix_files = list(output_cache_file_dir.glob("*.npz"))
-        combined_r2_matrix_files = self.load_r2_matrix_from_cache_files(output_cache_file_dir)
-        # remove the cache files
-        for cached_r2_matrix_file in cached_r2_matrix_files:
-            cached_r2_matrix_file.unlink()
-        # save the combined r2 matrix
-        print(f"Start saving the combined r2 matrix in {output_cache_file_dir}")
-        combined_r2_matrix_file = output_cache_file_dir / "combined_r2_matrix.npz"
-        save_npz(combined_r2_matrix_file, combined_r2_matrix_files)
-
-    def get_ldscore_using_r2_cache(self, annot_matrix, cached_r2_matrix_dir):
-        """
-        Compute the r2 matrix multiplication with annot_matrix
-        """
-        # Compute the r2 matrix multiplication with annot_matrix
-        cached_r2_matrix_dir = Path(cached_r2_matrix_dir)
-        # iter the cached r2 matrix files
-        result_matrix = np.zeros((self.m, annot_matrix.shape[1]))
-        cached_r2_matrix_files = list(cached_r2_matrix_dir.glob("*.npz"))
-        assert len(cached_r2_matrix_files) > 0, (
-            f"No cached r2 matrix files in {cached_r2_matrix_dir}"
-            f"Please run the function compute_r2_cache first!"
-        )
-        for r2_matrix_file in tqdm(
-            cached_r2_matrix_files, desc=f"Compute ld score for {cached_r2_matrix_dir.name}"
-        ):
-            print(f"Compute r2 matrix multiplication for {r2_matrix_file}")
-            r2_matrix = load_npz(r2_matrix_file)
-            result_matrix += r2_matrix.dot(annot_matrix)
-        return result_matrix
-
-    def load_r2_matrix_from_cache_files(self, cached_r2_matrix_dir):
-        """
-        Load the r2 matrix from cache
-        """
-        cached_r2_matrix_dir = Path(cached_r2_matrix_dir)
-        # iter the cached r2 matrix files
-        cached_r2_matrix_files = list(cached_r2_matrix_dir.glob("*.npz"))
-        assert len(cached_r2_matrix_files) > 0, (
-            f"No cached r2 matrix files in {cached_r2_matrix_dir}"
-            f"Please run the function compute_r2_cache first!"
-        )
-        # load the r2 matrix
-        r2_matrix = load_npz(cached_r2_matrix_files[0])
-        for r2_matrix_file in tqdm(
-            cached_r2_matrix_files[1:], desc=f"Load r2 matrix from {cached_r2_matrix_dir.name}"
-        ):
-            print(f"Load r2 matrix from {r2_matrix_file}")
-            r2_matrix += load_npz(r2_matrix_file)
-        # to float16
-        r2_matrix = r2_matrix.astype("float16")
-        return r2_matrix
-
-    def load_combined_r2_matrix(self, cached_r2_matrix_dir):
-        """
-        Load the combined r2 matrix
-        """
-        combined_r2_matrix_file = Path(cached_r2_matrix_dir) / "combined_r2_matrix.npz"
-        assert combined_r2_matrix_file.exists(), (
-            f"No combined r2 matrix file in {cached_r2_matrix_dir}"
-            f"Should delete the cache files and run the function compute_r2_cache first!"
-        )
-        # load the r2 matrix
-        r2_matrix = load_npz(combined_r2_matrix_file)
-        # to float16
-        r2_matrix = r2_matrix.astype("float16")
-        return r2_matrix
-
-
-def load_bfile(bfile_chr_prefix):
+def load_bfile(bfile_chr_prefix, keep_snps=None, keep_indivs=None, mafMin=None):
     PlinkBIMFile = ID_List_Factory(
         ["CHR", "SNP", "CM", "BP", "A1", "A2"], 1, ".bim", usecols=[0, 1, 2, 3, 4, 5]
     )
     PlinkFAMFile = ID_List_Factory(["IID"], 0, ".fam", usecols=[1])
 
-    snp_file, snp_obj = bfile_chr_prefix + ".bim", PlinkBIMFile
-    array_snps = snp_obj(snp_file)
-    m = len(array_snps.IDList)
-    print(f"Read list of {m} SNPs from {snp_file}")
-    #
+    snp_file = bfile_chr_prefix + ".bim"
+    array_snps = PlinkBIMFile(snp_file)
+
     # Load fam
-    ind_file, ind_obj = bfile_chr_prefix + ".fam", PlinkFAMFile
-    array_indivs = ind_obj(ind_file)
+    ind_file = bfile_chr_prefix + ".fam"
+    array_indivs = PlinkFAMFile(ind_file)
+
     n = len(array_indivs.IDList)
-    print(f"Read list of {n} individuals from {ind_file}")
 
     # Load genotype array
-    array_file, array_obj = bfile_chr_prefix + ".bed", PlinkBEDFileWithR2Cache
-    geno_array = array_obj(
-        array_file, n, array_snps, keep_snps=None, keep_indivs=None, mafMin=None
+    array_file = bfile_chr_prefix + ".bed"
+    geno_array = PlinkBEDFile(
+        array_file, n, array_snps, keep_snps=keep_snps, keep_indivs=keep_indivs, mafMin=mafMin
     )
 
     return array_snps, array_indivs, geno_array
-
-
-def generate_r2_matrix_chr_cache(bfile_chr_prefix, ld_wind_cm, output_cache_file_dir):
-    # Load genotype array
-    array_snps, array_indivs, geno_array = load_bfile(bfile_chr_prefix)
-    # Compute block lefts
-    # block_left = getBlockLefts(geno_array.df[:, 3], ld_wind_cm)
-    # Compute LD score
-    # r2_matrix = geno_array.load_r2_matrix_from_cache(output_cache_file_dir)
-
-
-def generate_r2_matrix_cache(bfile_prefix, chromosome_list, r2_cache_dir, ld_wind_cm=1):
-    r2_cache_dir = Path(r2_cache_dir)
-
-    for chr in chromosome_list:
-        output_cache_file_prefix = r2_cache_dir / f"chr{chr}"
-        output_cache_file_prefix.mkdir(parents=True, exist_ok=True)
-        bfile_chr_prefix = bfile_prefix + "." + str(chr)
-        generate_r2_matrix_chr_cache(
-            bfile_chr_prefix, ld_wind_cm=ld_wind_cm, output_cache_file_dir=output_cache_file_prefix
-        )
-        print(f"Compute r2 matrix for chr{chr} done!")

gsMap/utils/regression_read.py

@@ -89,37 +89,6 @@ def which_compression(fh):
     return suffix, compression
 
 
-def _read_ref_ld(ld_file):
-    suffix = ".l2.ldscore"
-    file = ld_file
-    first_fh = f"{file}1{suffix}"
-    s, compression = which_compression(first_fh)
-    #
-    ldscore_array = []
-    print(f"Reading ld score annotations from {file}[1-22]{suffix}.{compression}")
-
-    for chr in range(1, 23):
-        file_chr = f"{file}{chr}{suffix}{s}"
-        #
-        if compression == "parquet":
-            x = pd.read_parquet(file_chr)
-        elif compression == "feather":
-            x = pd.read_feather(file_chr)
-        else:
-            x = pd.read_csv(file_chr, compression=compression, sep="\t")
-
-        x = x.sort_values(by=["CHR", "BP"])  # SEs will be wrong unless sorted
-
-        columns_to_drop = ["MAF", "CM", "Gene", "TSS", "CHR", "BP"]
-        columns_to_drop = [col for col in columns_to_drop if col in x.columns]
-        x = x.drop(columns_to_drop, axis=1)
-
-        ldscore_array.append(x)
-    #
-    ref_ld = pd.concat(ldscore_array, axis=0)
-    return ref_ld
-
-
 def _read_ref_ld_v2(ld_file):
     suffix = ".l2.ldscore"
     file = ld_file
@@ -185,23 +154,6 @@ def M(fh, common=False):
     return np.array(M_array).reshape((1, len(M_array)))
 
 
-def _check_variance(M_annot, ref_ld):
-    """
-    Remove zero-variance LD Scores.
-    """
-    ii = ref_ld.iloc[:, 1:].var() == 0  # NB there is a SNP column here
-    if ii.all():
-        raise ValueError("All LD Scores have zero variance.")
-    else:
-        print("Removing partitioned LD Scores with zero variance.")
-        ii_snp = np.array([True] + list(~ii))
-        ii_m = np.array(~ii)
-        ref_ld = ref_ld.iloc[:, ii_snp]
-        M_annot = M_annot[:, ii_m]
-    # -
-    return M_annot, ref_ld, ii
-
-
 def _check_variance_v2(M_annot, ref_ld):
     ii = ref_ld.var() == 0
     if ii.all():
@@ -247,31 +199,3 @@ def _read_w_ld(w_file):
     w_ld.columns = ["SNP", "LD_weights"]
 
     return w_ld
-
-
-# Fun for merging
-def _merge_and_log(ld, sumstats, noun):
-    """
-    Wrap smart merge with log messages about # of SNPs.
-    """
-    sumstats = smart_merge(ld, sumstats)
-    msg = "After merging with {F}, {N} SNPs remain."
-    if len(sumstats) == 0:
-        raise ValueError(msg.format(N=len(sumstats), F=noun))
-    else:
-        print(msg.format(N=len(sumstats), F=noun))
-    # -
-    return sumstats
-
-
-def smart_merge(x, y):
-    """
-    Check if SNP columns are equal. If so, save time by using concat instead of merge.
-    """
-    if len(x) == len(y) and (x.index == y.index).all() and (x.SNP == y.SNP).all():
-        x = x.reset_index(drop=True)
-        y = y.reset_index(drop=True).drop("SNP", 1)
-        out = pd.concat([x, y], axis=1)
-    else:
-        out = pd.merge(x, y, how="inner", on="SNP")
-    return out