gsMap 1.65__py3-none-any.whl → 1.67__py3-none-any.whl

gsMap/find_latent_representation.py

@@ -1,145 +1,145 @@
 import logging
 import random
-
 import numpy as np
-import pandas as pd
 import scanpy as sc
 import torch
-from sklearn import preprocessing
-
-from gsMap.GNN_VAE.adjacency_matrix import Construct_Adjacency_Matrix
-from gsMap.GNN_VAE.train import Model_Train
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import LabelEncoder
+from gsMap.GNN_VAE.adjacency_matrix import construct_adjacency_matrix
+from gsMap.GNN_VAE.train import ModelTrainer
 from gsMap.config import FindLatentRepresentationsConfig
 
 logger = logging.getLogger(__name__)
 
+
 def set_seed(seed_value):
     """
-    Set seed for reproducibility in PyTorch.
+    Set seed for reproducibility in PyTorch and other libraries.
     """
-    torch.manual_seed(seed_value)  # Set the seed for PyTorch
-    np.random.seed(seed_value)  # Set the seed for NumPy
-    random.seed(seed_value)  # Set the seed for Python random module
+    torch.manual_seed(seed_value)
+    np.random.seed(seed_value)
+    random.seed(seed_value)
     if torch.cuda.is_available():
-        logger.info('Running use GPU')
-        torch.cuda.manual_seed(seed_value)  # Set seed for all CUDA devices
-        torch.cuda.manual_seed_all(seed_value)  # Set seed for all CUDA devices
+        logger.info('Using GPU for computations.')
+        torch.cuda.manual_seed(seed_value)
+        torch.cuda.manual_seed_all(seed_value)
     else:
-        logger.info('Running use CPU')
+        logger.info('Using CPU for computations.')
 
+def preprocess_data(adata, params):
+    """
+    Preprocess the AnnData
+    """
+    logger.info('Preprocessing data...')
+    adata.var_names_make_unique()
+
+    sc.pp.filter_genes(adata, min_cells=30)
+    if params.data_layer in adata.layers.keys():
+        adata.X = adata.layers[params.data_layer]
+    else:
+        raise ValueError(f'Invalid data layer: {params.data_layer}, please check the input data.')
 
-# The class for finding latent representations
-class Latent_Representation_Finder:
+    if params.data_layer in ['count', 'counts']:
 
-    def __init__(self, adata, args:FindLatentRepresentationsConfig):
-        self.adata = adata.copy()
-        self.Params = args
+        sc.pp.normalize_total(adata, target_sum=1e4)
+        sc.pp.log1p(adata)
 
-        # Standard process
-        if self.Params.data_layer == 'count' or self.Params.data_layer == 'counts':
-            self.adata.X = self.adata.layers[self.Params.data_layer]
-            sc.pp.highly_variable_genes(self.adata, flavor="seurat_v3", n_top_genes=self.Params.feat_cell)
-            sc.pp.normalize_total(self.adata, target_sum=1e4)
-            sc.pp.log1p(self.adata)
-            sc.pp.scale(self.adata)
-        else:
-            if self.Params.data_layer != 'X':
-                self.adata.X = self.adata.layers[self.Params.data_layer]
-            sc.pp.highly_variable_genes(self.adata, n_top_genes=self.Params.feat_cell)
+        # Identify highly variable genes
+        sc.pp.highly_variable_genes(
+            adata,
+            flavor="seurat_v3",
+            n_top_genes=params.feat_cell,
+        )
 
-    def Run_GNN_VAE(self, label, verbose='whole ST data'):
+    elif params.data_layer in adata.layers.keys():
+        logger.info(f'Using {params.data_layer} data...')
+        sc.pp.highly_variable_genes(
+            adata,
+            flavor="seurat",
+            n_top_genes=params.feat_cell,
+        )
 
-        # Construct the neighbouring graph
-        graph_dict = Construct_Adjacency_Matrix(self.adata, self.Params)
+    return adata
 
-        # Process the feature matrix
-        node_X = self.adata[:, self.adata.var.highly_variable].X
-        logger.info(f'The shape of feature matrix is {node_X.shape}.')
-        if self.Params.input_pca:
-            node_X = sc.pp.pca(node_X, n_comps=self.Params.n_comps)
+
+class LatentRepresentationFinder:
+    def __init__(self, adata, args: FindLatentRepresentationsConfig):
+        self.params = args
+
+        self.expression_array = adata[:, adata.var.highly_variable].X.copy()
+
+        if self.params.data_layer in ['count', 'counts']:
+            self.expression_array = sc.pp.scale(self.expression_array, max_value=10)
+
+        # Construct the neighboring graph
+        self.graph_dict = construct_adjacency_matrix(adata, self.params)
+
+    def compute_pca(self):
+        self.latent_pca = PCA(n_components=self.params.feat_cell).fit_transform(self.expression_array)
+        return self.latent_pca
+
+    def run_gnn_vae(self, label, verbose='whole ST data'):
+
+        # Use PCA if specified
+        if self.params.input_pca:
+            node_X = self.compute_pca()
+        else:
+            node_X = self.expression_array
 
         # Update the input shape
-        self.Params.n_nodes = node_X.shape[0]
-        self.Params.feat_cell = node_X.shape[1]
+        self.params.n_nodes = node_X.shape[0]
+        self.params.feat_cell = node_X.shape[1]
 
-        # Run GNN-VAE
-        logger.info(f'------Finding latent representations for {verbose}...')
-        gvae = Model_Train(node_X, graph_dict, self.Params, label)
+        # Run GNN
+        logger.info(f'Finding latent representations for {verbose}...')
+        gvae = ModelTrainer(node_X, self.graph_dict, self.params, label)
         gvae.run_train()
 
-        return gvae.get_latent()
+        del self.graph_dict
 
-    def Run_PCA(self):
-        sc.tl.pca(self.adata)
-        return self.adata.obsm['X_pca'][:, 0:self.Params.n_comps]
+        return gvae.get_latent()
 
 
-def run_find_latent_representation(args:FindLatentRepresentationsConfig):
+def run_find_latent_representation(args: FindLatentRepresentationsConfig):
     set_seed(2024)
-    num_features = args.feat_cell
-    args.hdf5_with_latent_path.parent.mkdir(parents=True, exist_ok=True,mode=0o755)
+
     # Load the ST data
-    logger.info(f'------Loading ST data of {args.sample_name}...')
-    adata = sc.read_h5ad(f'{args.input_hdf5_path}')
-    adata.var_names_make_unique()
-    adata.X = adata.layers[args.data_layer] if args.data_layer in adata.layers.keys() else adata.X
-    logger.info('The ST data contains %d cells, %d genes.' % (adata.shape[0], adata.shape[1]))
+    logger.info(f'Loading ST data of {args.sample_name}...')
+    adata = sc.read_h5ad(args.input_hdf5_path)
+    logger.info(f'The ST data contains {adata.shape[0]} cells, {adata.shape[1]} genes.')
+
     # Load the cell type annotation
-    if not args.annotation is None:
-        # remove cells without enough annotations
-        adata = adata[~pd.isnull(adata.obs[args.annotation]), :]
+    if args.annotation is not None:
+        # Remove cells without enough annotations
+        adata = adata[~adata.obs[args.annotation].isnull()]
         num = adata.obs[args.annotation].value_counts()
-        adata = adata[adata.obs[args.annotation].isin(num[num >= 30].index.to_list())]
+        valid_annotations = num[num >= 30].index.to_list()
+        adata = adata[adata.obs[args.annotation].isin(valid_annotations)]
 
-        le = preprocessing.LabelEncoder()
-        le.fit(adata.obs[args.annotation])
-        adata.obs['categorical_label'] = le.transform(adata.obs[args.annotation])
-        label = adata.obs['categorical_label'].to_list()
+        le = LabelEncoder()
+        adata.obs['categorical_label'] = le.fit_transform(adata.obs[args.annotation])
+        label = adata.obs['categorical_label'].to_numpy()
     else:
         label = None
-    # Find latent representations
-    latent_rep = Latent_Representation_Finder(adata, args)
-    latent_GVAE = latent_rep.Run_GNN_VAE(label)
-    latent_PCA = latent_rep.Run_PCA()
-    # Add latent representations to the spe data
-    logger.info(f'------Adding latent representations...')
-    adata.obsm["latent_GVAE"] = latent_GVAE
-    adata.obsm["latent_PCA"] = latent_PCA
-    # Run umap based on latent representations
-    for name in ['latent_GVAE', 'latent_PCA']:
-        sc.pp.neighbors(adata, n_neighbors=10, use_rep=name)
-        sc.tl.umap(adata)
-        adata.obsm['X_umap_' + name] = adata.obsm['X_umap']
 
-        # Find the latent representations hierarchically (optionally)
-    if not args.annotation is None and args.hierarchically:
-        logger.info(f'------Finding latent representations hierarchically...')
-        PCA_all = pd.DataFrame()
-        GVAE_all = pd.DataFrame()
+    # Preprocess data
+    adata = preprocess_data(adata, args)
 
-        for ct in adata.obs[args.annotation].unique():
-            adata_part = adata[adata.obs[args.annotation] == ct, :]
-            logger.info(adata_part.shape)
+    latent_rep = LatentRepresentationFinder(adata, args)
+    latent_gvae = latent_rep.run_gnn_vae(label)
+    latent_pca = latent_rep.compute_pca()
 
-            # Find latent representations for the selected ct
-            latent_rep = Latent_Representation_Finder(adata_part, args)
+    # Add latent representations to the AnnData object
+    logger.info('Adding latent representations...')
+    adata.obsm["latent_GVAE"] = latent_gvae
+    adata.obsm["latent_PCA"] = latent_pca
 
-            latent_PCA_part = pd.DataFrame(latent_rep.Run_PCA())
-            if adata_part.shape[0] <= args.n_comps:
-                latent_GVAE_part = latent_PCA_part
-            else:
-                latent_GVAE_part = pd.DataFrame(latent_rep.Run_GNN_VAE(label=None, verbose=ct))
-
-            latent_GVAE_part.index = adata_part.obs_names
-            latent_PCA_part.index = adata_part.obs_names
-
-            GVAE_all = pd.concat((GVAE_all, latent_GVAE_part), axis=0)
-            PCA_all = pd.concat((PCA_all, latent_PCA_part), axis=0)
-
-            args.feat_cell = num_features
+    # Run UMAP based on latent representations
+    for name in ['latent_GVAE', 'latent_PCA']:
+        sc.pp.neighbors(adata, n_neighbors=10, use_rep=name)
+        sc.tl.umap(adata)
+        adata.obsm['X_umap_' + name] = adata.obsm['X_umap']
 
-            adata.obsm["latent_GVAE_hierarchy"] = np.array(GVAE_all.loc[adata.obs_names,])
-            adata.obsm["latent_PCA_hierarchy"] = np.array(PCA_all.loc[adata.obs_names,])
-    logger.info(f'------Saving ST data...')
+    # Save the AnnData object
+    logger.info('Saving ST data...')
     adata.write(args.hdf5_with_latent_path)
-

gsMap/format_sumstats.py

@@ -150,10 +150,10 @@ def gwas_checkname(gwas, config):
         'Pos': 'SNP positions.'
     }
 
-    print(f'\nIterpreting column names as follows:')
+    logger.info(f'\nIterpreting column names as follows:')
     for key, value in interpreting.items():
         if key in new_name:
-            print(f'{name_dict[key]}: {interpreting[key]}')
+            logger.info(f'{name_dict[key]}: {interpreting[key]}')
 
     return gwas
 
@@ -242,7 +242,7 @@ def gwas_qc(gwas, config):
     Filter out SNPs based on INFO, FRQ, MAF, N, and Genotypes. 
     '''
     old = len(gwas)
-    print(f'\nFiltering SNPs as follows:')
+    logger.info(f'\nFiltering SNPs as follows:')
     # filter: SNPs with missing values
     drops = {'NA': 0, 'P': 0, 'INFO': 0, 'FRQ': 0, 'A': 0, 'SNP': 0, 'Dup': 0, 'N': 0}
 
@@ -250,28 +250,28 @@ def gwas_qc(gwas, config):
         lambda x: x != 'INFO', gwas.columns)).reset_index(drop=True)
 
     drops['NA'] = old - len(gwas)
-    print(f'Removed {drops["NA"]} SNPs with missing values.')
+    logger.info(f'Removed {drops["NA"]} SNPs with missing values.')
 
     # filter: SNPs with Info < 0.9
     if 'INFO' in gwas.columns:
         old = len(gwas)
         gwas = gwas.loc[filter_info(gwas['INFO'], config)]
         drops['INFO'] = old - len(gwas)
-        print(f'Removed {drops["INFO"]} SNPs with INFO <= 0.9.')
+        logger.info(f'Removed {drops["INFO"]} SNPs with INFO <= 0.9.')
 
     # filter: SNPs with MAF <= 0.01
     if 'FRQ' in gwas.columns:
         old = len(gwas)
         gwas = gwas.loc[filter_frq(gwas['FRQ'], config)]
         drops['FRQ'] += old - len(gwas)
-        print(f'Removed {drops["FRQ"]} SNPs with MAF <= 0.01.')
+        logger.info(f'Removed {drops["FRQ"]} SNPs with MAF <= 0.01.')
 
     # filter: P-value that out-of-bounds [0,1]
     if 'P' in gwas.columns:
         old = len(gwas)
         gwas = gwas.loc[filter_pvals(gwas['P'], config)]
         drops['P'] += old - len(gwas)
-        print(f'Removed {drops["P"]} SNPs with out-of-bounds p-values.')
+        logger.info(f'Removed {drops["P"]} SNPs with out-of-bounds p-values.')
 
     # filter: Variants that are strand-ambiguous
     if 'A1' in gwas.columns and 'A2' in gwas.columns:
@@ -279,21 +279,21 @@ def gwas_qc(gwas, config):
         gwas.A2 = gwas.A2.str.upper()
         gwas = gwas.loc[filter_alleles(gwas.A1 + gwas.A2)]
         drops['A'] += old - len(gwas)
-        print(f'Removed {drops["A"]} variants that were not SNPs or were strand-ambiguous.')
+        logger.info(f'Removed {drops["A"]} variants that were not SNPs or were strand-ambiguous.')
 
     # filter: Duplicated rs numbers
     if 'SNP' in gwas.columns:
         old = len(gwas)
         gwas = gwas.drop_duplicates(subset='SNP').reset_index(drop=True)
         drops['Dup'] += old - len(gwas)
-        print(f'Removed {drops["Dup"]} SNPs with duplicated rs numbers.')
+        logger.info(f'Removed {drops["Dup"]} SNPs with duplicated rs numbers.')
 
     # filter:Sample size
     n_min = gwas.N.quantile(0.9) / 1.5
     old = len(gwas)
     gwas = gwas[gwas.N >= n_min].reset_index(drop=True)
     drops['N'] += old - len(gwas)
-    print(f'Removed {drops["N"]} SNPs with N < {n_min}.')
+    logger.info(f'Removed {drops["N"]} SNPs with N < {n_min}.')
 
     return gwas
 
@@ -302,7 +302,7 @@ def variant_to_rsid(gwas, config):
     '''
     Convert variant id (Chr, Pos) to rsid
     '''
-    print("\nConverting the SNP position to rsid. This process may take some time.")
+    logger.info("\nConverting the SNP position to rsid. This process may take some time.")
     unique_ids = set(gwas['id'])
     chr_format = gwas['Chr'].unique().astype(str)
     chr_format = [re.sub(r'\d+', '', value) for value in chr_format][1]
@@ -347,7 +347,7 @@ def clean_SNP_id(gwas, config):
             gwas = gwas.loc[matching_id.id]
             gwas['SNP'] = matching_id.dbsnp
             num_fail = old - len(gwas)
-            print(f'Removed {num_fail} SNPs that did not convert to rsid.')
+            logger.info(f'Removed {num_fail} SNPs that did not convert to rsid.')
 
     return gwas
 
@@ -356,27 +356,27 @@ def gwas_metadata(gwas, config):
     '''
     Report key features of GWAS data
     '''
-    print('\nMetadata:')
+    logger.info('\nSummary of GWAS data:')
     CHISQ = (gwas.Z ** 2)
     mean_chisq = CHISQ.mean()
-    print('Mean chi^2 = ' + str(round(mean_chisq, 3)))
+    logger.info('Mean chi^2 = ' + str(round(mean_chisq, 3)))
     if mean_chisq < 1.02:
         logger.warning("Mean chi^2 may be too small.")
 
-    print('Lambda GC = ' + str(round(CHISQ.median() / 0.4549, 3)))
-    print('Max chi^2 = ' + str(round(CHISQ.max(), 3)))
-    print('{N} Genome-wide significant SNPs (some may have been removed by filtering).'.format(N=(CHISQ > 29).sum()))
+    logger.info('Lambda GC = ' + str(round(CHISQ.median() / 0.4549, 3)))
+    logger.info('Max chi^2 = ' + str(round(CHISQ.max(), 3)))
+    logger.info('{N} Genome-wide significant SNPs (some may have been removed by filtering).'.format(N=(CHISQ > 29).sum()))
 
 
 def gwas_format(config: FormatSumstatsConfig):
     '''
     Format GWAS data
     '''
-    print(f'------Formating gwas data for {config.sumstats}...')
+    logger.info(f'------Formating gwas data for {config.sumstats}...')
     compression_type = get_compression(config.sumstats)
     gwas = pd.read_csv(config.sumstats, delim_whitespace=True, header=0, compression=compression_type,
                        na_values=['.', 'NA'])
-    print(f'Read {len(gwas)} SNPs from {config.sumstats}.')
+    logger.info(f'Read {len(gwas)} SNPs from {config.sumstats}.')
 
     # Check name and format
     gwas = gwas_checkname(gwas, config)
@@ -402,6 +402,6 @@ def gwas_format(config: FormatSumstatsConfig):
     gwas = gwas[keep]
     out_name = config.out + appendix + '.gz'
 
-    print(f'\nWriting summary statistics for {len(gwas)} SNPs to {out_name}.')
+    logger.info(f'\nWriting summary statistics for {len(gwas)} SNPs to {out_name}.')
     gwas.to_csv(out_name, sep="\t", index=False,
                 float_format='%.3f', compression='gzip')