gsMap 1.64__tar.gz → 1.66__tar.gz

{gsmap-1.64 → gsmap-1.66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gsMap
-Version: 1.64
+Version: 1.66
 Summary: Genetics-informed pathogenic spatial mapping
 Author-email: liyang <songliyang@westlake.edu.cn>, wenhao <chenwenhao@westlake.edu.cn>
 Requires-Python: >=3.8

{gsmap-1.64 → gsmap-1.66}/src/gsMap/__init__.py

@@ -2,4 +2,4 @@
 Genetics-informed pathogenic spatial mapping
 '''
 
-__version__ = '1.64'
+__version__ = '1.66'

gsmap-1.66/src/gsMap/latent_to_gene.py

@@ -0,0 +1,241 @@
+import logging
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import scanpy as sc
+from scipy.sparse import csr_matrix
+from scipy.stats import gmean
+from scipy.stats import rankdata
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.neighbors import NearestNeighbors
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+from gsMap.config import LatentToGeneConfig
+
+logger = logging.getLogger(__name__)
+
+
+def find_neighbors(coor, num_neighbour):
+    """
+    Find Neighbors of each cell (based on spatial coordinates).
+    """
+    nbrs = NearestNeighbors(n_neighbors=num_neighbour).fit(coor)
+    distances, indices = nbrs.kneighbors(coor, return_distance=True)
+    cell_indices = np.arange(coor.shape[0])
+    cell1 = np.repeat(cell_indices, indices.shape[1])
+    cell2 = indices.flatten()
+    distance = distances.flatten()
+    spatial_net = pd.DataFrame({'Cell1': cell1, 'Cell2': cell2, 'Distance': distance})
+    return spatial_net
+
+
+def build_spatial_net(adata, annotation, num_neighbour):
+    """
+    Build spatial neighbourhood matrix for each spot (cell) based on the spatial coordinates.
+    """
+    logger.info(f'------Building spatial graph based on spatial coordinates...')
+
+    coor = adata.obsm['spatial']
+    if annotation is not None:
+        logger.info(f'Cell annotations are provided...')
+        spatial_net_list = []
+        # Cells with annotations
+        for ct in adata.obs[annotation].dropna().unique():
+            idx = np.where(adata.obs[annotation] == ct)[0]
+            coor_temp = coor[idx, :]
+            spatial_net_temp = find_neighbors(coor_temp, min(num_neighbour, coor_temp.shape[0]))
+            # Map back to original indices
+            spatial_net_temp['Cell1'] = idx[spatial_net_temp['Cell1'].values]
+            spatial_net_temp['Cell2'] = idx[spatial_net_temp['Cell2'].values]
+            spatial_net_list.append(spatial_net_temp)
+            logger.info(f'{ct}: {coor_temp.shape[0]} cells')
+
+        # Cells labeled as nan
+        if pd.isnull(adata.obs[annotation]).any():
+            idx_nan = np.where(pd.isnull(adata.obs[annotation]))[0]
+            logger.info(f'Nan: {len(idx_nan)} cells')
+            spatial_net_temp = find_neighbors(coor, num_neighbour)
+            spatial_net_temp = spatial_net_temp[spatial_net_temp['Cell1'].isin(idx_nan)]
+            spatial_net_list.append(spatial_net_temp)
+        spatial_net = pd.concat(spatial_net_list, axis=0)
+    else:
+        logger.info(f'Cell annotations are not provided...')
+        spatial_net = find_neighbors(coor, num_neighbour)
+
+    return spatial_net
+
+
+def find_neighbors_regional(cell_pos, spatial_net_dict, coor_latent, config, cell_annotations):
+    num_neighbour = config.num_neighbour
+    annotations = config.annotation
+
+    cell_use_pos = spatial_net_dict.get(cell_pos, [])
+    if len(cell_use_pos) == 0:
+        return []
+
+    cell_latent = coor_latent[cell_pos, :].reshape(1, -1)
+    neighbors_latent = coor_latent[cell_use_pos, :]
+    similarity = cosine_similarity(cell_latent, neighbors_latent).reshape(-1)
+
+    if annotations is not None:
+        cell_annotation = cell_annotations[cell_pos]
+        neighbor_annotations = cell_annotations[cell_use_pos]
+        mask = neighbor_annotations == cell_annotation
+        if not np.any(mask):
+            return []
+        similarity = similarity[mask]
+        cell_use_pos = cell_use_pos[mask]
+
+    if len(similarity) == 0:
+        return []
+
+    indices = np.argsort(-similarity)  # descending order
+    top_indices = indices[:num_neighbour]
+    cell_select_pos = cell_use_pos[top_indices]
+    return cell_select_pos
+
+
+def compute_regional_mkscore(cell_pos, spatial_net_dict, coor_latent, config, cell_annotations,
+                             ranks, frac_whole, adata_X_bool):
+    """
+    Compute gmean ranks of a region.
+    """
+    cell_select_pos = find_neighbors_regional(
+        cell_pos, spatial_net_dict, coor_latent, config, cell_annotations
+    )
+    if len(cell_select_pos) == 0:
+        return np.zeros(ranks.shape[1], dtype=np.float16)
+
+    # Ratio of expression ranks
+    ranks_tg = ranks[cell_select_pos, :]
+    gene_ranks_region = gmean(ranks_tg, axis=0)
+    gene_ranks_region[gene_ranks_region <= 1] = 0
+
+    if not config.no_expression_fraction:
+        # Ratio of expression fractions
+        frac_focal = adata_X_bool[cell_select_pos, :].sum(axis=0).A1 / len(cell_select_pos)
+        frac_region = frac_focal / frac_whole
+        frac_region[frac_region <= 1] = 0
+        frac_region[frac_region > 1] = 1
+
+        # Simultaneously consider the ratio of expression fractions and ranks
+        gene_ranks_region = gene_ranks_region * frac_region
+
+    mkscore = np.exp(gene_ranks_region ** 1.5) - 1
+    return mkscore.astype(np.float16, copy=False)
+
+
+def run_latent_to_gene(config: LatentToGeneConfig):
+    logger.info('------Loading the spatial data...')
+    adata = sc.read_h5ad(config.hdf5_with_latent_path)
+
+    if config.annotation is not None:
+        logger.info(f'------Cell annotations are provided as {config.annotation}...')
+        adata = adata[~pd.isnull(adata.obs[config.annotation]), :]
+
+    # Homologs transformation
+    if config.homolog_file is not None:
+        logger.info(f'------Transforming the {config.species} to HUMAN_GENE_SYM...')
+        homologs = pd.read_csv(config.homolog_file, sep='\t')
+        if homologs.shape[1] != 2:
+            raise ValueError(
+                "Homologs file must have two columns: one for the species and one for the human gene symbol.")
+
+        homologs.columns = [config.species, 'HUMAN_GENE_SYM']
+        homologs.set_index(config.species, inplace=True)
+        adata = adata[:, adata.var_names.isin(homologs.index)]
+        logger.info(f"{adata.shape[1]} genes retained after homolog transformation.")
+        if adata.shape[1] < 100:
+            raise ValueError("Too few genes retained in ST data (<100).")
+        adata.var_names = homologs.loc[adata.var_names, 'HUMAN_GENE_SYM'].values
+        adata = adata[:, ~adata.var_names.duplicated()]
+
+    # Remove cells and genes that are not expressed
+    logger.info(f'Number of cells, genes of the input data: {adata.shape[0]},{adata.shape[1]}')
+    adata = adata[adata.X.sum(axis=1) > 0, adata.X.sum(axis=0) > 0]
+    logger.info(f'Number of cells, genes after transformation: {adata.shape[0]},{adata.shape[1]}')
+
+    # Create mappings
+    n_cells = adata.n_obs
+    n_genes = adata.n_vars
+
+    if config.annotation is not None:
+        cell_annotations = adata.obs[config.annotation].values
+    else:
+        cell_annotations = None
+
+    # Build the spatial graph
+    spatial_net = build_spatial_net(adata, config.annotation, config.num_neighbour_spatial)
+    spatial_net_dict = spatial_net.groupby('Cell1')['Cell2'].apply(np.array).to_dict()
+
+    # Extract the latent representation
+    coor_latent = adata.obsm[config.latent_representation]
+    coor_latent = coor_latent.astype(np.float32)
+
+    # Compute ranks
+    logger.info('------Ranking the spatial data...')
+    adata_X = adata.X.tocsr()
+    ranks = np.zeros((n_cells, n_genes), dtype=np.float32)
+
+    for i in tqdm(range(n_cells), desc="Computing ranks per cell"):
+        data = adata_X[i, :].toarray().flatten()
+        ranks[i, :] = rankdata(data, method='average')
+
+    # Geometric mean across slices
+    if config.gM_slices is not None:
+        logger.info('Geometrical mean across multiple slices is provided.')
+        gM_df = pd.read_parquet(config.gM_slices)
+        if config.species is not None:
+            homologs = pd.read_csv(config.homolog_file, sep='\t', header=None)
+            if homologs.shape[1] < 2:
+                raise ValueError(
+                    "Homologs file must have at least two columns: one for the species and one for the human gene symbol.")
+            homologs.columns = [config.species, 'HUMAN_GENE_SYM']
+            homologs.set_index(config.species, inplace=True)
+            gM_df = gM_df.loc[gM_df.index.isin(homologs.index)]
+            gM_df.index = homologs.loc[gM_df.index, 'HUMAN_GENE_SYM'].values
+        common_genes = np.intersect1d(adata.var_names, gM_df.index)
+        gM_df = gM_df.loc[common_genes]
+        gM = gM_df['G_Mean'].values
+        adata = adata[:, common_genes]
+        ranks = ranks[:, np.isin(adata.var_names, common_genes)]
+    else:
+        gM = gmean(ranks, axis=0)
+
+    # Compute the fraction of each gene across cells
+    adata_X_bool = adata_X.astype(bool)
+    frac_whole = np.asarray(adata_X_bool.sum(axis=0)).flatten() / n_cells
+
+    # Normalize the ranks
+    ranks = ranks / gM
+
+    # Compute marker scores in parallel
+    logger.info('------Computing marker scores...')
+
+    def compute_mk_score_wrapper(cell_pos):
+        return compute_regional_mkscore(
+            cell_pos, spatial_net_dict, coor_latent, config, cell_annotations, ranks, frac_whole, adata_X_bool
+        )
+
+    mk_scores = [compute_mk_score_wrapper(cell_pos) for cell_pos in tqdm(range(n_cells), desc="Calculating marker scores")]
+    mk_score = np.vstack(mk_scores).T
+
+    # Remove mitochondrial genes
+    gene_names = adata.var_names.values.astype(str)
+    mt_gene_mask = ~(np.char.startswith(gene_names, 'MT-') | np.char.startswith(gene_names, 'mt-'))
+    mk_score = mk_score[mt_gene_mask, :]
+    gene_names = gene_names[mt_gene_mask]
+
+    # Save the marker scores
+    logger.info(f'------Saving marker scores ...')
+    output_file_path = Path(config.mkscore_feather_path)
+    output_file_path.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
+    mk_score_df = pd.DataFrame(mk_score, index=gene_names, columns=adata.obs_names)
+    mk_score_df.reset_index(inplace=True)
+    mk_score_df.rename(columns={'index': 'HUMAN_GENE_SYM'}, inplace=True)
+    mk_score_df.to_feather(output_file_path)
+
+    # Save the modified adata object to disk
+    adata.write(config.hdf5_with_latent_path)

gsmap-1.66/src/gsMap/setup.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+import setuptools
+
+if __name__ == "__main__":
+    setuptools.setup(name='gsMap')

gsmap-1.64/src/gsMap/latent_to_gene.py

@@ -1,218 +0,0 @@
-import logging
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-import scanpy as sc
-from scipy.stats import gmean
-from scipy.stats import rankdata
-from sklearn.metrics.pairwise import cosine_similarity
-from sklearn.neighbors import NearestNeighbors
-from tqdm import tqdm
-
-from gsMap.config import LatentToGeneConfig
-
-logger = logging.getLogger(__name__)
-
-
-def find_Neighbors(coor, num_neighbour):
-    """
-    find Neighbors of each cell (based on spatial coordinates)
-    """
-    nbrs = NearestNeighbors(n_neighbors=num_neighbour).fit(coor)
-    distances, indices = nbrs.kneighbors(coor, return_distance=True)
-
-    KNN_list = [pd.DataFrame(zip([it] * indices[it].shape[0], indices[it], distances[it])) for it in
-                range(indices.shape[0])]
-    KNN_df = pd.concat(KNN_list)
-    KNN_df.columns = ['Cell1', 'Cell2', 'Distance']
-
-    spatial_net = KNN_df.copy()
-    id_cell_trans = dict(zip(range(coor.shape[0]), np.array(coor.index)))
-
-    spatial_net['Cell1'] = spatial_net['Cell1'].map(id_cell_trans)
-    spatial_net['Cell2'] = spatial_net['Cell2'].map(id_cell_trans)
-
-    return spatial_net
-
-
-def _build_spatial_net(adata, annotation, num_neighbour):
-    """
-    1 Build spatial neighbourhood matrix for each spot (cell) based on the spatial coord
-    """
-    logger.info(f'------Building spatial graph based on spatial coordinates...')
-
-    coor = pd.DataFrame(adata.obsm['spatial'])
-    coor.index = adata.obs.index
-
-    if not annotation is None:
-        logger.info(f'Cell annotations are provided...')
-        spatial_net = pd.DataFrame()
-        # Cells with annotations
-        for ct in adata.obs[annotation].dropna().unique():
-            coor_temp = coor.loc[adata.obs[annotation] == ct, :]
-            spatial_net_temp = find_Neighbors(coor_temp, min(num_neighbour, coor_temp.shape[0]))
-            spatial_net = pd.concat((spatial_net, spatial_net_temp), axis=0)
-            logger.info(f'{ct}: {coor_temp.shape[0]} cells')
-
-        # Cells labeled as nan
-        if pd.isnull(adata.obs[annotation]).any():
-            cell_nan = adata.obs.index[np.where(pd.isnull(adata.obs[annotation]))[0]]
-            logger.info(f'Nan: {len(cell_nan)} cells')
-
-            spatial_net_temp = find_Neighbors(coor, num_neighbour)
-            spatial_net_temp = spatial_net_temp.loc[spatial_net_temp.Cell1.isin(cell_nan), :]
-            spatial_net = pd.concat((spatial_net, spatial_net_temp), axis=0)
-    else:
-        logger.info(f'Cell annotations are not provided...')
-        spatial_net = find_Neighbors(coor, num_neighbour)
-
-    return spatial_net
-
-
-def find_Neighbors_Regional(cell):
-    cell_use = spatial_net_dict[cell]
-    similarity = cosine_similarity(coor_latent.loc[cell].values.reshape(1, -1),
-                                   coor_latent.loc[cell_use].values).reshape(-1)
-    if not args.annotation is None:
-        annotation = adata.obs[args.annotation]
-        df = pd.DataFrame({'Cell2': cell_use, 'Similarity': similarity, 'Annotation': annotation[cell_use]})
-        df = df.loc[df.loc[cell_use, 'Annotation'] == df.loc[cell, 'Annotation']]
-    else:
-        df = pd.DataFrame({'Cell2': cell_use, 'Similarity': similarity})
-
-    df = df.sort_values(by='Similarity', ascending=False)
-    cell_select = df.Cell2[0:args.num_neighbour].to_list()
-
-    return cell_select
-
-
-def _compute_regional_mkscore(cell_tg, ):
-    """
-    compute gmean ranks of a region
-    """
-    cell_select = find_Neighbors_Regional(cell_tg)
-
-    # Ratio of expression ranks
-    ranks_tg = ranks.loc[cell_select]
-    gene_ranks_region = gmean(ranks_tg, axis=0)
-    gene_ranks_region[gene_ranks_region <= 1] = 0
-
-    if not args.no_expression_fraction:
-        # Ratio of expression fractions
-        frac_focal = expressed_mask.loc[cell_select].sum(0) / len(cell_select)
-        frac_region = frac_focal / frac_whole
-        frac_region[frac_region <= 1] = 0
-        frac_region[frac_region > 1] = 1
-
-        # Simultaneously consider the ratio of expression fractions and ranks
-        gene_ranks_region = (gene_ranks_region * frac_region).values
-
-    mkscore = np.exp(gene_ranks_region ** 1.5) - 1
-    return mkscore.astype(np.float16, copy=False)
-
-
-def run_latent_to_gene(config: LatentToGeneConfig):
-    global adata, coor_latent, spatial_net, ranks, frac_whole, args, spatial_net_dict, expressed_mask
-    args = config
-    # Load and process the spatial data
-    logger.info('------Loading the spatial data...')
-    adata = sc.read_h5ad(config.hdf5_with_latent_path)
-
-    logger.info('------Ranking the spatial data...')
-    adata.layers['rank'] = rankdata(adata.X.toarray().astype(np.float32), axis=1).astype(np.float32)
-
-    if not config.annotation is None:
-        logger.info(f'------Cell annotations are provided as {config.annotation}...')
-        adata = adata[~pd.isnull(adata.obs[config.annotation]), :]
-
-    # Homologs transformation
-    if not config.homolog_file is None:
-        logger.info(f'------Transforming the {config.species} to HUMAN_GENE_SYM...')
-        homologs = pd.read_csv(config.homolog_file, sep='\t')
-        if homologs.shape[1] != 2:
-            raise ValueError(
-                "Homologs file must have two columns: one for the species and one for the human gene symbol.")
-
-        homologs.columns = [config.species, 'HUMAN_GENE_SYM']
-        homologs.set_index(config.species, inplace=True)
-        adata = adata[:, adata.var_names.isin(homologs.index)]
-        # Log the number of genes left after homolog transformation
-        logger.info(f"{adata.shape[1]} genes retained after homolog transformation.")
-        if adata.shape[1] < 100:
-            raise ValueError("Too few genes retained in ST data (<100).")
-        adata.var_names = homologs.loc[adata.var_names, 'HUMAN_GENE_SYM'].values
-        # drop duplicated genes
-        adata = adata[:, ~adata.var_names.duplicated()]
-
-    # Remove cells that do not express any genes after transformation, and genes that are not expressed in any cells.
-    logger.info(f'Number of cells, genes of the input data: {adata.shape[0]},{adata.shape[1]}')
-    adata = adata[adata.X.sum(axis=1) > 0, adata.X.sum(axis=0) > 0]
-    logger.info(f'Number of cells, genes after transformation: {adata.shape[0]},{adata.shape[1]}')
-    # Buid the spatial graph
-    spatial_net = _build_spatial_net(adata, config.annotation, config.num_neighbour_spatial)
-    spatial_net.set_index('Cell1', inplace=True)
-    # convert the spatial graph to a dictionary cell1 to cells in the neighbourhood
-    spatial_net_dict = spatial_net.groupby(spatial_net.index).Cell2.apply(list).to_dict()
-
-    # Extract the latent representation
-    coor_latent = pd.DataFrame(adata.obsm[config.latent_representation])
-    coor_latent.index = adata.obs.index
-    # Find marker genes
-    cell_list = adata.obs.index.tolist()
-
-    # Load the geometrical mean across slices
-    if config.gM_slices is not None:
-        logger.info('Geometrical mean across multiple slices is provided.')
-        gM = pd.read_parquet(config.gM_slices)
-        if config.species is not None:
-            homologs = pd.read_csv(config.homolog_file, sep='\t', header=None)
-            if homologs.shape[1] < 2:
-                raise ValueError(
-                    "Homologs file must have at least two columns: one for the species and one for the human gene symbol.")
-            homologs.columns = [config.species, 'HUMAN_GENE_SYM']
-            homologs.set_index(config.species, inplace=True)
-            gM = gM.loc[gM.index.isin(homologs.index)]
-            gM.index = homologs.loc[gM.index, 'HUMAN_GENE_SYM'].values
-        common_gene = np.intersect1d(adata.var_names, gM.index)
-        gM = gM.loc[common_gene]
-        gM = gM['G_Mean'].to_numpy()
-        adata = adata[:, common_gene]
-    else:
-        gM = gmean(adata.layers['rank'], axis=0)
-
-    # Compute the fraction of each gene across cells
-    expressed_mask = pd.DataFrame((adata.X > 0).toarray(), index=adata.obs.index, columns=adata.var.index)
-    # frac_whole = np.array((adata_layer > 0).sum(axis=0))[0] / (adata.shape[0])
-    frac_whole = np.array(expressed_mask.sum(axis=0)) / (adata.shape[0])
-    # Normalize the geometrical mean
-    ranks = adata.layers['rank'] / gM
-    ranks = pd.DataFrame(ranks, index=adata.obs_names)
-    ranks.columns = adata.var.index
-    mk_score = [
-        _compute_regional_mkscore(cell_tg)
-        for cell_tg in tqdm(cell_list,
-                            desc="Finding markers (Rank-based approach) | cells")
-    ]
-    # Normalize the marker scores
-    mk_score = pd.DataFrame(np.vstack(mk_score).T, index=adata.var_names, columns=cell_list)
-    # mk_score_normalized = mk_score.div(mk_score.sum())*1e+2
-
-    # Remove the mitochondrial genes from mk_score
-    mt_gene_mask = ~adata.var_names.str.startswith(('MT-', 'mt-'))
-    mk_score = mk_score[mt_gene_mask]
-    adata = adata[:, mt_gene_mask]
-
-    # # Save the mk_score DataFrame to an adata layer
-    # adata.layers['mkscore'] = mk_score.values.T
-
-    # Save the marker scores
-    logger.info(f'------Saving marker scores ...')
-    output_file_path = Path(config.mkscore_feather_path)
-    output_file_path.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
-    mk_score.reset_index(inplace=True)
-    mk_score.rename(columns={mk_score.columns[0]: 'HUMAN_GENE_SYM'}, inplace=True)
-    mk_score.to_feather(output_file_path)
-
-    # Save the modified adata object to disk
-    adata.write(config.hdf5_with_latent_path)

gsmap-1.64/test/GPS-snakemake-workflow-macaque.smk

@@ -1,268 +0,0 @@
-from pathlib import Path
-
-import numpy as np
-
-workdir: '/storage/yangjianLab/chenwenhao/projects/202312_GPS/data/macaque/processed'
-# workdir: '/storage/yangjianLab/chenwenhao/projects/202312_GPS/data/GPS_test/macaque'
-sample_name = "Cortex_151507"
-chrom = "all"
-QOS = "huge"
-# chrom = range(1,23)
-trait_names = [
-    'PGC3_SCZ_wave3_public_INFO80'
-]
-root = "/storage/yangjianLab/songliyang/SpatialData/Data/Brain/macaque/Cell/processed/h5ad"
-# sample_names = [file.strip().split('.')[0]
-#                 for file in open(f'{root}/representative_slices2').readlines()]
-#
-# sample_names = '''
-# T33_macaque1  T44_macaque1  T82_macaque1  T97_macaque1  T125_macaque1  T127_macaque1  T129_macaque1  T131_macaque1  T135_macaque1  T137_macaque1  T139_macaque1
-# '''.strip().split()
-sample_names=[]
-for file in Path(root).glob('*.h5ad'):
-    sample_names.append(file.stem)
-sample_names.remove('T825_macaque3') # due to 25% of spot don't have spatial coordinates
-
-annotation = "SubClass"
-data_type = "SCT"
-# sample_names = ['T584_macaque2']
-
-rule all:
-    input:
-        expand('{sample_name}/spatial_ldsc/{sample_name}.spatial_ldsc.done',trait_name=trait_names,sample_name=sample_names)
-
-
-# expand('{sample_name}/cauchy_combination/{sample_name}_{trait_name}.Cauchy.csv.gz',trait_name=trait_names,sample_name=sample_names)
-    # expand('{sample_name}/cauchy_combination/{sample_name}_{trait_name}.Cauchy.csv.gz',trait_name=trait_names,sample_name=sample_names)
-
-rule test_run:
-    input:
-        [f'{sample_name}/generate_ldscore/{sample_name}_generate_ldscore_chr{chrom}.done' for sample_name in
-         sample_names]
-
-# localrules: find_latent_representations,latent_to_gene
-def get_annotation(wildcards):
-    if wildcards.sample_name.endswith('3'):
-        print(wildcards.sample_name,'will use None as annotation')
-        return None
-    else:
-        print(wildcards.sample_name,'will use SubClass as annotation')
-        return 'SubClass'
-
-
-rule find_latent_representations:
-    input:
-        hdf5_path=f'{root}/{{sample_name}}.h5ad'
-    output:
-        hdf5_output='{sample_name}/find_latent_representations/{sample_name}_add_latent.h5ad'
-    params:
-        annotation= get_annotation,
-        type=data_type,
-        epochs=300,
-        feat_hidden1=256,
-        feat_hidden2=128,
-        feat_cell=3000,
-        gcn_hidden1=64,
-        gcn_hidden2=30,
-        p_drop=0.1,
-        gcn_lr=0.001,
-        gcn_decay=0.01,
-        n_neighbors=11,
-        label_w=1,
-        rec_w=1,
-        n_comps=300,
-        weighted_adj=False,
-        nheads=3,
-        var=False,
-        convergence_threshold=1e-4,
-        hierarchically=False
-    threads:
-        3
-    benchmark: '{sample_name}/find_latent_representations/{sample_name}_add_latent.h5ad.benchmark'
-    resources:
-        mem_mb_per_cpu=lambda wildcards, threads, attempt: 20_000 * np.log2(attempt + 1),
-        qos=QOS
-    run:
-        command = f"""
-gsmap run_find_latent_representations \
-    --input_hdf5_path {input.hdf5_path} \
-    --sample_name {wildcards.sample_name} \
-    --output_hdf5_path {output.hdf5_output} \
-    { '--annotation ' + params.annotation if params.annotation is not None else ''} \
-    --type {params.type} \
-    --epochs {params.epochs} \
-    --feat_hidden1 {params.feat_hidden1} \
-    --feat_hidden2 {params.feat_hidden2} \
-    --feat_cell {params.feat_cell} \
-    --gcn_hidden1 {params.gcn_hidden1} \
-    --gcn_hidden2 {params.gcn_hidden2} \
-    --p_drop {params.p_drop} \
-    --gcn_lr {params.gcn_lr} \
-    --gcn_decay {params.gcn_decay} \
-    --n_neighbors {params.n_neighbors} \
-    --label_w {params.label_w} \
-    --rec_w {params.rec_w} \
-    --n_comps {params.n_comps} \
-    {'--weighted_adj' if params.weighted_adj else ''} \
-    --nheads {params.nheads} \
-    {'--var' if params.var else ''} \
-    --convergence_threshold {params.convergence_threshold} \
-    {'--hierarchically' if params.hierarchically else ''}
-        """
-        shell(
-            f'{command}'
-        )
-
-
-rule latent_to_gene:
-    input:
-        hdf5_with_latent_path=rules.find_latent_representations.output.hdf5_output
-    output:
-        feather_path='{sample_name}/latent_to_gene/{sample_name}_gene_marker_score.feather'
-    params:
-        latent_representation="latent_GVAE",
-        num_neighbour=51,
-        num_neighbour_spatial=201,
-        species='MACAQUE_GENE_SYM',
-        gs_species='/storage/yangjianLab/songliyang/SpatialData/homologs/macaque_human_homologs.txt',
-        gM_slices=None,
-        annotation=get_annotation,
-        type=data_type
-    threads:
-        1
-    resources:
-        mem_mb_per_cpu=lambda wildcards, threads, attempt: 70_000 * np.log2(attempt + 1),
-        qos=QOS
-    benchmark: '{sample_name}/latent_to_gene/{sample_name}_gene_marker_score.feather.benchmark'
-    run:
-        command = f"""
-gsmap run_latent_to_gene \
-    --input_hdf5_with_latent_path {input.hdf5_with_latent_path} \
-    --sample_name {wildcards.sample_name} \
-    --output_feather_path {output.feather_path} \
-    { '--annotation ' + params.annotation if params.annotation is not None else ''} \
-    --type {params.type} \
-    --latent_representation {params.latent_representation} \
-    --num_neighbour {params.num_neighbour} \
-    --num_neighbour_spatial {params.num_neighbour_spatial} \
-     {'--species ' + params.species if params.species is not None else ''} \
-     {'--gs_species ' + params.gs_species if params.gs_species is not None else ''} \
-     {'--gM_slices ' + params.gM_slices if params.gM_slices is not None else ''}
-"""
-        shell(
-            f'{command}'
-        )
-
-
-rule generate_ldscore:
-    input:
-        mkscore_feather_file=rules.latent_to_gene.output.feather_path
-    output:
-        done='{sample_name}/generate_ldscore/{sample_name}_generate_ldscore_chr{chrom}.done'
-    params:
-        ld_score_save_dir='{sample_name}/generate_ldscore',
-        gtf_annotation_file="/storage/yangjianLab/songliyang/ReferenceGenome/GRCh37/gencode.v39lift37.annotation.gtf",
-        bfile_root="/storage/yangjianLab/sharedata/LDSC_resource/1000G_EUR_Phase3_plink/1000G.EUR.QC",
-        keep_snp_root="/storage/yangjianLab/sharedata/LDSC_resource/hapmap3_snps/hm",
-        gene_window_size=50000,
-        enhancer_annotation_file=None,
-        snp_multiple_enhancer_strategy='max_mkscore',
-        gene_window_enhancer_priority=None,
-        spots_per_chunk=1000,
-        ld_wind=1,
-        ld_unit="CM",
-        additional_baseline_annotation_dir_path=None
-        # additional_baseline_annotation_dir_path='/storage/yangjianLab/chenwenhao/projects/202312_GPS/data/resource/ldsc/baseline_v1.2/remove_base'
-    benchmark: '{sample_name}/generate_ldscore/{sample_name}_generate_ldscore_chr{chrom}.done.benchmark'
-    threads:
-        3
-    resources:
-        mem_mb_per_cpu=lambda wildcards, threads, attempt: 50_000 / threads * np.log2(attempt + 1),
-        qos=QOS
-    run:
-        command = f"""
-        gsmap run_generate_ldscore \
-            --sample_name {wildcards.sample_name} \
-            --chrom {wildcards.chrom} \
-            --ldscore_save_dir {params.ld_score_save_dir} \
-            --mkscore_feather_file {input.mkscore_feather_file} \
-            --bfile_root {params.bfile_root} \
-            --keep_snp_root {params.keep_snp_root} \
-            --gtf_annotation_file {params.gtf_annotation_file} \
-            --gene_window_size {params.gene_window_size} \
-            {'--enhancer_annotation_file ' + params.enhancer_annotation_file if params.enhancer_annotation_file is not None else ''} \
-            --snp_multiple_enhancer_strategy {params.snp_multiple_enhancer_strategy} \
-            {'--gene_window_enhancer_priority ' + params.gene_window_enhancer_priority if params.gene_window_enhancer_priority is not None else ''} \
-            --spots_per_chunk {params.spots_per_chunk} \
-            --ld_wind {params.ld_wind} \
-            --ld_unit {params.ld_unit} \
-            { '--additional_baseline_annotation_dir_path ' +  params.additional_baseline_annotation_dir_path if params.additional_baseline_annotation_dir_path is not None else '' }
-        """
-        shell(command)
-        shell('touch {output.done}')
-
-
-def get_h2_file(wildcards):
-    gwas_root = "/storage/yangjianLab/songliyang/GWAS_trait/LDSC"
-    return f"{gwas_root}/{wildcards.trait_name}.sumstats.gz",
-
-
-def get_ldscore(wildcards):
-    if chrom == "all":
-        return f"{wildcards.sample_name}/generate_ldscore/{wildcards.sample_name}_generate_ldscore_chr{chrom}.done"
-    else:
-        assert tuple(chrom) == tuple(range(1,23)), "chrom must be all or range(1,23)"
-        return [f"{wildcards.sample_name}/generate_ldscore/{wildcards.sample_name}_generate_ldscore_chr{c}.done" for
-                c in chrom]
-
-
-rule spatial_ldsc:
-    input:
-        # h2_file=get_h2_file,
-        generate_ldscore_done=get_ldscore
-    output:
-        done='{sample_name}/spatial_ldsc/{sample_name}.spatial_ldsc.done'
-    params:
-        ldscore_input_dir=rules.generate_ldscore.params.ld_score_save_dir,
-        ldsc_save_dir='{sample_name}/spatial_ldsc',
-        w_file="/storage/yangjianLab/sharedata/LDSC_resource/LDSC_SEG_ldscores/weights_hm3_no_hla/weights.",
-        sumstats_config_file='/storage/yangjianLab/chenwenhao/projects/202312_GPS/src/gsMap/example/sumstats_config_sub.yaml',
-        all_chunk = None
-    threads:
-        10
-    benchmark:
-        '{sample_name}/spatial_ldsc/{sample_name}.spatial_ldsc.done.benchmark'
-    resources:
-        mem_mb_per_cpu=lambda wildcards, threads, attempt: 40_000 / threads * np.log2(attempt + 1),
-        qos=QOS,
-        partition='intel-sc3,amd-ep2'
-    run:
-       command = f"""
-        gsmap run_spatial_ldsc --w_file {params.w_file} --sample_name {wildcards.sample_name} --num_processes {threads} --ldscore_input_dir {params.ldscore_input_dir} --ldsc_save_dir {params.ldsc_save_dir} --sumstats_config_file {params.sumstats_config_file} {f'--all_chunk {params.all_chunk}' if params.all_chunk else ''}
-        """
-       shell(
-           f'{command}'
-           'touch {output.done}'
-       )
-
-
-rule cauchy_combination:
-    output:
-        done='{sample_name}/cauchy_combination/{sample_name}_{trait_name}.Cauchy.csv.gz'
-    input:
-        hdf5_path=rules.find_latent_representations.output.hdf5_output,
-        ldsc_done=rules.spatial_ldsc.output.done
-    params:
-        cauchy_save_dir='{sample_name}/cauchy_combination',
-        annotation=annotation,
-        ldsc_dir=rules.spatial_ldsc.params.ldsc_save_dir
-    benchmark:
-        '{sample_name}/cauchy_combination/{sample_name}_{trait_name}.Cauchy.csv.gz.benchmark'
-    threads:
-        2
-    resources:
-        mem_mb_per_cpu=25_000
-    shell:
-        """
-        gsmap run_cauchy_combination --input_hdf5_path {input.hdf5_path} --input_ldsc_dir {params.ldsc_dir} --sample_name {wildcards.sample_name} --output_cauchy_dir {params.cauchy_save_dir} --trait_name {wildcards.trait_name} --annotation {params.annotation}
-        """

gsmap-1.64/test/GPS-snakemake-workflow.smk

@@ -1,229 +0,0 @@
-import numpy as np
-
-workdir: '/storage/yangjianLab/chenwenhao/projects/202312_GPS/data/GPS_test/Nature_Neuroscience_2021/snake_workdir'
-sample_names = ["Cortex_151507"]
-# chrom = "all"
-
-chrom = range(1,23)
-# trait_names=[
-#     'ADULT1_ADULT2_ONSET_ASTHMA'
-# ]
-annotation= "layer_guess"
-data_type = 'count'
-rule all:
-    input:
-        expand('{sample_name}/spatial_ldsc/{sample_name}.spatial_ldsc.done', sample_name=sample_names)
-        # expand('{sample_name}/cauchy_combination/{sample_name}_{trait_name}.Cauchy.csv.gz', trait_name=trait_names, sample_name=sample_names)
-
-rule find_latent_representations:
-    input:
-        hdf5_path = "/storage/yangjianLab/songliyang/SpatialData/Data/Brain/Human/Nature_Neuroscience_2021/processed/h5ad/Cortex_151507.h5ad"
-    output:
-        hdf5_output='{sample_name}/find_latent_representations/{sample_name}_add_latent.h5ad'
-    params:
-        annotation=annotation,
-        type=data_type,
-        epochs=300,
-        feat_hidden1=256,
-        feat_hidden2=128,
-        feat_cell=3000,
-        gcn_hidden1=64,
-        gcn_hidden2=30,
-        p_drop=0.1,
-        gcn_lr=0.001,
-        gcn_decay=0.01,
-        n_neighbors=11,
-        label_w=1,
-        rec_w=1,
-        n_comps=300,
-        weighted_adj=False,
-        nheads=3,
-        var=False,
-        convergence_threshold=1e-4,
-        hierarchically=False
-    threads:
-        1
-    benchmark: '{sample_name}/find_latent_representations/{sample_name}_add_latent.h5ad.benchmark'
-    run:
-        command = f"""
-gsmap run_find_latent_representations \
-    --input_hdf5_path {input.hdf5_path} \
-    --sample_name {wildcards.sample_name} \
-    --output_hdf5_path {output.hdf5_output} \
-    { '--annotation ' + params.annotation if params.annotation is not None else ''} \
-    --type {params.type} \
-    --epochs {params.epochs} \
-    --feat_hidden1 {params.feat_hidden1} \
-    --feat_hidden2 {params.feat_hidden2} \
-    --feat_cell {params.feat_cell} \
-    --gcn_hidden1 {params.gcn_hidden1} \
-    --gcn_hidden2 {params.gcn_hidden2} \
-    --p_drop {params.p_drop} \
-    --gcn_lr {params.gcn_lr} \
-    --gcn_decay {params.gcn_decay} \
-    --n_neighbors {params.n_neighbors} \
-    --label_w {params.label_w} \
-    --rec_w {params.rec_w} \
-    --n_comps {params.n_comps} \
-    {'--weighted_adj' if params.weighted_adj else ''} \
-    --nheads {params.nheads} \
-    {'--var' if params.var else ''} \
-    --convergence_threshold {params.convergence_threshold} \
-    {'--hierarchically' if params.hierarchically else ''}
-        """
-        shell(
-            f'{command}'
-        )
-
-
-rule latent_to_gene:
-    input:
-        hdf5_with_latent_path=rules.find_latent_representations.output.hdf5_output
-    output:
-        feather_path='{sample_name}/latent_to_gene/{sample_name}_gene_marker_score.feather'
-    params:
-        latent_representation="latent_GVAE",
-        num_neighbour=51,
-        num_neighbour_spatial=201,
-        species=None,
-        gs_species=None,
-        gM_slices=None,
-        annotation=annotation,
-        type=data_type
-    threads:
-        1
-    resources:
-        mem_mb_per_cpu=lambda wildcards, threads, attempt: 70_000 * np.log2(attempt + 1),
-        qos='huge'
-    benchmark: '{sample_name}/latent_to_gene/{sample_name}_gene_marker_score.feather.benchmark'
-    run:
-        command = f"""
-gsmap run_latent_to_gene \
-    --input_hdf5_with_latent_path {input.hdf5_with_latent_path} \
-    --sample_name {wildcards.sample_name} \
-    --output_feather_path {output.feather_path} \
-    { '--annotation ' + params.annotation if params.annotation is not None else ''} \
-    --type {params.type} \
-    --latent_representation {params.latent_representation} \
-    --num_neighbour {params.num_neighbour} \
-    --num_neighbour_spatial {params.num_neighbour_spatial} \
-     {'--species ' + params.species if params.species is not None else ''} \
-     {'--gs_species ' + params.gs_species if params.gs_species is not None else ''} \
-     {'--gM_slices ' + params.gM_slices if params.gM_slices is not None else ''}
-"""
-        shell(
-            f'{command}'
-        )
-
-
-
-rule generate_ldscore:
-    input:
-        mkscore_feather_file=rules.latent_to_gene.output.feather_path
-    output:
-        done='{sample_name}/generate_ldscore/{sample_name}_generate_ldscore_chr{chrom}.done'
-    params:
-        ld_score_save_dir='{sample_name}/generate_ldscore',
-        gtf_annotation_file="/storage/yangjianLab/songliyang/ReferenceGenome/GRCh37/gencode.v39lift37.annotation.gtf",
-        bfile_root="/storage/yangjianLab/sharedata/LDSC_resource/1000G_EUR_Phase3_plink/1000G.EUR.QC",
-        keep_snp_root="/storage/yangjianLab/sharedata/LDSC_resource/hapmap3_snps/hm",
-        gene_window_size=50000,
-        enhancer_annotation_file=None,
-        snp_multiple_enhancer_strategy='max_mkscore',
-        gene_window_enhancer_priority=None,
-        spots_per_chunk=5000,
-        ld_wind=1,
-        ld_unit="CM",
-        additional_baseline_annotation_dir_path='/storage/yangjianLab/chenwenhao/projects/202312_GPS/data/resource/ldsc/baseline_v1.2/remove_base'
-    benchmark: '{sample_name}/generate_ldscore/{sample_name}_generate_ldscore_chr{chrom}.done.benchmark'
-    threads:
-        3
-    resources:
-        mem_mb_per_cpu=lambda wildcards, threads, attempt: 45_000 / threads * np.log2(attempt + 1),
-        qos='huge'
-    run:
-        command = f"""
-        gsmap run_generate_ldscore \
-            --sample_name {wildcards.sample_name} \
-            --chrom {wildcards.chrom} \
-            --ldscore_save_dir {params.ld_score_save_dir} \
-            --mkscore_feather_file {input.mkscore_feather_file} \
-            --bfile_root {params.bfile_root} \
-            --keep_snp_root {params.keep_snp_root} \
-            --gtf_annotation_file {params.gtf_annotation_file} \
-            --gene_window_size {params.gene_window_size} \
-            {'--enhancer_annotation_file ' + params.enhancer_annotation_file if params.enhancer_annotation_file is not None else ''} \
-            --snp_multiple_enhancer_strategy {params.snp_multiple_enhancer_strategy} \
-            {'--gene_window_enhancer_priority ' + params.gene_window_enhancer_priority if params.gene_window_enhancer_priority is not None else ''} \
-            --spots_per_chunk {params.spots_per_chunk} \
-            --ld_wind {params.ld_wind} \
-            --ld_unit {params.ld_unit} \
-            { '--additional_baseline_annotation_dir_path' +  params.additional_baseline_annotation_dir_path if params.additional_baseline_annotation_dir_path is not None else '' }
-        """
-        shell(command)
-        shell('touch {output.done}')
-
-
-def get_h2_file(wildcards):
-    gwas_root = "/storage/yangjianLab/songliyang/GWAS_trait/LDSC"
-    return f"{gwas_root}/{wildcards.trait_name}.sumstats.gz",
-
-
-def get_ldscore(wildcards):
-    if chrom == "all":
-        return f"{wildcards.sample_name}/generate_ldscore/{wildcards.sample_name}_generate_ldscore_chr{chrom}.done"
-    else:
-        assert tuple(chrom) == tuple(range(1,23)), "chrom must be all or range(1,23)"
-        return [f"{wildcards.sample_name}/generate_ldscore/{wildcards.sample_name}_generate_ldscore_chr{c}.done" for
-                c in chrom]
-
-
-rule spatial_ldsc:
-    input:
-        # h2_file=get_h2_file,
-        generate_ldscore_done=get_ldscore
-    output:
-        done='{sample_name}/spatial_ldsc/{sample_name}.spatial_ldsc.done'
-    params:
-        ldscore_input_dir=rules.generate_ldscore.params.ld_score_save_dir,
-        ldsc_save_dir='{sample_name}/spatial_ldsc',
-        w_file="/storage/yangjianLab/sharedata/LDSC_resource/LDSC_SEG_ldscores/weights_hm3_no_hla/weights.",
-        sumstats_config_file='/storage/yangjianLab/chenwenhao/projects/202312_GPS/src/gsMap/example/sumstats_config_sub.yaml',
-        all_chunk = None
-    threads:
-        2
-    benchmark:
-        '{sample_name}/spatial_ldsc/{sample_name}.spatial_ldsc.done.benchmark'
-    resources:
-        mem_mb_per_cpu=lambda wildcards, threads, attempt: 60_000 / threads * np.log2(attempt + 1),
-        qos='huge'
-    run:
-       command = f"""
-        gsmap run_spatial_ldsc --w_file {params.w_file} --sample_name {wildcards.sample_name} --num_processes {threads} --ldscore_input_dir {params.ldscore_input_dir} --ldsc_save_dir {params.ldsc_save_dir} --sumstats_config_file {params.sumstats_config_file} {f'--all_chunk {params.all_chunk}' if params.all_chunk else ''}
-        """
-       shell(
-           f'{command}'
-           'touch {output.done}'
-       )
-
-
-rule cauchy_combination:
-    output:
-        done='{sample_name}/cauchy_combination/{sample_name}_{trait_name}.Cauchy.csv.gz'
-    input:
-        hdf5_path=rules.find_latent_representations.output.hdf5_output,
-        ldsc_done=rules.spatial_ldsc.output.done
-    params:
-        cauchy_save_dir='{sample_name}/cauchy_combination',
-        annotation=annotation,
-        ldsc_dir=rules.spatial_ldsc.params.ldsc_save_dir
-    benchmark:
-        '{sample_name}/cauchy_combination/{sample_name}_{trait_name}.Cauchy.csv.gz.benchmark'
-    threads:
-        2
-    resources:
-        mem_mb_per_cpu=25_000
-    shell:
-        """
-        gsmap run_cauchy_combination --input_hdf5_path {input.hdf5_path} --input_ldsc_dir {params.ldsc_dir} --sample_name {wildcards.sample_name} --output_cauchy_dir {params.cauchy_save_dir} --trait_name {wildcards.trait_name} --annotation {params.annotation}
-        """