gsMap 1.62__py3-none-any.whl → 1.63__py3-none-any.whl

gsMap/main.py

@@ -1,14 +1,6 @@
 from gsMap import (__version__)
 from gsMap.config import *
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-handler = logging.StreamHandler()
-handler.setFormatter(logging.Formatter(
-    '[{asctime}] {levelname:8s} {filename} {message}', style='{'))
-logger.addHandler(handler)
-
-
 def main():
     parser = create_parser()
     args = parser.parse_args()
@@ -20,7 +12,7 @@ def main():
     )
 
 def create_parser():
-    parser = argparse.ArgumentParser(description=" gsMap: Genetics-informed pathogenic spatial mapping",
+    parser = argparse.ArgumentParser(description=" gsMap: genetically informed spatial mapping of cells for complex traits",
                                      formatter_class=argparse.RawTextHelpFormatter,
                                      prog='gsMap'
                                      )

gsMap/report.py

@@ -0,0 +1,160 @@
+import logging
+import os
+import shutil
+
+import pandas as pd
+from jinja2 import Environment, FileSystemLoader
+
+import gsMap
+from gsMap.cauchy_combination_test import run_Cauchy_combination
+from gsMap.config import CauchyCombinationConfig, ReportConfig
+from gsMap.diagnosis import run_Diagnosis
+
+logger = logging.getLogger(__name__)
+
+# Load the Jinja2 environment
+try:
+    from importlib.resources import files
+
+    template_dir = files('gsMap').joinpath('templates')
+except (ImportError, FileNotFoundError):
+    # Fallback to a relative path if running in development mode
+    template_dir = os.path.join(os.path.dirname(__file__), 'templates')
+
+# Set up Jinja2 environment
+env = Environment(loader=FileSystemLoader(template_dir))
+
+# Load the template
+template = env.get_template('report_template.html')
+
+def copy_files_to_report_dir(result_dir, report_dir, files_to_copy):
+    """Copy specified files (HTML or PNG) to the report directory."""
+    os.makedirs(report_dir, exist_ok=True)
+    for file in files_to_copy:
+        shutil.copy2(file, os.path.join(report_dir, os.path.basename(file)))
+
+
+def load_cauchy_table(csv_file):
+    """Load the Cauchy combination table from a compressed CSV file using Pandas."""
+    df = pd.read_csv(csv_file, compression='gzip')
+    table_data = df[['annotation', 'p_cauchy', 'p_median']].to_dict(orient='records')
+    return table_data
+
+
+def load_gene_diagnostic_info(csv_file):
+    """Load the Gene Diagnostic Info CSV file and return the top 50 rows."""
+    df = pd.read_csv(csv_file)
+    top_50 = df.head(50).to_dict(orient='records')
+    return top_50
+
+
+def embed_html_content(file_path):
+    """Read the content of an HTML file and return it as a string."""
+    with open(file_path, 'r') as f:
+        return f.read()
+
+def check_and_run_cauchy_combination(config):
+    cauchy_result_file = config.get_cauchy_result_file(config.trait_name)
+    if cauchy_result_file.exists():
+        logger.info(
+            f"Cauchy combination already done for trait {config.trait_name}. Results saved at {cauchy_result_file}. Skipping...")
+    else:
+        logger.info(f"Running Cauchy combination for trait {config.trait_name}...")
+        cauchy_config = CauchyCombinationConfig(
+            workdir=config.workdir,
+            sample_name=config.sample_name,
+            annotation=config.annotation,
+            trait_name=config.trait_name,
+        )
+        run_Cauchy_combination(cauchy_config)
+
+    df = pd.read_csv(cauchy_result_file, compression='gzip')
+    table_data = df[['annotation', 'p_cauchy', 'p_median']].to_dict(orient='records')
+
+    return table_data
+
+def run_report(config: ReportConfig, run_parameters=None):
+
+    logger.info('Running gsMap Diagnosis Module')
+    run_Diagnosis(config)
+    logger.info('gsMap Diagnosis running successfully')
+
+    report_dir = config.get_report_dir(config.trait_name)
+    gene_diagnostic_info_file = config.get_gene_diagnostic_info_save_path(config.trait_name)
+    gene_diagnostic_info = load_gene_diagnostic_info(gene_diagnostic_info_file)
+
+    # Load data (Cauchy table and gene diagnostic info)
+    cauchy_table = check_and_run_cauchy_combination(config)
+
+    # Paths to PNGs for gene expression and GSS distribution
+    gss_distribution_dir = config.get_GSS_plot_dir(config.trait_name)
+
+    gene_plots = []
+    plot_select_gene_list = config.get_GSS_plot_select_gene_file(config.trait_name).read_text().splitlines()
+    for gene_name in plot_select_gene_list:
+        expression_png = gss_distribution_dir / f"{config.sample_name}_{gene_name}_Expression_Distribution.png"
+        gss_png = gss_distribution_dir / f"{config.sample_name}_{gene_name}_GSS_Distribution.png"
+        # check if expression and GSS plots exist
+        if not os.path.exists(expression_png) or not os.path.exists(gss_png):
+            print(f"Skipping gene {gene_name} as expression or GSS plot is missing.")
+            continue
+        gene_plots.append({
+            'name': gene_name,
+            'expression_plot': expression_png.relative_to(report_dir),  # Path for gene expression plot
+            'gss_plot': gss_png.relative_to(report_dir)  # Path for GSS distribution plot
+        })
+
+    # # Copy PNG files to the report directory
+    # copy_files_to_report_dir(result_dir, report_dir, [gene['expression_plot'] for gene in gene_plots] + [gene['gss_plot'] for gene in gene_plots])
+
+    # Update paths to point to copied images inside the report folder
+    # for gene in gene_plots:
+    #     gene['expression_plot'] = os.path.join(os.path.basename(gene['expression_plot']))
+    #     gene['gss_plot'] = os.path.join(os.path.basename(gene['gss_plot']))
+
+    # Sample data for other report components
+    title = f"{config.sample_name} Genetic Spatial Mapping Report"
+
+    genetic_mapping_plot = embed_html_content(config.get_gsMap_html_plot_save_path(config.trait_name))
+    manhattan_plot = embed_html_content(config.get_manhattan_html_plot_path(config.trait_name))
+
+    gsmap_version = gsMap.__version__
+    # Render the template with dynamic content, including the run parameters
+
+    trait_name = config.trait_name
+    default_run_parameters = {
+        "Sample Name": config.sample_name,
+        "Trait Name": trait_name,
+        "Summary Statistics File": config.sumstats_file,
+        "HDF5 Path": config.hdf5_with_latent_path,
+        "Annotation": config.annotation,
+        "Spatial LDSC Save Directory": config.ldsc_save_dir,
+        "Cauchy Directory": config.cauchy_save_dir,
+        "Report Directory": config.get_report_dir(trait_name),
+        "gsMap Report File": config.get_gsMap_report_file(trait_name),
+        "Gene Diagnostic Info File": config.get_gene_diagnostic_info_save_path(trait_name),
+        "Report Generation Date": pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S'),
+    }
+
+    if run_parameters is not None:
+        default_run_parameters.update(run_parameters)
+
+
+    output_html = template.render(
+        title=title,
+        genetic_mapping_plot=genetic_mapping_plot,  # Inlined genetic mapping plot
+        manhattan_plot=manhattan_plot,  # Inlined Manhattan plot
+        cauchy_table=cauchy_table,
+        gene_plots=gene_plots,  # List of PNG paths for gene plots
+        gsmap_version=gsmap_version,
+        parameters=default_run_parameters,  # Pass the run parameters to the template
+        gene_diagnostic_info=gene_diagnostic_info  # Include top 50 gene diagnostic info rows
+    )
+
+    # Save the generated HTML report in the 'report' directory
+    report_file = config.get_gsMap_report_file(config.trait_name)
+    with open(report_file, "w") as f:
+        f.write(output_html)
+
+    logger.info(f"Report generated successfully! Saved at {report_file}.")
+    logger.info(f"Copy the report directory to your local PC and open the HTML report file in a web browser to view the report.")

gsMap/run_all_mode.py

@@ -0,0 +1,195 @@
+import logging
+import time
+from pathlib import Path
+
+from gsMap.cauchy_combination_test import run_Cauchy_combination
+from gsMap.config import GenerateLDScoreConfig, SpatialLDSCConfig, LatentToGeneConfig, \
+    FindLatentRepresentationsConfig, CauchyCombinationConfig, RunAllModeConfig, ReportConfig
+from gsMap.find_latent_representation import run_find_latent_representation
+from gsMap.generate_ldscore import run_generate_ldscore
+from gsMap.latent_to_gene import run_latent_to_gene
+from gsMap.report import run_report
+from gsMap.spatial_ldsc_multiple_sumstats import run_spatial_ldsc
+
+
+
+def format_duration(seconds):
+    hours = int(seconds // 3600)
+    minutes = int((seconds % 3600) // 60)
+    return f"{hours}h {minutes}m"
+
+
+def run_pipeline(config: RunAllModeConfig):
+    # # Set up logging
+    log_file = Path(config.workdir) / config.sample_name / 'gsMap_pipeline.log'
+    log_file.parent.mkdir(parents=True, exist_ok=True)
+    logging.basicConfig(
+        level=logging.INFO,
+        format='[{asctime}] {levelname:.5s} | {name} - {message}',
+        handlers=[
+            logging.FileHandler(log_file),
+        ],
+        style='{'
+    )
+
+    logger = logging.getLogger('gsMap.pipeline')
+    logger.info("Starting pipeline with configuration: %s", config)
+
+    find_latent_config = FindLatentRepresentationsConfig(
+        workdir=config.workdir,
+        input_hdf5_path=config.hdf5_path,
+        sample_name=config.sample_name,
+        annotation=config.annotation,
+        data_layer=config.data_layer
+    )
+
+    latent_to_gene_config = LatentToGeneConfig(
+        workdir=config.workdir,
+        sample_name=config.sample_name,
+        annotation=config.annotation,
+        latent_representation='latent_GVAE',
+        num_neighbour=51,
+        num_neighbour_spatial=201,
+        homolog_file=config.homolog_file
+    )
+
+    ldscore_config = GenerateLDScoreConfig(
+        workdir=config.workdir,
+        sample_name=config.sample_name,
+        chrom='all',
+        # ldscore_save_dir=f"{config.workdir}/{config.sample_name}/generate_ldscore",
+        # mkscore_feather_file=latent_to_gene_config.output_feather_path,
+        bfile_root=config.bfile_root,
+        keep_snp_root=config.keep_snp_root,
+        gtf_annotation_file=config.gtffile,
+        spots_per_chunk=5_000,
+        baseline_annotation_dir=config.baseline_annotation_dir,
+        SNP_gene_pair_dir=config.SNP_gene_pair_dir,
+        ldscore_save_format='quick_mode'
+
+    )
+
+    pipeline_start_time = time.time()
+
+    # Step 1: Find latent representations
+    start_time = time.time()
+    logger.info("Step 1: Finding latent representations")
+    if Path(find_latent_config.hdf5_with_latent_path).exists():
+        logger.info(
+            f"Find latent representations already done. Results saved at {find_latent_config.hdf5_with_latent_path}. Skipping...")
+    else:
+        run_find_latent_representation(find_latent_config)
+    end_time = time.time()
+    logger.info(f"Step 1 completed in {format_duration(end_time - start_time)}.")
+
+    # Step 2: Latent to gene
+    start_time = time.time()
+    logger.info("Step 2: Mapping latent representations to genes")
+    if Path(latent_to_gene_config.mkscore_feather_path).exists():
+        logger.info(
+            f"Latent to gene mapping already done. Results saved at {latent_to_gene_config.mkscore_feather_path}. Skipping...")
+    else:
+        run_latent_to_gene(latent_to_gene_config)
+    end_time = time.time()
+    logger.info(f"Step 2 completed in {format_duration(end_time - start_time)}.")
+
+    # Step 3: Generate LDScores
+    start_time = time.time()
+    logger.info("Step 3: Generating LDScores")
+
+    # check if LDscore has been generated by the done file
+    ldsc_done_file = Path(ldscore_config.ldscore_save_dir) / f"{config.sample_name}_generate_ldscore.done"
+    if ldsc_done_file.exists():
+        logger.info(f"Basic LDScore generation already done. Results saved at {ldscore_config.ldscore_save_dir}. Skipping...")
+    else:
+        run_generate_ldscore(ldscore_config)
+        end_time = time.time()
+        logger.info(f"Step 3 completed in {format_duration(end_time - start_time)}.")
+        # create a done file
+        ldsc_done_file.touch()
+
+    # Step 4: Spatial LDSC
+    start_time = time.time()
+    logger.info("Step 4: Running spatial LDSC")
+
+    sumstats_config = config.sumstats_config_dict
+    for trait_name in sumstats_config:
+        logger.info("Running spatial LDSC for trait: %s", trait_name)
+        # detect if the spatial LDSC has been done:
+        spatial_ldsc_result_file = Path(config.ldsc_save_dir) / f"{config.sample_name}_{trait_name}.csv.gz"
+
+        if spatial_ldsc_result_file.exists():
+            logger.info(
+                f"Spatial LDSC already done for trait {trait_name}. Results saved at {spatial_ldsc_result_file}. Skipping...")
+            continue
+
+        spatial_ldsc_config_trait = SpatialLDSCConfig(
+            workdir=config.workdir,
+            sumstats_file=sumstats_config[trait_name],
+            trait_name=trait_name,
+            w_file=config.w_file,
+            sample_name=config.sample_name,
+            # ldscore_save_dir=spatial_ldsc_config.ldscore_save_dir,
+            # ldsc_save_dir=spatial_ldsc_config.ldsc_save_dir,
+            num_processes=config.max_processes,
+            ldscore_save_format='quick_mode',
+            snp_gene_weight_adata_path=config.snp_gene_weight_adata_path,
+        )
+        run_spatial_ldsc(spatial_ldsc_config_trait)
+    end_time = time.time()
+    logger.info(f"Step 4 completed in {format_duration(end_time - start_time)}.")
+
+    # Step 5: Cauchy combination test
+    start_time = time.time()
+    logger.info("Step 6: Running Cauchy combination test")
+    '/storage/yangjianLab/chenwenhao/projects/202312_GPS/test/20240817_vanilla_pipeline_mouse_embryo_v4/E16.5_E1S1.MOSTA/cauchy_combination/E16.5_E1S1.MOSTA_Depression_2023_NatureMed.Cauchy.csv.gz'
+    for trait_name in sumstats_config:
+        # check if the cauchy combination has been done
+        cauchy_result_file = config.get_cauchy_result_file(trait_name)
+        if cauchy_result_file.exists():
+            logger.info(
+                f"Cauchy combination already done for trait {trait_name}. Results saved at {cauchy_result_file}. Skipping...")
+            continue
+        cauchy_config = CauchyCombinationConfig(
+            workdir=config.workdir,
+            sample_name=config.sample_name,
+            annotation=config.annotation,
+            trait_name=trait_name,
+        )
+        run_Cauchy_combination(cauchy_config)
+    end_time = time.time()
+    logger.info(f"Step 5 completed in {format_duration(end_time - start_time)}.")
+
+    # Step 6: Generate final report
+    for trait_name in sumstats_config:
+        logger.info("Running final report generation for trait: %s", trait_name)
+        report_config = ReportConfig(
+            workdir=config.workdir,
+            sample_name=config.sample_name,
+            annotation=config.annotation,
+            trait_name=trait_name,
+            plot_type='all',
+            top_corr_genes=50,
+            selected_genes=None,
+            sumstats_file=sumstats_config[trait_name],
+        )
+        # Create the run parameters dictionary for each trait
+        run_parameter_dict = {
+            "Sample Name": config.sample_name,
+            "Trait Name": trait_name,
+            "Summary Statistics File": sumstats_config[trait_name],
+            "HDF5 Path": config.hdf5_path,
+            "Annotation": config.annotation,
+            "Number of Processes": config.max_processes,
+            "Spatial LDSC Save Directory": config.ldsc_save_dir,
+            "Cauchy Directory": config.cauchy_save_dir,
+            "Report Directory": config.get_report_dir(trait_name),
+            "gsMap Report File": config.get_gsMap_report_file(trait_name),
+            "Gene Diagnostic Info File": config.get_gene_diagnostic_info_save_path(trait_name),
+            "Spending Time": format_duration(time.time() - pipeline_start_time),
+        }
+
+        # Pass the run parameter dictionary to the report generation function
+        run_report(report_config, run_parameters=run_parameter_dict)
+
+    logger.info("Pipeline completed successfully.")