gsMap3D 0.1.0a1__py3-none-any.whl

gsMap/config/decorators.py

@@ -0,0 +1,302 @@
+"""
+Decorators for CLI and resource tracking.
+"""
+
+import functools
+import inspect
+import logging
+import os
+import re
+import subprocess
+import sys
+import threading
+import time
+from dataclasses import fields
+from functools import wraps
+from typing import Annotated, Any, get_args, get_origin
+
+import psutil
+import pyfiglet
+
+logger = logging.getLogger("gsMap")
+
+
+def process_cpu_time(proc: psutil.Process):
+    """Calculate total CPU time for a process."""
+    cpu_times = proc.cpu_times()
+    total = cpu_times.user + cpu_times.system
+    return total
+
+
+def track_resource_usage(func):
+    """
+    Decorator to track resource usage during function execution.
+    Logs memory usage, CPU time, and wall clock time at the end of the function.
+    """
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        # Get the current process
+        process = psutil.Process(os.getpid())
+
+        # Initialize tracking variables
+        peak_memory = 0
+        cpu_percent_samples = []
+        stop_thread = False
+
+        # Function to monitor resource usage
+        def resource_monitor():
+            nonlocal peak_memory, cpu_percent_samples
+            while not stop_thread:
+                try:
+                    # Get current memory usage in MB
+                    current_memory = process.memory_info().rss / (1024 * 1024)
+                    peak_memory = max(peak_memory, current_memory)
+
+                    # Get CPU usage percentage
+                    cpu_percent = process.cpu_percent(interval=None)
+                    if cpu_percent > 0:  # Skip initial zero readings
+                        cpu_percent_samples.append(cpu_percent)
+
+                    time.sleep(0.5)
+                except Exception:
+                    pass
+
+        # Start resource monitoring in a separate thread
+        monitor_thread = threading.Thread(target=resource_monitor)
+        monitor_thread.daemon = True
+        monitor_thread.start()
+
+        # Get start times
+        start_wall_time = time.time()
+        start_cpu_time = process_cpu_time(process)
+
+        try:
+            # Run the actual function
+            result = func(*args, **kwargs)
+            return result
+        finally:
+            # Stop the monitoring thread
+            stop_thread = True
+            monitor_thread.join(timeout=1.0)
+
+            # Calculate elapsed times
+            end_wall_time = time.time()
+            end_cpu_time = process_cpu_time(process)
+
+            wall_time = end_wall_time - start_wall_time
+            cpu_time = end_cpu_time - start_cpu_time
+
+            # Calculate average CPU percentage
+            avg_cpu_percent = (
+                sum(cpu_percent_samples) / len(cpu_percent_samples) if cpu_percent_samples else 0
+            )
+
+            # Adjust for macOS if needed
+            if sys.platform == "darwin":
+                from gsMap.utils import macos_timebase_factor
+                factor = macos_timebase_factor()
+                cpu_time *= factor
+                avg_cpu_percent *= factor
+
+            # Format memory for display
+            if peak_memory < 1024:
+                memory_str = f"{peak_memory:.2f} MB"
+            else:
+                memory_str = f"{peak_memory / 1024:.2f} GB"
+
+            # Format times for display
+            if wall_time < 60:
+                wall_time_str = f"{wall_time:.2f} seconds"
+            elif wall_time < 3600:
+                wall_time_str = f"{wall_time / 60:.2f} minutes"
+            else:
+                wall_time_str = f"{wall_time / 3600:.2f} hours"
+
+            if cpu_time < 60:
+                cpu_time_str = f"{cpu_time:.2f} seconds"
+            elif cpu_time < 3600:
+                cpu_time_str = f"{cpu_time / 60:.2f} minutes"
+            else:
+                cpu_time_str = f"{cpu_time / 3600:.2f} hours"
+
+            # Log the resource usage
+            logger.info("Resource usage summary:")
+            logger.info(f"  • Wall clock time: {wall_time_str}")
+            logger.info(f"  • CPU time: {cpu_time_str}")
+            logger.info(f"  • Average CPU utilization: {avg_cpu_percent:.1f}%")
+            logger.info(f"  • Peak memory usage: {memory_str}")
+
+    return wrapper
+
+
+def show_banner(command_name: str, version: str = "1.73.5"):
+    """Display gsMap banner and version information."""
+    command_name = command_name.replace("_", " ")
+    gsMap_main_logo = pyfiglet.figlet_format(
+        "gsMap",
+        font="doom",
+        width=80,
+        justify="center",
+    ).rstrip()
+    print(gsMap_main_logo, flush=True)
+    version_number = "Version: " + version
+    print(version_number.center(80), flush=True)
+    print("=" * 80, flush=True)
+    logger.info(f"Running {command_name}...")
+
+    # Record start time for the log message
+    start_time = time.strftime("%Y-%m-%d %H:%M:%S")
+    logger.info(f"Started at: {start_time}")
+
+
+def dataclass_typer(func):
+    """
+    Decorator to convert a function that takes a dataclass config
+    into a Typer command with individual CLI options.
+    """
+    sig = inspect.signature(func)
+
+    # Get the dataclass type from the function signature
+    config_param = list(sig.parameters.values())[0]
+    config_class = config_param.annotation
+
+    @wraps(func)
+    @track_resource_usage  # Add resource tracking
+    def wrapper(**kwargs):
+        # Show banner
+        try:
+            from gsMap import __version__
+            version = __version__
+        except ImportError:
+            version = "development"
+        show_banner(func.__name__, version)
+
+        # Create the config instance
+        config = config_class(**kwargs)
+        result = func(config)
+
+        # Record end time
+        end_time = time.strftime("%Y-%m-%d %H:%M:%S")
+        logger.info(f"Finished at: {end_time}")
+
+        return result
+
+    # Build new parameters from dataclass fields
+    from dataclasses import MISSING
+    params = []
+
+    core_only = getattr(config_class, "__core_only__", False)
+
+    def is_core_field(field_name: str, cls: type[Any]) -> bool:
+        """Check if a field originates from a 'Core' config class."""
+        for base in cls.__mro__:
+            if field_name in getattr(base, "__annotations__", {}):
+                # If ANY class in the inheritance chain for this field explicitly disables quick mode, honor it
+                if getattr(base, "__display_in_quick_mode_cli__", True) is False:
+                    return False
+        return True
+
+    for field in fields(config_class):
+        # Only include fields with Annotated type hints in the CLI
+        # This allows internal fields to be excluded from CLI parameters
+
+        # Check if the field type is Annotated
+        if get_origin(field.type) != Annotated:
+            continue
+
+        # Get Annotated metadata
+        annotated_args = get_args(field.type)
+
+        # Check for explicit display override in Annotated metadata
+        # e.g., Annotated[int, typer.Option(...), {"__display_in_quick_mode_cli__": True}]
+        display_override = None
+        for arg in annotated_args:
+            if isinstance(arg, dict) and "__display_in_quick_mode_cli__" in arg:
+                display_override = arg["__display_in_quick_mode_cli__"]
+                break
+
+        if core_only:
+            # If field explicitly says True, we show it even if class says False
+            if display_override is True:
+                pass
+            # If field explicitly says False, we hide it
+            elif display_override is False:
+                continue
+            # Otherwise, fall back to class-level logic
+            elif not is_core_field(field.name, config_class):
+                continue
+
+        # Get the actual type and typer.Option from Annotated
+        # Annotated[type, typer.Option(...)] -> type is at args[0]
+        get_args(field.type)[0]
+
+        # Determine the default value
+        if field.default is not MISSING:
+            # Field has a default value, use it as the parameter default
+            default_value = field.default
+        elif field.default_factory is not MISSING:
+            # Field has a default factory, call it to get the default value
+            default_value = field.default_factory()
+        else:
+            # No default, parameter is required
+            default_value = inspect.Parameter.empty
+
+        # Create the parameter
+        if default_value is not inspect.Parameter.empty:
+            param = inspect.Parameter(
+                field.name,
+                inspect.Parameter.KEYWORD_ONLY,
+                annotation=field.type,  # Keep the full Annotated type
+                default=default_value
+            )
+        else:
+            param = inspect.Parameter(
+                field.name,
+                inspect.Parameter.KEYWORD_ONLY,
+                annotation=field.type  # Keep the full Annotated type
+            )
+        params.append(param)
+
+    # Update the wrapper's signature
+    wrapper.__signature__ = inspect.Signature(params)
+
+    # Preserve the original function's docstring
+    wrapper.__doc__ = func.__doc__
+
+    return wrapper
+
+
+@functools.cache
+def macos_timebase_factor():
+    """
+    On MacOS, `psutil.Process.cpu_times()` is not accurate, check activity monitor instead.
+    see: https://github.com/giampaolo/psutil/issues/2411#issuecomment-2274682289
+    """
+    default_factor = 1
+    ioreg_output_lines = []
+
+    try:
+        result = subprocess.run(
+            ["ioreg", "-p", "IODeviceTree", "-c", "IOPlatformDevice"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        ioreg_output_lines = result.stdout.splitlines()
+    except subprocess.CalledProcessError as e:
+        print(f"Command failed: {e}")
+        return default_factor
+
+    if not ioreg_output_lines:
+        return default_factor
+
+    for line in ioreg_output_lines:
+        if "timebase-frequency" in line:
+            match = re.search(r"<([0-9a-fA-F]+)>", line)
+            if not match:
+                return default_factor
+            byte_data = bytes.fromhex(match.group(1))
+            timebase_freq = int.from_bytes(byte_data, byteorder="little")
+            # Typically, it should be 1000/24.
+            return pow(10, 9) / timebase_freq
+    return default_factor

gsMap/config/find_latent_config.py

@@ -0,0 +1,276 @@
+"""
+Configuration for finding latent representations.
+"""
+
+import logging
+from collections import OrderedDict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Annotated
+
+import typer
+import yaml
+
+from gsMap.config.base import ConfigWithAutoPaths
+from gsMap.config.utils import (
+    process_h5ad_inputs,
+    validate_h5ad_structure,
+    verify_homolog_file_format,
+)
+
+logger = logging.getLogger("gsMap.config")
+
+@dataclass
+class FindLatentModelConfig:
+    __display_in_quick_mode_cli__ = False
+
+    feat_cell: Annotated[int, typer.Option(
+        help="Number of top variable features to retain",
+        min=100,
+        max=10000
+    )] = 2000
+
+    # Feature extraction parameters
+    n_neighbors: Annotated[int, typer.Option(
+        help="Number of neighbors for LGCN",
+        min=1,
+        max=50
+    )] = 10
+
+    K: Annotated[int, typer.Option(
+        help="Graph convolution depth for LGCN",
+        min=1,
+        max=10
+    )] = 3
+
+    # Model parameters
+    hidden_size: Annotated[int, typer.Option(
+        help="Units in the first hidden layer",
+        min=32,
+        max=512
+    )] = 128
+
+    embedding_size: Annotated[int, typer.Option(
+        help="Size of the latent embedding layer",
+        min=8,
+        max=128
+    )] = 32
+
+    # Transformer parameters
+    use_tf: Annotated[bool, typer.Option(
+        "--use-tf",
+        help="Enable transformer module"
+    )] = False
+
+    module_dim: Annotated[int, typer.Option(
+        help="Dimensionality of transformer modules",
+        min=10,
+        max=100
+    )] = 30
+
+    hidden_gmf: Annotated[int, typer.Option(
+        help="Hidden units for global mean feature extractor",
+        min=32,
+        max=512
+    )] = 128
+
+    n_modules: Annotated[int, typer.Option(
+        help="Number of transformer modules",
+        min=4,
+        max=64
+    )] = 16
+
+    nhead: Annotated[int, typer.Option(
+        help="Number of attention heads in transformer",
+        min=1,
+        max=16
+    )] = 4
+
+    n_enc_layer: Annotated[int, typer.Option(
+        help="Number of transformer encoder layers",
+        min=1,
+        max=8
+    )] = 2
+
+    # Training parameters
+    distribution: Annotated[str, typer.Option(
+        help="Distribution type for loss calculation",
+        case_sensitive=False
+    )] = "nb"
+
+
+    batch_size: Annotated[int, typer.Option(
+        help="Batch size for training",
+        min=32,
+        max=4096
+    )] = 1024
+
+    itermax: Annotated[int, typer.Option(
+        help="Maximum number of training iterations",
+        min=10,
+        max=1000
+    )] = 100
+
+    patience: Annotated[int, typer.Option(
+        help="Early stopping patience",
+        min=1,
+        max=50
+    )] = 10
+
+    two_stage: Annotated[bool, typer.Option(
+        "--two-stage/--single-stage",
+        help="Tune the cell embeddings based on the provided annotation"
+    ),{"__display_in_quick_mode_cli__": True}] = False
+
+    do_sampling: Annotated[bool, typer.Option(
+        "--do-sampling/--no-sampling",
+        help="Down-sampling cells in training"
+    )] = True
+
+    n_cell_training: Annotated[int, typer.Option(
+        help="Number of cells used for training",
+        min=1000,
+        max=1000000
+    ), {"__display_in_quick_mode_cli__": True}] = 100000
+
+
+
+@dataclass
+class FindLatentCoreConfig:
+    h5ad_path: Annotated[list[Path] | None, typer.Option(
+        help="Space-separated list of h5ad file paths. Sample names are derived from file names without suffix.",
+        exists=True,
+        file_okay=True,
+    )] = None
+
+    h5ad_yaml: Annotated[Path | None, typer.Option(
+        help="YAML file with sample names and h5ad paths",
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+    )] = None
+
+    h5ad_list_file: Annotated[Path | None, typer.Option(
+        help="Each row is a h5ad file path, sample name is the file name without suffix",
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+    )] = None
+
+    sample_h5ad_dict: OrderedDict | None = None
+
+    data_layer: Annotated[str, typer.Option(
+        help="Gene expression raw counts data layer in h5ad layers, e.g., 'count', 'counts'. Other wise use 'X' for adata.X"
+    )] = "X"
+
+    spatial_key: Annotated[str, typer.Option(
+        help="Spatial key in adata.obsm storing spatial coordinates"
+    )] = "spatial"
+
+    annotation: Annotated[str | None, typer.Option(
+        help="Annotation of cell type in adata.obs to use"
+    )] = None
+
+    homolog_file: Annotated[Path | None, typer.Option(
+        help="Path to homologous gene conversion file",
+        exists=True,
+        file_okay=True,
+        dir_okay=False
+    )] = None
+
+    species: str | None = None
+
+    latent_representation_niche: Annotated[str, typer.Option(
+        help="Key for spatial niche embedding in obsm"
+    )] = "emb_niche"
+
+    latent_representation_cell: Annotated[str, typer.Option(
+        help="Key for cell identity embedding in obsm"
+    )] = "emb_cell"
+
+
+
+    high_quality_cell_qc: Annotated[bool, typer.Option(
+        "--high-quality-cell-qc/--no-high-quality-cell-qc",
+        help="Enable/disable high quality cell QC based on module scores. If enabled, it will compute DEG and module scores."
+    )] = True
+
+
+@dataclass
+class FindLatentRepresentationsConfig(FindLatentModelConfig, FindLatentCoreConfig, ConfigWithAutoPaths):
+    """Find Latent Configuration"""
+
+
+    def __post_init__(self):
+        super().__post_init__()
+
+        # Define input options
+        input_options = {
+            'h5ad_yaml': ('h5ad_yaml', 'yaml'),
+            'h5ad_path': ('h5ad_path', 'list'),
+            'h5ad_list_file': ('h5ad_list_file', 'file'),
+        }
+
+        # Process h5ad inputs
+        self.sample_h5ad_dict = process_h5ad_inputs(self, input_options)
+
+        if not self.sample_h5ad_dict:
+            raise ValueError(
+                "At least one of h5ad_yaml, h5ad_path, h5ad_list_file, or spe_file_list must be provided"
+            )
+
+        # Define required and optional fields for validation
+        required_fields = {
+            'data_layer': ('layers', self.data_layer, 'Data layer'),
+            'spatial_key': ('obsm', self.spatial_key, 'Spatial key'),
+        }
+
+        # Add annotation as required if provided
+        if self.annotation:
+            required_fields['annotation'] = ('obs', self.annotation, 'Annotation')
+
+        # Validate h5ad structure
+        validate_h5ad_structure(self.sample_h5ad_dict, required_fields)
+
+        # Log final sample count
+        logger.info(f"Loaded and validated {len(self.sample_h5ad_dict)} samples")
+
+        # Check if at least one sample is provided
+        if len(self.sample_h5ad_dict) == 0:
+            raise ValueError("No valid samples found in the provided input")
+
+        # Verify homolog file format if provided
+        verify_homolog_file_format(self)
+
+        self.show_config(FindLatentRepresentationsConfig)
+
+
+def check_find_latent_done(config: FindLatentRepresentationsConfig) -> bool:
+    """
+    Check if find_latent step is done by verifying validity of metadata and output files.
+    """
+    metadata_path = config.find_latent_metadata_path
+    if not metadata_path.exists():
+        return False
+
+    try:
+        with open(metadata_path) as f:
+            metadata = yaml.safe_load(f)
+
+        if 'outputs' not in metadata or 'latent_files' not in metadata['outputs']:
+            return False
+
+        latent_files = metadata['outputs']['latent_files']
+        if not latent_files:
+            return False
+
+        # Verify all listed files exist
+        all_exist = True
+        for sample, path_str in latent_files.items():
+            if not Path(path_str).exists():
+                all_exist = False
+                break
+        return all_exist
+    except Exception as e:
+        logger.warning(f"Error checking find_latent metadata: {e}")
+        return False

gsMap/config/format_sumstats_config.py

@@ -0,0 +1,54 @@
+"""
+Configuration for formatting GWAS summary statistics.
+"""
+
+from dataclasses import dataclass
+from typing import Annotated, Literal
+
+import typer
+
+
+@dataclass
+class FormatSumstatsConfig:
+    """Configuration for formatting GWAS summary statistics."""
+
+    sumstats: Annotated[str, typer.Option(help="Path to gwas summary data")]
+    out: Annotated[str, typer.Option(help="Path to save the formatted gwas data")]
+
+    # Arguments for specify column name
+    snp: Annotated[str | None, typer.Option(help="Name of snp column")] = None
+    a1: Annotated[str | None, typer.Option(help="Name of effect allele column")] = None
+    a2: Annotated[str | None, typer.Option(help="Name of none-effect allele column")] = None
+    info: Annotated[str | None, typer.Option(help="Name of info column")] = None
+    beta: Annotated[str | None, typer.Option(help="Name of gwas beta column.")] = None
+    se: Annotated[str | None, typer.Option(help="Name of gwas standar error of beta column")] = None
+    p: Annotated[str | None, typer.Option(help="Name of p-value column")] = None
+    frq: Annotated[str | None, typer.Option(help="Name of A1 ferquency column")] = None
+    n: Annotated[str | None, typer.Option(help="Name of sample size column")] = None
+    z: Annotated[str | None, typer.Option(help="Name of gwas Z-statistics column")] = None
+    OR: Annotated[str | None, typer.Option(help="Name of gwas OR column")] = None
+    se_OR: Annotated[str | None, typer.Option(help="Name of standar error of OR column")] = None
+
+    # Arguments for convert SNP (chr, pos) to rsid
+    chr: Annotated[str, typer.Option(help="Name of SNP chromosome column")] = "Chr"
+    pos: Annotated[str, typer.Option(help="Name of SNP positions column")] = "Pos"
+    dbsnp: Annotated[str | None, typer.Option(help="Path to reference dnsnp file")] = None
+    chunksize: Annotated[int, typer.Option(help="Chunk size for loading dbsnp file")] = 1000000
+
+    # Arguments for output format and quality
+    format: Annotated[Literal["gsMap", "COJO"], typer.Option(help="Format of output data", case_sensitive=False)] = "gsMap"
+    info_min: Annotated[float, typer.Option(help="Minimum INFO score.")] = 0.9
+    maf_min: Annotated[float, typer.Option(help="Minimum MAF.")] = 0.01
+    keep_chr_pos: Annotated[bool, typer.Option(help="Keep SNP chromosome and position columns in the output data")] = False
+
+    def __post_init__(self):
+        # Handle n being potentially a number passed as a string from CLI
+        if isinstance(self.n, str):
+            try:
+                if "." in self.n:
+                    self.n = float(self.n)
+                else:
+                    self.n = int(self.n)
+            except ValueError:
+                # Leave as string if it's a column name
+                pass