protrider 0.9.1__py3-none-any.whl

protrider/__init__.py

@@ -0,0 +1,5 @@
+from .config import ProtriderConfig, load_config
+from .model import ModelInfo
+from .pipeline import Result, run
+
+__all__ = ["ProtriderConfig", "ModelInfo", "Result", "run", "load_config"]

protrider/cli.py

@@ -0,0 +1,265 @@
+from collections import defaultdict
+
+import numpy as np
+import yaml
+from pathlib import Path
+import click
+import torch
+import logging
+
+from .pipeline import run as run_pipeline
+from .config import load_config
+from . import plots
+
+logger = logging.getLogger(__name__)
+
+
+@click.group()
+def cli() -> None:
+    """# PROTRIDER
+
+    PROTRIDER is a package for calling protein outliers on mass spectrometry data
+
+    Links:
+    - Publication: FIXME
+    - Official code repository: https://github.com/gagneurlab/PROTRIDER
+
+    """
+    pass
+
+
+@cli.group(chain=True)
+@click.option(
+    "--config",
+    help="Configuration file containing PROTRIDER custom options",
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    '--plot_title',
+    type=str,
+    default="",
+    help="Title of the plots"
+)
+@click.pass_context
+def plot(ctx, config: str, plot_title: str = ""):
+    """
+    Plot the results of a PROTRIDER run.
+    """
+    if config is None:
+        click.echo('No config file provided. Exiting.')
+        return
+
+    config = yaml.load(open(config), Loader=yaml.FullLoader)
+    config = defaultdict(lambda: None, config)
+    if config['verbose']:
+        logging.basicConfig(
+            level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', )
+    else:
+        logging.basicConfig(
+            level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', )
+
+    out_dir = config['out_dir']
+    if Path(out_dir).is_dir() is False:
+        click.echo(f'Output directory {out_dir} does not exist. Exiting.')
+        return
+
+    ctx.obj = config
+    ctx.obj['plot_title'] = plot_title
+
+
+@plot.command('all')
+@click.pass_context
+def plot_all(ctx):
+    """
+    Plot all results for a PROTRIDER run.
+    """
+    if ctx.obj is None:
+        return
+    out_dir = ctx.obj['out_dir']
+    plot_title = ctx.obj['plot_title']
+    logger.info("plotting all plots")
+    plots.plot_pvals(out_dir, ctx.obj['pval_dist'], plot_title)
+    plots.plot_aberrant_per_sample(out_dir, plot_title)
+    plots.plot_encoding_dim(out_dir, ctx.obj['find_q_method'], plot_title)
+    plots.plot_training_loss(out_dir, plot_title)
+    plots.plot_correlation_heatmap(
+        out_dir, ctx.obj['sample_annotation'], plot_title, None)
+
+
+@plot.command('pvals')
+@click.pass_context
+def plot_pvals(ctx):
+    """
+    Plot pvalue plots for a PROTRIDER run.
+    """
+    if ctx.obj is None:
+        return
+    out_dir = ctx.obj['out_dir']
+    plot_title = ctx.obj['plot_title']
+    logger.info("plotting pvalue plots")
+    plots.plot_pvals(out_dir, ctx.obj['pval_dist'], plot_title)
+
+
+@plot.command('aberrant_per_sample')
+@click.pass_context
+def plot_aberrant_per_sample(ctx):
+    """
+    Plot number of aberrant proteins per sample for a PROTRIDER run.
+    """
+    if ctx.obj is None:
+        return
+    out_dir = ctx.obj['out_dir']
+    plot_title = ctx.obj['plot_title']
+    logger.info("plotting number of aberrant proteins per sample")
+    plots.plot_aberrant_per_sample(out_dir, plot_title)
+
+
+@plot.command('encoding_dim')
+@click.pass_context
+def plot_encoding_dim(ctx):
+    """
+    Plot encoding dimension search plot for a PROTRIDER run.
+    """
+    if ctx.obj is None:
+        return
+    out_dir = ctx.obj['out_dir']
+    plot_title = ctx.obj['plot_title']
+    logger.info("plotting encoding dimension search plot")
+    plots.plot_encoding_dim(out_dir, ctx.obj['find_q_method'], plot_title)
+
+
+@plot.command('training_loss')
+@click.pass_context
+def plot_training_loss(ctx):
+    """
+    Plot training loss history for a PROTRIDER run.
+    """
+    if ctx.obj is None:
+        return
+    out_dir = ctx.obj['out_dir']
+    plot_title = ctx.obj['plot_title']
+    logger.info("plotting training loss")
+    plots.plot_training_loss(out_dir, plot_title)
+
+
+@plot.command('expected_vs_observed')
+@click.option(
+    '--protein_id',
+    type=str,
+    help="Id of the protein to plot"
+)
+@click.pass_context
+def plot_expected_vs_observed(ctx, protein_id: str):
+    """
+    Plot expected vs observed protein intensity for a PROTRIDER run.
+    """
+    if ctx.obj is None:
+        return
+    if protein_id is None:
+        click.echo(
+            'No protein_id provided for expected vs observed plot. Exiting.')
+        return
+    out_dir = ctx.obj['out_dir']
+    plot_title = ctx.obj['plot_title']
+    logger.info(
+        f"plotting expected vs observed protein intensitiy for protein {protein_id}")
+    plots.plot_expected_vs_observed(out_dir, protein_id, plot_title)
+
+
+@plot.command('correlation_heatmap')
+@click.option(
+    '--covariate',
+    type=str,
+    help="Name of the covariate to color the samples by"
+)
+@click.pass_context
+def plot_correlation_heatmap(ctx, covariate: str):
+    """
+    Plot correlation heatmap for a PROTRIDER run.
+    """
+    if ctx.obj is None:
+        return
+    out_dir = ctx.obj['out_dir']
+    plot_title = ctx.obj['plot_title']
+    logger.info("plotting correlation heatmap")
+    plots.plot_correlation_heatmap(
+        out_dir, ctx.obj['sample_annotation'], plot_title, covariate_name=covariate)
+
+
+@cli.command('run')
+@click.option(
+    "--config",
+    help="Configuration file containing PROTRIDER custom options",
+    type=click.Path(exists=True, dir_okay=False),
+)
+def run_cli(config: str):
+    """Run the PROTRIDER pipeline.
+    """
+    run(config)
+
+
+def run(config_path: str):
+    """Run the PROTRIDER pipeline.
+    """
+    if config_path is None:
+        click.echo('No config file provided. Exiting.')
+        return
+
+    # Load and validate config using ProtriderConfig dataclass
+    config = load_config(config_path)
+
+    out_dir = config.out_dir
+    if out_dir is None:
+        click.echo('Output directory has not been specified in the config. Exiting.')
+        return
+    
+    if config.verbose:
+        logging.basicConfig(
+            level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', )
+    else:
+        logging.basicConfig(
+            level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', )
+
+    logger.info('Starting protrider')
+    logger.info("Config:\n%s", yaml.dump(config.as_dict(), default_flow_style=False))
+
+    path = Path(config.out_dir)
+    path.mkdir(parents=True, exist_ok=True)
+
+    # Log function and base function are now computed in config.__post_init__
+    if config.log_func is None:
+        logger.warning(
+            'No log function passed for preprocessing. \nAssuming data is already log transformed.')
+
+    if (config.find_q_method == 'OHT') and config.cov_used:
+        logger.warning('OHT has not been evaluated with covariates yet')
+
+    if (config.find_q_method == 'OHT') and config.presence_absence:
+        logger.warning(
+            'OHT has not been evaluated on presence/absence analysis yet')
+
+    if config.seed is not None:
+        logger.info('Setting random seed: %s', config.seed)
+        torch.manual_seed(config.seed)
+        np.random.seed(config.seed)
+
+    # Run PROTRIDER - all inputs are in config
+    result, model_info, fit_parameters, gs_results = run_pipeline(config)
+
+    # Save results
+    # Save wide format (individual CSV files)
+    result.save(config.out_dir, format="wide")
+    # Save long format summary
+    result.save(config.out_dir, format="long", include_all=config.report_all)
+    # Save model information
+    model_info.save(config.out_dir)
+    # Save config
+    config.save(config.out_dir)
+    # Save fit parameters
+    fit_parameters.to_csv(config.out_dir)
+    # Save grid search results if available
+    if gs_results is not None:
+        gs_results.to_csv(config.out_dir)
+
+if __name__ == '__main__':
+    cli(obj={})

protrider/config.py

@@ -0,0 +1,216 @@
+"""Configuration loading utilities."""
+import yaml
+import torch
+from pathlib import Path
+from typing import Union, Callable
+from dataclasses import dataclass, field
+from typing import Optional, List, Literal
+import numpy as np
+
+@dataclass
+class ProtriderConfig:
+    """Configuration for PROTRIDER pipeline.
+    
+    Matrix Format Requirements:
+    ---------------------------
+    input_intensities: File path (str)
+        - File format: columns = samples, rows = proteins
+    
+    sample_annotation: File path (str) or None
+        - Format: rows = samples
+    """
+    
+    # I/O paths
+    input_intensities: str  # File path only
+    input_format: Literal["proteins_as_rows", "proteins_as_columns"] = "proteins_as_rows"
+    index_col: str = "protein_ID"
+    out_dir: Optional[str] = None  # File path or None
+    sample_annotation: Optional[str] = None  # File path or None
+    
+    # Preprocessing params
+    max_allowed_NAs_per_protein: float = 0.3
+    log_func_name: Optional[Literal["log", "log2", "log10"]] = "log"
+    
+    # Computed fields (set in __post_init__)
+    log_func: Optional[Callable] = field(init=False, repr=False, default=None)
+    base_fn: Callable = field(init=False, repr=False, default=None)
+    device_torch: torch.device = field(init=False, repr=False, default=None)
+    
+    # Covariates
+    cov_used: Optional[List[str]] = None
+    
+    # Reproducibility
+    seed: Optional[int] = 42
+    
+    # Grid search outlier injection params
+    inj_freq: float = 1e-3
+    inj_mean: float = 3
+    inj_sd: float = 1.6
+    gs_epochs: int = None
+    
+    # Model params
+    autoencoder_training: bool = True
+    n_layers: int = 1
+    n_epochs: int = 100
+    lr: float = 1e-4
+    batch_size: Optional[int] = None
+    find_q_method: str = "OHT"  # "OHT", "gs", or an integer
+    init_pca: bool = True
+    h_dim: Optional[int] = None
+    patience: int = 50
+    min_delta: float = 1e-4
+    
+    # Presence absence modelling
+    presence_absence: bool = False
+    lambda_presence_absence: float = 0.5
+
+    # Wandb logging
+    use_wandb: bool = False
+    wandb_project: Optional[str] = "protrider"
+    wandb_name: Optional[str] = None
+    
+    # Statistical params
+    pval_dist: Literal["gaussian", "t"] = "t"
+    pval_adj: Literal["by", "bh"] = "by"
+    pval_sided: Literal["two-sided", "left", "right"] = "two-sided"
+    pseudocount: float = 0.01
+    common_degrees_freedom: bool = True  # whether to use common degrees of freedom; more stable for small datasets
+    
+    # Reporting params
+    outlier_threshold: float = 0.1
+    report_all: bool = True
+    
+    # Runtime params
+    verbose: bool = False
+    device: Literal["gpu", "cpu"] = "gpu"
+    n_jobs: int = -1  # Number of parallel jobs, -1 means using all processors
+    
+    # Model checkpoint path for saving/loading trained models
+    # If None: train from scratch and save to out_dir/model.pt
+    # If path exists: load model from this path and skip training
+    # If path doesn't exist: train and save to this path
+    checkpoint_path: Optional[str] = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization and set computed fields."""
+        # Validation
+        if self.max_allowed_NAs_per_protein < 0 or self.max_allowed_NAs_per_protein > 1:
+            raise ValueError("max_allowed_NAs_per_protein must be between 0 and 1")
+        
+        if self.n_layers < 1:
+            raise ValueError("n_layers must be at least 1")
+        
+        if self.n_epochs < 1:
+            raise ValueError("n_epochs must be at least 1")
+        
+        if self.lr <= 0:
+            raise ValueError("lr must be positive")
+        
+        if self.outlier_threshold < 0 or self.outlier_threshold > 1:
+            raise ValueError("outlier_threshold must be between 0 and 1")
+        
+        if self.find_q_method not in ["OHT", "gs", "bs"] and not self.find_q_method.isdigit():
+            raise ValueError("find_q_method must be 'OHT', 'gs', 'bs' or an integer string")
+        
+        if self.presence_absence and self.n_layers != 1:
+            import warnings
+            warnings.warn("Presence absence modeling is only validated with n_layers=1")
+        
+        # Set log_func and base_fn based on log_func_name
+        if self.log_func_name == "log2":
+            self.log_func = np.log2
+            self.base_fn = lambda x: 2 ** x
+        elif self.log_func_name == "log10":
+            self.log_func = np.log10
+            self.base_fn = lambda x: 10 ** x
+        elif self.log_func_name == "log":
+            self.log_func = np.log
+            self.base_fn = np.exp
+        elif self.log_func_name is None:
+            self.log_func = None
+            self.base_fn = np.exp
+        else:
+            raise ValueError(f"Log func {self.log_func_name} not supported.")
+        
+        # Set PyTorch device
+        self.device_torch = torch.device("cuda" if (torch.cuda.is_available() and self.device == 'gpu') else "cpu")
+    
+    def save(self, out_dir: Union[str, Path]) -> None:
+        """
+        Save configuration to a YAML file.
+        
+        Only serializable fields (those with init=True) are saved.
+        Computed fields like log_func, base_fn, and device_torch are excluded.
+        
+        Args:
+            out_dir: Output directory path where config.yaml will be saved
+        """
+        import logging
+        
+        logger = logging.getLogger(__name__)
+        out_dir = Path(out_dir)
+        out_p = out_dir / 'config.yaml'
+        
+        # Only save fields that are part of __init__ (exclude computed fields)
+        config_dict = self.as_dict()
+        
+        with open(out_p, 'w') as f:
+            yaml.safe_dump(config_dict, f)
+        
+        logger.info(f"Saved run config to {out_p}")
+
+    def as_dict(self) -> dict:
+        """
+        Convert the ProtriderConfig dataclass to a dictionary,
+        excluding non-serializable fields.
+        
+        Returns:
+            dict: Dictionary representation of the configuration
+        """
+        import dataclasses
+        
+        return {
+            f.name: getattr(self, f.name)
+            for f in dataclasses.fields(self)
+            if f.init  # Only include fields that are initialized (excludes computed fields)
+        }
+
+
+def load_config(config_path: Union[str, Path]) -> ProtriderConfig:
+    """
+    Load PROTRIDER configuration from a YAML file.
+    
+    Args:
+        config_path: Path to the YAML configuration file
+        
+    Returns:
+        ProtriderConfig object with validated configuration
+        
+    Raises:
+        FileNotFoundError: If config file doesn't exist
+        ValueError: If configuration is invalid
+    """
+    config_path = Path(config_path)
+    
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config file not found: {config_path}")
+    
+    with open(config_path, 'r') as f:
+        config_dict = yaml.safe_load(f)
+    
+    if config_dict is None:
+        raise ValueError(f"Empty configuration file: {config_path}")
+    
+    # Handle scientific notation that gets loaded as strings
+    if 'lr' in config_dict and isinstance(config_dict['lr'], str):
+        config_dict['lr'] = float(config_dict['lr'])
+    if 'inj_freq' in config_dict and isinstance(config_dict['inj_freq'], str):
+        config_dict['inj_freq'] = float(config_dict['inj_freq'])
+    
+    # Convert to ProtriderConfig, which will validate the fields
+    try:
+        config = ProtriderConfig(**config_dict)
+    except TypeError as e:
+        raise ValueError(f"Invalid configuration: {e}")
+    
+    return config

protrider/datasets/__init__.py

@@ -0,0 +1 @@
+from .datasets import PCADataset, ProtriderDataset, ProtriderSubset

protrider/datasets/covariates.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from pathlib import Path
+from typing import Optional
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+__all__ = ['parse_covariates']
+
+def parse_covariates(sa_file: Optional[str], cov_used: Optional[list]) -> tuple[np.ndarray, np.ndarray]:
+    """Parse covariates from sample annotation file.
+    
+    Args:
+        sa_file: Path to sample annotation file (CSV/TSV)
+        cov_used: List of covariate column names to use
+        
+    Returns:
+        tuple: (covariates, centered_covariates_noNA)
+            - covariates: Raw covariates with NAs replaced by 0
+            - centered_covariates_noNA: Centered numerical covariates with NAs replaced by 0
+    """
+    if sa_file is None:
+        raise ValueError("Sample annotation file is required.")
+    if cov_used is None:
+        raise ValueError("Covariates to use must be specified.")
+
+    # Read sample annotation file
+    sample_anno = read_annotation_file(sa_file)
+    logger.info(f'Finished reading sample annotation with shape: {sample_anno.shape}')
+    
+    # Process covariates
+    processed_covariates = _process_covariates(sample_anno[cov_used])
+    
+    # Combine all covariate types
+    covariates, centered_covariates = _combine_covariates(processed_covariates)
+    
+    # Validate output
+    assert np.isnan(covariates).sum() == 0, "Covariates contain NaN values"
+    assert np.isnan(centered_covariates).sum() == 0, "Centered covariates contain NaN values"
+    
+    return covariates, centered_covariates
+
+
+def read_annotation_file(sa_file):
+    """Read sample annotation file based on file extension."""
+    file_extension = Path(sa_file).suffix
+    if file_extension == '.csv':
+        return pd.read_csv(sa_file)
+    elif file_extension == '.tsv':
+        return pd.read_csv(sa_file, sep="\t")
+    else:
+        raise ValueError(f"Unsupported file type: {file_extension}")
+
+
+def _is_numeric_dtype(dtype):
+    """Check if pandas dtype is numeric."""
+    return pd.api.types.is_numeric_dtype(dtype)
+
+
+def _process_covariates(cov_data):
+    """Process covariates into numerical, categorical, and NA indicator variables."""
+    numerical_covs = []
+    categorical_covs = []
+    na_indicator_covs = []
+    
+    for col_name, series in cov_data.items():
+        # Skip covariates with insufficient variation
+        if series.nunique() < 2:
+            logger.warning(f"Covariate '{col_name}' has less than 2 unique values. Skipping...")
+            continue
+        
+        # Create NA indicator if needed
+        if series.isna().any():
+            na_indicator = series.isna().astype(int).to_frame(name=f'{col_name}_NA')
+            na_indicator_covs.append(na_indicator)
+        
+        # Process based on data type
+        if _is_numeric_dtype(series.dtype) and series.dtype != bool:
+            numerical_covs.append(series.to_frame())
+        else:
+            # One-hot encode categorical variables
+            categorical_encoded = pd.get_dummies(series, drop_first=True, dummy_na=False, prefix=col_name)
+            categorical_covs.append(categorical_encoded)
+    
+    # Log covariate counts
+    logger.info(f'No. numerical covariates used: {len(numerical_covs)}')
+    logger.info(f'No. categorical covariates used: {len(categorical_covs)}')
+    logger.info(f'No. NA indicator covariates used: {len(na_indicator_covs)}')
+    
+    return {
+        'numerical': numerical_covs,
+        'categorical': categorical_covs,
+        'na_indicators': na_indicator_covs
+    }
+
+
+def _combine_covariates(processed_covariates):
+    """Combine processed covariates into final arrays."""
+    covariate_arrays = []
+    centered_covariate_arrays = []
+    
+    # Process numerical covariates
+    if processed_covariates['numerical']:
+        numerical_data = pd.concat(processed_covariates['numerical'], axis=1).values
+        # Center numerical data and handle NAs
+        means = np.nanmean(numerical_data, axis=0, keepdims=True)
+        centered_numerical = numerical_data - means
+        
+        # Replace NAs with 0
+        numerical_no_na = np.where(np.isnan(numerical_data), 0, numerical_data)
+        centered_numerical_no_na = np.where(np.isnan(centered_numerical), 0, centered_numerical)
+        
+        covariate_arrays.append(numerical_no_na)
+        centered_covariate_arrays.append(centered_numerical_no_na)
+    
+    # Process categorical covariates (no centering needed)
+    if processed_covariates['categorical']:
+        categorical_data = pd.concat(processed_covariates['categorical'], axis=1).values
+        covariate_arrays.append(categorical_data)
+        centered_covariate_arrays.append(categorical_data)
+    
+    # Process NA indicators (no centering needed)
+    if processed_covariates['na_indicators']:
+        na_data = pd.concat(processed_covariates['na_indicators'], axis=1).values
+        covariate_arrays.append(na_data)
+        centered_covariate_arrays.append(na_data)
+    
+    # Concatenate all covariate types
+    if covariate_arrays:
+        covariates = np.concatenate(covariate_arrays, axis=1)
+        centered_covariates = np.concatenate(centered_covariate_arrays, axis=1)
+    else:
+        raise ValueError("No valid covariates found.")
+    
+    return covariates, centered_covariates