sortscore 0.1.0b2__py3-none-any.whl

sortscore/__init__.py

@@ -0,0 +1,19 @@
+"""
+sortscore: A modular Python package for Sort-seq variant analysis.
+
+This package provides tools for analyzing Sort-seq experimental data,
+calculating activity scores, and generating visualizations.
+"""
+
+__version__ = "0.1.0b2"
+__author__ = "Caitlyn Chitwood"
+__email__ = "c.chitwood@wustl.edu"
+
+# Import main classes for convenience
+from .utils.load_experiment import ExperimentConfig
+from .analysis.score import calculate_full_activity_scores
+
+__all__ = [
+    "ExperimentConfig", 
+    "calculate_full_activity_scores"
+]

sortscore/__main__.py

@@ -0,0 +1,7 @@
+"""
+Allow running the sortscore package as a module with: python -m sortscore
+"""
+from sortscore.cli import main
+
+if __name__ == "__main__":
+    main()

sortscore/analysis/__init__.py

@@ -0,0 +1,7 @@
+"""
+Analysis subpackage for Sort-seq variant analysis workflows.
+
+This subpackage contains modules for configuration, parameters, I/O, and utilities.
+"""
+
+from .score import calculate_activity_scores

sortscore/analysis/aa_scores.py

@@ -0,0 +1,221 @@
+"""
+Amino acid scores processing and export for Sort-seq analysis.
+
+This module provides functions for processing amino acid scores from DNA variant data,
+including aggregation of synonymous codons, statistical analysis, and file export.
+"""
+import os
+import logging
+import pandas as pd
+import numpy as np
+from scipy import stats as scipy_stats
+from typing import Tuple, List
+from sortscore.analysis.statistics import calculate_codon_and_replicate_variance
+
+
+def process_and_save_aa_scores(scores_df: pd.DataFrame, experiment, scores_dir: str, 
+                              output_suffix: str, analysis_logger) -> None:
+    """
+    Process and save amino acid scores from variant data.
+    
+    This function handles the complete AA scores workflow including:
+    - Filtering out NaN values
+    - Checking if codon aggregation is needed
+    - Calculating appropriate statistics (with/without codon variance)
+    - Rounding score columns
+    - Saving to CSV file
+    - Logging output
+    
+    Parameters
+    ----------
+    scores_df : pd.DataFrame
+        DataFrame containing variant scores and annotations
+    experiment : ExperimentConfig
+        Experiment configuration containing metadata
+    scores_dir : str
+        Directory to save scores file
+    output_suffix : str
+        Suffix for output filename
+    analysis_logger : AnalysisLogger
+        Logger instance for recording outputs
+        
+    Examples
+    --------
+    >>> process_and_save_aa_scores(scores_df, experiment, 'output/scores', 'suffix', logger)
+    """
+    if 'aa_seq_diff' not in scores_df.columns:
+        return
+        
+    # Determine score column
+    if experiment.avg_method == 'simple-avg':
+        score_col = 'avgscore'
+    else:
+        score_col_suffix = experiment.avg_method.replace('-', '_')
+        score_col = f'avgscore_{score_col_suffix}'
+    
+    # Filter out rows with NaN values first
+    scores_df_drop_nan = scores_df.dropna(subset=[score_col])
+    
+    # Find replicate score columns
+    rep_score_columns = [col for col in scores_df_drop_nan.columns 
+                        if col.startswith('Rep') and col.endswith('.score')]
+    
+    # Check aggregation needs and process scores
+    aa_scores = _check_codon_num(scores_df_drop_nan, score_col, rep_score_columns)
+    
+    # Round score columns to integers
+    aa_scores = _round_score_columns(aa_scores)
+    
+    # Save to file
+    aa_scores_file = os.path.join(scores_dir, f"{experiment.experiment_name}_aa_scores_{output_suffix}.csv")
+    aa_scores.to_csv(aa_scores_file, index=False)
+    logging.info(f"Saved AA scores to {aa_scores_file} ({len(aa_scores)} unique AA variants)")
+    
+    # Log file output  
+    analysis_logger.log_output_file(
+        'aa_scores',
+        f"{experiment.experiment_name}_aa_scores_{output_suffix}.csv", 
+        aa_scores_file,
+        variant_count=len(aa_scores)
+    )
+
+
+def _check_codon_num(scores_df_drop_nan: pd.DataFrame, score_col: str, 
+                                  rep_score_columns: List[str]) -> pd.DataFrame:
+    """
+    Check if codon aggregation is needed and process AA scores accordingly.
+    
+    This function checks if there are multiple codons per AA variant and processes
+    the data using either DNA->AA aggregation (with codon variance) or AA-only
+    statistics (replicate variance only).
+    
+    Parameters
+    ----------
+    scores_df_drop_nan : pd.DataFrame
+        DataFrame with NaN values already filtered out
+    score_col : str
+        Name of the score column to use
+    rep_score_columns : List[str]
+        List of replicate score column names
+        
+    Returns
+    -------
+    pd.DataFrame
+        Processed AA scores with appropriate statistics
+    """
+    # Check if there are multiple codons per AA variant (DNA->AA case)
+    aa_variant_counts = scores_df_drop_nan.groupby(['aa_seq_diff', 'annotate_aa']).size()
+    needs_aggregation = (aa_variant_counts > 1).any()
+    
+    if needs_aggregation:
+        return _process_dna_to_aa_aggregation(scores_df_drop_nan, score_col, rep_score_columns)
+    else:
+        return _process_aa_only_scores(scores_df_drop_nan, rep_score_columns)
+
+
+def _process_dna_to_aa_aggregation(scores_df_drop_nan: pd.DataFrame, score_col: str, 
+                                  rep_score_columns: List[str]) -> pd.DataFrame:
+    """
+    Process DNA->AA aggregation case with codon variance decomposition.
+    
+    Parameters
+    ----------
+    scores_df_drop_nan : pd.DataFrame
+        DataFrame with variant scores (NaN filtered)
+    score_col : str
+        Name of the score column to use
+    rep_score_columns : List[str]
+        List of replicate score column names
+        
+    Returns
+    -------
+    pd.DataFrame
+        Aggregated AA scores with codon and replicate statistics
+    """
+    # DNA->AA aggregation case: aggregate synonymous variants
+    columns_to_average = ['avgscore', 'avgscore_rep_weighted'] + rep_score_columns
+    
+    # Calculate standard deviation and count of codon-level scores before AA aggregation
+    aa_scores_std = scores_df_drop_nan.groupby(['aa_seq_diff', 'annotate_aa'])[score_col].agg(['std', 'count']).reset_index()
+    aa_scores_std.columns = ['aa_seq_diff', 'annotate_aa', 'SD_codon', 'n_codons']
+    
+    # Calculate mean scores for aggregation
+    aa_scores = scores_df_drop_nan.groupby(['aa_seq_diff', 'annotate_aa'])[columns_to_average].mean().reset_index()
+    
+    # Merge the standard deviation and count of codon scores
+    aa_scores = aa_scores.merge(aa_scores_std, on=['aa_seq_diff', 'annotate_aa'], how='left')
+    
+    # Calculate statistics with codon and replicate variance decomposition
+    aa_scores = calculate_codon_and_replicate_variance(aa_scores, rep_score_columns)
+    
+    return aa_scores
+
+
+def _process_aa_only_scores(scores_df_drop_nan: pd.DataFrame, rep_score_columns: List[str]) -> pd.DataFrame:
+    """
+    Process AA-only case with simple replicate statistics.
+    
+    Parameters
+    ----------
+    scores_df_drop_nan : pd.DataFrame
+        DataFrame with variant scores (NaN filtered)
+    rep_score_columns : List[str]
+        List of replicate score column names
+        
+    Returns
+    -------
+    pd.DataFrame
+        AA scores with replicate statistics only
+    """
+    # AA-only case: no aggregation needed, just copy the data
+    columns_to_include = ['aa_seq_diff', 'annotate_aa', 'avgscore', 'avgscore_rep_weighted'] + rep_score_columns
+    
+    aa_scores = scores_df_drop_nan[columns_to_include].copy()
+    
+    # Calculate simple replicate statistics (no codon variance)
+    if len(rep_score_columns) >= 2:
+        aa_rep_mean = aa_scores[rep_score_columns].mean(axis=1)
+        aa_rep_std = aa_scores[rep_score_columns].std(axis=1, ddof=1)
+        
+        # Calculate n_measurements (just number of non-empty replicates)
+        n_measurements = aa_scores[rep_score_columns].notna().sum(axis=1)
+        
+        # Calculate SEM using only replicate variance
+        sem = aa_rep_std / np.sqrt(n_measurements)
+        
+        # Calculate 95% CI using t-distribution
+        df_actual = n_measurements - 1
+        t_critical = scipy_stats.t.ppf(0.975, df_actual)
+        aa_margin_of_error = t_critical * sem
+        
+        aa_scores['SD_rep'] = aa_rep_std.round().astype('Int64')
+        aa_scores['CV_rep'] = (aa_rep_std / aa_rep_mean).round(3)
+        aa_scores['n_measurements'] = n_measurements.astype('Int64')
+        aa_scores['SEM'] = sem.round().astype('Int64')
+        aa_scores['CI_lower'] = (aa_rep_mean - aa_margin_of_error).round().astype('Int64')
+        aa_scores['CI_upper'] = (aa_rep_mean + aa_margin_of_error).round().astype('Int64')
+    
+    return aa_scores
+
+
+def _round_score_columns(aa_scores: pd.DataFrame) -> pd.DataFrame:
+    """
+    Round score columns to integers for cleaner output.
+    
+    Parameters
+    ----------
+    aa_scores : pd.DataFrame
+        DataFrame containing score columns
+        
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame with score columns rounded to integers
+    """
+    # Round score columns to integers
+    score_columns = [col for col in aa_scores.columns if 'score' in col.lower()]
+    for col in score_columns:
+        if aa_scores[col].dtype in ['float64', 'float32']:
+            aa_scores[col] = aa_scores[col].round().astype('Int64')
+    
+    return aa_scores

sortscore/analysis/annotation.py

@@ -0,0 +1,191 @@
+"""
+Sequence annotation utilities for Sort-seq variant analysis.
+
+This module provides functions for annotating variant DataFrames with sequence differences,
+translations, and other derived sequence information.
+
+Examples
+--------
+>>> from sortscore.analysis.annotation import annotate_scores_dataframe
+>>> annotated_df = annotate_scores_dataframe(scores_df, experiment)
+"""
+import pandas as pd
+from sortscore.utils.sequence_parsing import compare_to_reference, compare_codon_lists, translate_dna
+
+# TODO: #37 redundant, see if we can remove
+def annotate_scores_dataframe(
+    scores_df: pd.DataFrame, 
+    wt_dna_seq: str, 
+    mutagenesis_type: str = 'aa',
+) -> pd.DataFrame:
+    """
+    Add sequence annotation columns to a scores DataFrame.
+    
+    Parameters
+    ----------
+    scores_df : pd.DataFrame
+        DataFrame with variant sequences and scores.
+    wt_dna_seq : str
+        Wild-type DNA reference sequence.
+    mutagenesis_type : str, default 'aa'
+        Type of mutagenesis ('codon', 'snv', 'aa').
+    
+    Returns
+    -------
+    annotated_df : pd.DataFrame
+        DataFrame with added annotation columns.
+        
+    Examples
+    --------
+    >>> annotated_df = annotate_scores_dataframe(scores_df, wt_seq, 'dna')
+    """
+    df = scores_df.copy()
+    
+    # Check if aa_seq_diff already exists (from pre-annotated data)
+    has_pre_annotated_aa = 'aa_seq_diff' in df.columns
+    
+    # Treat 'dna' as a DNA-sequence variant type (full-length DNA sequences)
+    if mutagenesis_type in {'codon', 'snv'}:
+        # Add codon differences
+        df['codon_diff'] = df['variant_seq'].apply(
+            lambda x: compare_codon_lists(wt_dna_seq, x)
+        )
+        df['codon_diff'] = df['codon_diff'].fillna('')
+        
+        # Add DNA sequence differences
+        df['dna_seq_diff'] = df['variant_seq'].apply(
+            lambda x: compare_to_reference(wt_dna_seq, x)
+        )
+        df['dna_seq_diff'] = df['dna_seq_diff'].fillna('')
+        
+        # Add AA sequence annotations only if not pre-annotated
+        if not has_pre_annotated_aa:
+            wt_aa_seq = translate_dna(wt_dna_seq)
+            df['aa_seq'] = df['variant_seq'].apply(translate_dna)
+            df['aa_seq_diff'] = df['aa_seq'].apply(
+                lambda x: compare_to_reference(wt_aa_seq, x)
+            )
+            df['aa_seq_diff'] = df['aa_seq_diff'].fillna('')
+        
+    elif mutagenesis_type == 'aa':
+        # For AA variants, add sequence differences only if not pre-annotated
+        if not has_pre_annotated_aa:
+            wt_aa_seq = translate_dna(wt_dna_seq) if len(wt_dna_seq) % 3 == 0 else wt_dna_seq
+            df['aa_seq_diff'] = df['variant_seq'].apply(
+                lambda x: compare_to_reference(wt_aa_seq, x)
+            )
+            df['aa_seq_diff'] = df['aa_seq_diff'].fillna('')
+    
+    # Map stop codon representations to * for standard notation in aa_seq_diff column
+    if 'aa_seq_diff' in df.columns:
+        df['aa_seq_diff'] = df['aa_seq_diff'].str.replace('X', '*', regex=False)
+        df['aa_seq_diff'] = df['aa_seq_diff'].str.replace('Ter', '*', regex=False)
+    
+    # Add functional annotations
+    df = add_variant_categories(df)
+    
+    return df
+
+# TODO: #37 isn't this redundant with similar functions
+def add_sequence_differences(
+    df: pd.DataFrame,
+    wt_dna_seq: str,
+    mutagenesis_type: str = 'aa',
+) -> pd.DataFrame:
+    """
+    Add sequence difference columns to a DataFrame.
+    
+    Parameters
+    ----------
+    df : pd.DataFrame
+        DataFrame with variant sequences.
+    wt_dna_seq : str
+        Wild-type DNA sequence.
+    mutagenesis_type : str, default 'aa'
+        Type of mutagenesis ('codon', 'snv', 'aa').
+        
+    Returns
+    -------
+    df : pd.DataFrame
+        DataFrame with sequence difference columns added.
+    """
+    df = df.copy()
+    
+    if mutagenesis_type in {'codon', 'snv'}:
+        # Add DNA sequence differences
+        df['dna_seq_diff'] = df['variant_seq'].apply(
+            lambda x: compare_to_reference(wt_dna_seq, x)
+        )
+        df['dna_seq_diff'] = df['dna_seq_diff'].fillna('')
+        
+        # Add AA sequence differences
+        wt_aa_seq = translate_dna(wt_dna_seq)
+        df['aa_seq'] = df['variant_seq'].apply(translate_dna)
+        df['aa_seq_diff'] = df['aa_seq'].apply(
+            lambda x: compare_to_reference(wt_aa_seq, x)
+        )
+        df['aa_seq_diff'] = df['aa_seq_diff'].fillna('')
+        
+    elif mutagenesis_type == 'aa':
+        # For AA variants, sequences are already amino acids
+        wt_aa_seq = translate_dna(wt_dna_seq) if len(wt_dna_seq) % 3 == 0 else wt_dna_seq
+        df['aa_seq_diff'] = df['variant_seq'].apply(
+            lambda x: compare_to_reference(wt_aa_seq, x)
+        )
+        df['aa_seq_diff'] = df['aa_seq_diff'].fillna('')
+    
+    return df
+
+def classify_aa_variant(aa_diff, dna_diff=None):
+    if not aa_diff or aa_diff == '':
+        # Check if this is true WT (no DNA changes) or synonymous (DNA changes but same AA)
+        if dna_diff is not None and (not dna_diff or dna_diff == ''):
+            return 'wt_dna'
+        else:
+            return 'synonymous'
+    elif '*' in aa_diff:
+        return 'nonsense'
+    else:
+        return 'missense_aa'
+
+def classify_dna_variant(dna_diff, aa_diff):
+    if not dna_diff or dna_diff == '':
+        return 'wt_dna'
+    elif not aa_diff or aa_diff == '':
+        return 'synonymous'
+    else:
+        return 'missense_dna'
+
+
+def add_variant_categories(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Add variant category annotations based on existing sequence difference columns.
+    
+    Parameters
+    ----------
+    df : pd.DataFrame
+        DataFrame with sequence difference columns (aa_seq_diff, dna_seq_diff).
+        
+    Returns
+    -------
+    df : pd.DataFrame
+        DataFrame with variant category columns added.
+    """
+    df = df.copy()
+    
+    # Classify DNA variants 
+    if 'dna_seq_diff' in df.columns:
+        if 'aa_seq_diff' in df.columns:
+            df['annotate_dna'] = df.apply(lambda row: classify_dna_variant(row['dna_seq_diff'], row['aa_seq_diff']), axis=1)
+        else:
+            df['annotate_dna'] = df['dna_seq_diff'].apply(lambda x: 'missense_dna' if x else 'wt_dna')
+    
+    
+    # Classify variants based on AA changes
+    if 'aa_seq_diff' in df.columns:
+        if 'dna_seq_diff' in df.columns:
+            df['annotate_aa'] = df.apply(lambda row: classify_aa_variant(row['aa_seq_diff'], row['dna_seq_diff']), axis=1)
+        else:
+            df['annotate_aa'] = df['aa_seq_diff'].apply(classify_aa_variant)
+
+    return df

sortscore/analysis/batch_config.py

@@ -0,0 +1,170 @@
+"""
+Batch configuration module for Sort-seq variant analysis.
+
+This module provides framework for combining and normalizing multiple 
+Sort-seq experiments in batch processing workflows.
+"""
+
+import json
+from pathlib import Path
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Any
+
+
+@dataclass
+class BatchConfig:
+    """
+    Dataclass for batch processing configuration.
+    
+    This class manages configuration for combining and normalizing multiple Sort-seq experiments.
+    It supports two normalization methods:
+    
+    1. **2-pole normalization**: Uses synonymous and pathogenic variants as reference points
+    2. **Z-score scaled three-step normalization** (default): Creates standardized scale where synonymous 
+       variants center around 0 with unit variance, making cross-experiment comparisons meaningful
+    
+    Attributes
+    ----------
+    batch_normalization_method : str, optional
+        Normalization method to use ('zscore_2pole', '2pole', or 'zscore_center'), default 'zscore_2pole'
+    pathogenic_control_type : str, optional
+        Type of pathogenic control ('nonsense' or 'custom'), default 'nonsense'  
+    pathogenic_variants : Optional[List[str]], optional
+        Custom pathogenic variants if using 'custom' pathogenic_control_type
+    combined_output_dir : str, optional
+        Directory for final combined results, default current directory
+    """
+    experiments: List[Dict[str, Any]]
+    batch_normalization_method: str = 'zscore_2pole'
+    pathogenic_control_type: str = 'nonsense'
+    pathogenic_variants: Optional[List[str]] = None
+    combined_output_dir: str = './normalized'
+    
+    @staticmethod
+    def from_json(json_path: str) -> 'BatchConfig':
+        """
+        Load batch configuration from a JSON file.
+        
+        Parameters
+        ----------
+        json_path : str
+            Path to the JSON batch config file
+            
+        Returns
+        -------
+        config : BatchConfig
+            Loaded batch configuration
+        """
+        json_path_obj = Path(json_path).expanduser().resolve()
+        with open(json_path_obj, 'r') as f:
+            data = json.load(f)
+        if 'experiments' not in data:
+            raise ValueError("Batch configuration requires 'experiments'")
+        args: Dict[str, Any] = {'experiments': data['experiments']}
+        
+        # Optional fields (only add if present to preserve defaults)
+        optional_fields = [
+            'batch_normalization_method', 'pathogenic_control_type', 
+            'pathogenic_variants', 'combined_output_dir'
+        ]
+        
+        for field in optional_fields:
+            if field in data:
+                args[field] = data[field]
+
+        # Resolve output paths relative to the config file directory.
+        config_dir = json_path_obj.parent
+        experiments = []
+        for entry in args['experiments']:
+            entry_cfg = dict(entry)
+            if 'output_dir' in entry_cfg and entry_cfg['output_dir'] is not None:
+                entry_cfg['output_dir'] = str(
+                    (config_dir / Path(str(entry_cfg['output_dir'])).expanduser()).resolve()
+                )
+            experiments.append(entry_cfg)
+        args['experiments'] = experiments
+
+        if 'combined_output_dir' in args and args['combined_output_dir'] is not None:
+            args['combined_output_dir'] = str(
+                (config_dir / Path(str(args['combined_output_dir'])).expanduser()).resolve()
+            )
+        
+        return BatchConfig(**args)
+    
+    def validate_config(self) -> None:
+        """
+        Validate batch configuration parameters.
+        
+        Raises
+        ------
+        ValueError
+            If configuration parameters are invalid
+        FileNotFoundError
+            If experiment config files don't exist
+        """
+        if not self.experiments:
+            raise ValueError("experiments must contain at least one entry")
+
+        seen_tiles = set()
+        for idx, cfg in enumerate(self.experiments):
+            if 'tile' not in cfg:
+                raise ValueError(f"experiments[{idx}] missing 'tile'")
+            if 'output_dir' not in cfg:
+                raise ValueError(f"experiments[{idx}] missing 'output_dir'")
+            if 'wt_seq' not in cfg:
+                raise ValueError(f"experiments[{idx}] missing 'wt_seq'")
+            if 'min_pos' not in cfg:
+                raise ValueError(f"experiments[{idx}] missing 'min_pos'")
+            if 'max_pos' not in cfg:
+                raise ValueError(f"experiments[{idx}] missing 'max_pos'")
+            try:
+                tile = int(cfg['tile'])
+            except Exception as e:
+                raise ValueError(f"experiments[{idx}].tile must be int-like") from e
+            try:
+                min_pos = int(cfg['min_pos'])
+                max_pos = int(cfg['max_pos'])
+            except Exception as e:
+                raise ValueError(f"experiments[{idx}].min_pos/max_pos must be int-like") from e
+            if min_pos >= max_pos:
+                raise ValueError(f"experiments[{idx}] min_pos must be less than max_pos")
+            if tile in seen_tiles:
+                raise ValueError(f"Duplicate tile value in experiments: {tile}")
+            seen_tiles.add(tile)
+
+            out_dir = Path(str(cfg['output_dir'])).expanduser().resolve()
+            if not out_dir.exists():
+                raise FileNotFoundError(f"experiments[{idx}] output_dir does not exist: {out_dir}")
+        
+        # Validate normalization method
+        valid_methods = ['zscore_2pole', '2pole', 'zscore_center']
+        if self.batch_normalization_method not in valid_methods:
+            raise ValueError(f"Invalid normalization method: {self.batch_normalization_method}. "
+                           f"Must be one of: {valid_methods}")
+        
+        # Validate pathogenic control type
+        valid_control_types = ['nonsense', 'custom']
+        if self.pathogenic_control_type not in valid_control_types:
+            raise ValueError(f"Invalid pathogenic control type: {self.pathogenic_control_type}. "
+                           f"Must be one of: {valid_control_types}")
+        
+        # If using custom pathogenic controls, ensure variants are specified
+        if self.pathogenic_control_type == 'custom' and not self.pathogenic_variants:
+            raise ValueError("pathogenic_variants must be specified when using 'custom' pathogenic_control_type")
+        
+    def get_batch_config_dict(self) -> Dict[str, Any]:
+        """
+        Convert batch configuration to dictionary format for processing.
+        
+        Returns
+        -------
+        Dict[str, Any]
+            Configuration dictionary for batch processing functions
+        """
+        return {
+            'experiments': self.experiments,
+            'batch_normalization_method': self.batch_normalization_method,
+            'pathogenic_control_type': self.pathogenic_control_type,
+            'pathogenic_variants': self.pathogenic_variants,
+            'combined_output_dir': self.combined_output_dir
+        }