mutcleaner 0.1.0__py3-none-any.whl

mutcleaner/__init__.py

@@ -0,0 +1,58 @@
+# mutcleaner/__init__.py
+"""
+MutCleaner: An efficient framework for cleaning, standardizing, and processing biological mutation data.
+"""
+
+__author__ = "Yuxiang Tang and Ziyu Shi"
+
+__version__ = "0.1.0"
+
+from .core import (
+    # Alphabet
+    alphabet,
+    # Codon
+    codon,
+    # Mutation
+    mutation,
+    # Sequence
+    sequence,
+    # Dataset
+    MutationDataset,
+    # Pipeline
+    Pipeline,
+    pipeline_step,
+    multiout_step,
+    create_pipeline,
+)
+
+from .cleaners import (
+    basic_cleaners,
+    cdna_proteolysis_cleaner,
+    human_domainome_sup2_cleaner,
+    proteingym_dms_substitutions_cleaner,
+    ddg_dtm_cleaners,
+    archstabms_1e10_cleaner,
+    human_myoglobin_cleaner,
+    ctxm_cleaner,
+    trpb_cleaner,
+    antitoxin_pard3_cleaner,
+    rbd_antibody_cleaner,
+    rbd_ace2_cleaner,
+)
+
+from .utils import (
+    download,
+    download_cdna_proteolysis_source_file,
+    download_proteingym_source_file,
+    download_human_domainome_source_file,
+    download_ddg_dtm_source_file,
+    list_datasets_with_built_in_cleaners,
+    show_download_instructions,
+    download_archstabms1e10_source_file,
+    download_human_myoglobin_source_file,
+    download_ctxm_source_file,
+    download_trpb_source_file,
+    download_antitoxin_pard3_source_file,
+    download_rbd_antibody_source_file,
+    download_rbd_ace2_source_file,
+)

mutcleaner/cleaners/__init__.py

@@ -0,0 +1,108 @@
+"""Dataset-specific cleaning pipelines for MutCleaner."""
+
+from .cdna_proteolysis_cleaner import (
+    CDNAProteolysisCleanerConfig,
+    create_cdna_proteolysis_cleaner,
+    clean_cdna_proteolysis_dataset,
+)
+
+from .proteingym_dms_substitutions_cleaner import (
+    ProteinGymCleanerConfig,
+    create_proteingym_dms_substitutions_cleaner,
+    clean_proteingym_dms_substitutions_dataset,
+)
+
+from .human_domainome_sup2_cleaner import (
+    HumanDomainomeSup2CleanerConfig,
+    create_human_domainome_sup2_cleaner,
+    clean_human_domainome_sup2_dataset,
+)
+
+from .ddg_dtm_cleaners import (
+    DdgDtmCleanerConfig,
+    create_ddg_dtm_cleaner,
+    clean_ddg_dtm_dataset,
+)
+
+from .archstabms_1e10_cleaner import (
+    ArchStabMS1E10CleanerConfig,
+    create_archstabms_1e10_cleaner,
+    clean_archstabms_1e10_dataset,
+)
+
+from .antitoxin_pard3_cleaner import (
+    AntitoxinParD3CleanerConfig,
+    create_antitoxin_pard3_cleaner,
+    clean_antitoxin_pard3_dataset,
+)
+
+from .trpb_cleaner import (
+    TrpBCleanerConfig,
+    create_trpb_cleaner,
+    clean_trpb_dataset,
+)
+
+from .ctxm_cleaner import (
+    CTXMCleanerConfig,
+    create_ctxm_cleaner,
+    clean_ctxm_dataset,
+)
+
+from .human_myoglobin_cleaner import (
+    HumanMyoglobinCleanerConfig,
+    create_human_myoglobin_cleaner,
+    clean_human_myoglobin_dataset,
+)
+
+from .rbd_antibody_cleaner import (
+    RBDAntibodyCleanerConfig,
+    create_rbd_antibody_cleaner,
+    clean_rbd_antibody_dataset,
+)
+
+from .rbd_ace2_cleaner import (
+    RBDACE2CleanerConfig,
+    create_rbd_ace2_cleaner,
+    clean_rbd_ace2_dataset,
+)
+
+
+
+__all__ = [
+    "create_cdna_proteolysis_cleaner",
+    "clean_cdna_proteolysis_dataset",
+    "CDNAProteolysisCleanerConfig",
+    "create_proteingym_dms_substitutions_cleaner",
+    "clean_proteingym_dms_substitutions_dataset",
+    "ProteinGymCleanerConfig",
+    "create_human_domainome_sup2_cleaner",
+    "clean_human_domainome_sup2_dataset",
+    "HumanDomainomeSup2CleanerConfig",
+    "create_ddg_dtm_cleaner",
+    "clean_ddg_dtm_dataset",
+    "DdgDtmCleanerConfig",
+    "ArchStabMS1E10CleanerConfig",
+    "create_archstabms_1e10_cleaner",
+    "clean_archstabms_1e10_dataset",
+    "ArchStabMS1E10CleanerConfig",
+    "create_archstabms_1e10_cleaner",
+    "clean_archstabms_1e10_dataset",
+    "AntitoxinParD3CleanerConfig",
+    "create_antitoxin_pard3_cleaner",
+    "clean_antitoxin_pard3_dataset",
+    "TrpBCleanerConfig",
+    "create_trpb_cleaner",
+    "clean_trpb_dataset",
+    "CTXMCleanerConfig",
+    "create_ctxm_cleaner",
+    "clean_ctxm_dataset",
+    "HumanMyoglobinCleanerConfig",
+    "create_human_myoglobin_cleaner",
+    "clean_human_myoglobin_dataset",
+    "RBDAntibodyCleanerConfig",
+    "create_rbd_antibody_cleaner",
+    "clean_rbd_antibody_dataset",
+    "RBDACE2CleanerConfig",
+    "create_rbd_ace2_cleaner",
+    "clean_rbd_ace2_dataset",
+]

mutcleaner/cleaners/antitoxin_pard3_cleaner.py

@@ -0,0 +1,341 @@
+# mutcleaner/cleaners/antitoxin_pard3_cleaner.py
+from __future__ import annotations
+
+import pandas as pd
+from typing import TYPE_CHECKING
+from dataclasses import dataclass, field
+from pathlib import Path
+import logging
+
+from .base_config import BaseCleanerConfig
+from .basic_cleaners import (
+    read_dataset,
+    extract_and_rename_columns,
+    filter_and_clean_data,
+    convert_data_types,
+    convert_to_mutation_dataset_format,
+    validate_mutations,
+    add_columns,
+    apply_mutations_to_sequences,
+    average_labels_by_name,
+    subtract_labels_by_wt,
+)
+from .antitoxin_pard3_custom_cleaners import (
+    simplify_mutations,
+)
+
+from ..core.dataset import MutationDataset
+from ..core.pipeline import Pipeline, create_pipeline
+
+if TYPE_CHECKING:
+    from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+__all__ = [
+    "AntitoxinParD3CleanerConfig",
+    "create_antitoxin_pard3_cleaner",
+    "clean_antitoxin_pard3_dataset",
+]
+
+
+def __dir__() -> List[str]:
+    return __all__
+
+
+# Create module logger
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class AntitoxinParD3CleanerConfig(BaseCleanerConfig):
+    """
+    Configuration class for Antitoxin dataset cleaner.
+    Inherits from BaseCleanerConfig and adds Antitoxin-specific configuration options.
+
+    Simply run `mutcleaner.download_antitoxin_source_file()` to download the dataset.
+
+    Alternatively, the raw Antitoxin file can be obtained from:
+
+    - Hugging Face: https://huggingface.co/datasets/xulab-research/TidyMut/blob/main/antitoxin/antitoxin.csv
+
+    Attributes
+    ----------
+    column_mapping : Dict[str, str]
+        Mapping from source to target column names
+    filters : Dict[str, Callable]
+        Filter conditions for data cleaning
+    wt_sequence : str
+        Wildtype sequence for the dataset, used for mutation validation
+    type_conversions : Dict[str, str]
+        Data type conversion specifications
+    validate_mut_workers : int
+        Number of workers for mutation validation, set to -1 to use all available CPUs
+    process_workers : int
+        Number of workers for applying mutations to sequences, set to -1 to use all available CPUs
+    label_columns : List[str]
+        List of score columns to process
+    primary_label_column : str
+        Primary score column for the dataset
+    pipeline_name : str
+        Name of the cleaning pipeline
+    """
+
+    # Column mapping configuration
+    column_mapping: Dict[str, str] = field(
+        default_factory=lambda: {
+            "mutation": "mut_info",
+            "label": "label",
+        }
+    )
+
+    # Data filtering configuration
+    filters: Dict[str, Callable] = field(
+        default_factory=lambda: {
+            "label": lambda s: pd.to_numeric(s, errors="coerce").notna()
+        }
+    )
+
+    # obtained from the article
+    wt_sequence = "MANVEKMSVAVTPQQAAVMREAVEAGEYATASEIVREAVRDWLAKRELRHDDIRRLRQLWDEGKASGRPEPVDFDALRKEARQKLTEVPPNGR"
+
+    # Type conversion configuration
+    type_conversions: Dict[str, str] = field(default_factory=lambda: {"label": "float"})
+
+    # Mutation validation parameters
+    validate_mut_workers: int = 16
+
+    process_workers: int = 16
+
+    # Score columns configuration
+    label_columns: List[str] = field(default_factory=lambda: ["label"])
+    
+    primary_label_column: str = "label"
+
+    # Override default pipeline name
+    pipeline_name: str = "Antitoxin Pipeline"
+
+    def validate(self) -> None:
+        """Validate Antitoxin-specific configuration parameters
+
+        Raises
+        ------
+        ValueError
+            If configuration is invalid
+        """
+        # Call parent validation
+        super().validate()
+
+        # Validate score columns
+        if not self.label_columns:
+            raise ValueError("label_columns cannot be empty")
+
+        if self.primary_label_column not in self.label_columns:
+            raise ValueError(
+                f"primary_label_column '{self.primary_label_column}' "
+                f"must be in label_columns {self.label_columns}"
+            )
+
+        # Validate column mapping
+        required_mappings = {"mutation"}
+        missing = required_mappings - set(self.column_mapping.keys())
+        if missing:
+            raise ValueError(f"Missing required column mappings: {missing}")
+
+
+def create_antitoxin_pard3_cleaner(
+    dataset_or_path: Optional[Union[pd.DataFrame, str, Path]] = None,
+    config: Optional[
+        Union[AntitoxinParD3CleanerConfig, Dict[str, Any], str, Path]
+    ] = None,
+) -> Pipeline:
+    """Create Antitoxin dataset cleaning pipeline
+
+    Parameters
+    ----------
+    dataset_or_path : Optional[Union[pd.DataFrame, str, Path]], default=None
+        Raw dataset DataFrame or file path to Antitoxin dataset.
+    config : Optional[Union[AntitoxinCleanerConfig, Dict[str, Any], str, Path]]
+        Configuration for the cleaning pipeline. Can be:
+        - AntitoxinCleanerConfig object
+        - Dictionary with configuration parameters (merged with defaults)
+        - Path to JSON configuration file (str or Path)
+        - None (uses default configuration)
+
+    Returns
+    -------
+    Pipeline
+        Pipeline: The cleaning pipeline used
+
+    Raises
+    ------
+    TypeError
+        If config has invalid type
+    ValueError
+        If configuration validation fails
+    """
+    # Handle configuration parameter
+    if config is None:
+        final_config = AntitoxinParD3CleanerConfig()
+    elif isinstance(config, AntitoxinParD3CleanerConfig):
+        final_config = config
+    elif isinstance(config, dict):
+        # Partial configuration - merge with defaults
+        default_config = AntitoxinParD3CleanerConfig()
+        final_config = default_config.merge(config)
+    elif isinstance(config, (str, Path)):
+        # Load from file
+        final_config = AntitoxinParD3CleanerConfig.from_json(config)
+    else:
+        raise TypeError(
+            f"config must be AntitoxinParD3CleanerConfig, dict, str, Path or None, "
+            f"got {type(config)}"
+        )
+
+    # Log configuration summary
+    logger.info(
+        f"Antitoxin dataset will be cleaned with pipeline: {final_config.pipeline_name}"
+    )
+    logger.debug(f"Configuration:\n{final_config.get_summary()}")
+
+    try:
+        # Create pipeline
+        pipeline = create_pipeline(dataset_or_path, final_config.pipeline_name)
+
+        # Add cleaning steps
+        pipeline = (
+            pipeline.delayed_then(
+                extract_and_rename_columns,
+                column_mapping=final_config.column_mapping,
+            )
+            .delayed_then(
+                filter_and_clean_data,
+                filters=final_config.filters,
+            )
+            .delayed_then(
+                convert_data_types,
+                type_conversions=final_config.type_conversions,
+            )
+            .delayed_then(
+                add_columns, 
+                columns_to_add={
+                    "antitoxin": "name",
+                    "wt_seq": final_config.wt_sequence,
+                },
+            )
+            .delayed_then(
+                simplify_mutations,
+                mutation_column=final_config.column_mapping.get("mutation", "mutation"),
+                mutation_sep=":",
+            )
+            .delayed_then(
+                validate_mutations,
+                mutation_column=final_config.column_mapping.get("mutation", "mutation"),
+                mutation_sep=",",
+                is_zero_based=True,
+                exclude_patterns="WT",
+                num_workers=final_config.validate_mut_workers,
+            )
+            .delayed_then(
+                average_labels_by_name,
+                name_columns=(
+                    "name",
+                    final_config.column_mapping.get("mutation", "mutation"),
+                ),
+                label_columns=final_config.primary_label_column,
+            )
+            .delayed_then(
+                subtract_labels_by_wt,
+                name_column="name",
+                label_columns=final_config.primary_label_column,
+                mutation_column=final_config.column_mapping.get("mutation", "mutation"),
+                wt_identifier="WT",
+                in_place=True,
+            )
+            .delayed_then(
+                apply_mutations_to_sequences,
+                sequence_column="wt_seq",
+                name_column="name",
+                mutation_column=final_config.column_mapping.get("mutation", "mutation"),
+                is_zero_based=True,
+                sequence_type="protein",
+                num_workers=final_config.process_workers,
+            )
+            .delayed_then(
+                convert_to_mutation_dataset_format,
+                name_column="name",
+                mutation_column=final_config.column_mapping.get("mutation", "mutation"),
+                sequence_column="wt_seq",
+                label_column=final_config.primary_label_column,
+                is_zero_based=True,
+            )
+        )
+
+        # Create pipeline based on dataset_or_path type
+        if isinstance(dataset_or_path, (str, Path)):
+            pipeline.add_delayed_step(read_dataset, 0, file_format="csv")
+        elif not isinstance(dataset_or_path, pd.DataFrame):
+            raise TypeError(
+                f"dataset_or_path must be pd.DataFrame or str/Path, "
+                f"got {type(dataset_or_path)}"
+            )
+
+        return pipeline
+
+    except Exception as e:
+        logger.error(f"Error in creating aav capsid cleaning pipeline: {str(e)}")
+        raise RuntimeError(f"Error in creating aav capsid cleaning pipeline: {str(e)}")
+
+
+def clean_antitoxin_pard3_dataset(
+    pipeline: Pipeline,
+) -> Tuple[Pipeline, MutationDataset]:
+    """Clean Antitoxin dataset using configurable pipeline
+
+    Parameters
+    ----------
+    pipeline : Pipeline
+        Antitoxin dataset cleaning pipeline
+
+    Returns
+    -------
+    Tuple[Pipeline, MutationDataset]
+        - Pipeline: The cleaned pipeline
+        - MutationDataset: The cleaned Antitoxin dataset
+
+    Examples
+    --------
+    Use default configuration:
+
+    >>> pipeline = create_antitoxin_cleaner(df)  # df is raw Antitoxin dataset file
+
+    Use partial configuration:
+
+    >>> pipeline = create_antitoxin_cleaner(df, config={
+    ...     "validate_mut_workers": 8,
+    ... })
+
+    Load configuration from file:
+
+    >>> pipeline = create_antitoxin_cleaner(df, config="config.json")
+    >>> pipeline, dataset = clean_antitoxin_dataset(pipeline)
+    """
+    try:
+        # Run pipeline
+        pipeline.execute()
+
+        # Extract results
+        antitoxin_dataset_df, antitoxin_ref_seq = pipeline.data
+        antitoxin_dataset = MutationDataset.from_dataframe(
+            antitoxin_dataset_df, antitoxin_ref_seq
+        )
+
+        logger.info(
+            f"Successfully cleaned antitoxin dataset: "
+            f"{len(antitoxin_dataset_df)} mutations from {len(antitoxin_ref_seq)} proteins"
+        )
+
+        return pipeline, antitoxin_dataset
+    except Exception as e:
+        logger.error(f"Error in running antitoxin dataset cleaning pipeline: {str(e)}")
+        raise RuntimeError(
+            f"Error in running antitoxin dataset cleaning pipeline: {str(e)}"
+        )

mutcleaner/cleaners/antitoxin_pard3_custom_cleaners.py

@@ -0,0 +1,137 @@
+import re
+import pandas as pd
+import numpy as np
+from typing import List
+from tqdm import tqdm
+
+from ..core.pipeline import pipeline_step
+from ..core.sequence import ProteinSequence
+
+__all__ = ["infer_wt_sequence", "add_wild_type_sequence", "simplify_mutations"]
+
+
+@pipeline_step
+def infer_wt_sequence(
+    dataset: pd.DataFrame,
+    mutation_col: str,
+    sequence_col: str = "mut_seq",
+    wt_column: str = "wt_seq",
+) -> pd.DataFrame:
+    """
+    infer wild type seuquence from mutation_sequence and mutation information
+
+    Argument
+    --------
+        dataset : pd.DataFrame
+            the dataset which can be inferred wild-type sequence with mutation sequence and mutation information
+        mutation_col : str
+            the column contatins mutation information
+        sequence_col : str, default="mut_seq"
+            the column contains mutation sequence
+        wt_column : str, default="wt_seq"
+            the column that can be inferred by mutation sequence and mutation information
+
+    Returns
+    -------
+        dataset: pd.DataFrame
+            the dataset contatins added wild-type sequence
+    """
+    if dataset.empty:
+        raise ValueError("Dataset is empty. Cannot infer WT sequence.")
+
+    dataset = dataset.copy()
+
+    seq_len = len(dataset[sequence_col].iat[0])
+    sequences: List[str] = dataset[sequence_col].tolist()
+    mutation_seqs: List[str] = dataset[mutation_col].tolist()
+
+    wt_sequence_list = []
+
+    try:
+        mut_matrix = (
+            np.array(mutation_seqs, dtype="U").view("U1").reshape(len(dataset), -1)
+        )
+        seq_matrix = np.array(sequences, dtype="U").view("U1").reshape(len(dataset), -1)
+    except ValueError:
+        raise ValueError("Sequences in the DataFrame are not of equal length.")
+
+    for i in range(seq_len):
+        match_indices = np.where(mut_matrix[:, i] == "_")[0]
+
+        if match_indices.size > 0:
+            idx = match_indices[0]
+            original_char = seq_matrix[idx, i]
+            wt_sequence_list.append(original_char)
+        else:
+            print(
+                f"Warning: Position {i} is mutated in All sequences. Cannot recover wild-type sequence."
+            )
+            wt_sequence_list.append("X")
+
+    wt_seq = "".join(wt_sequence_list)
+
+    dataset[wt_column] = wt_seq
+
+    return dataset
+
+
+@pipeline_step
+def simplify_mutations(
+    dataset: pd.DataFrame,
+    mutation_column: str = "mut_info",
+    mutation_sep: str = ":",
+) -> pd.DataFrame:
+    """
+    Simplify mutation strings by removing no-change tokens like L47L.
+    Keeps unknown tokens unchanged (does not filter rows).
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> df = pd.DataFrame({
+    >>> 'name':["protein1", "protein1", "protein1"]
+    >>> 'mut_info':["L47L:D51D:I52I","L47A:D51D:I52A","L47_:D51D:I52I"]
+    >>> })
+    >>> df = simplify_mutations(df)
+    >>> df[mut_info]
+        pd.Series("WT","L47A,I52A","L47_") # only simplify the mutations but not fliter
+    """
+    if mutation_column not in dataset.columns:
+        raise ValueError(f"the {mutation_column} is not in the dataset")
+
+    dataset = dataset.copy()
+
+    # only for simplifying
+    # validate the legality of mutations
+    mutation_regex = re.compile(r"^([A-Z\*_])(\d+)([A-Z\*_])$")
+
+    def simplify_single_mutations(mutations):
+        if pd.isna(mutations):
+            return mutations
+
+        mutations = str(mutations).strip()
+        if not mutations:
+            return mutations
+
+        keep = []
+        for single_mutation_tok in mutations.split(mutation_sep):
+            single_mutation_tok = single_mutation_tok.strip()
+            if not single_mutation_tok:
+                continue
+            single_mutation = mutation_regex.match(single_mutation_tok)
+            if single_mutation is None:
+                keep.append(single_mutation_tok)
+                continue
+            wt, pos, mut = (
+                single_mutation.group(1),
+                single_mutation.group(2),
+                single_mutation.group(3),
+            )
+            if wt != mut:
+                keep.append(f"{wt}{pos}{mut}")
+
+        return "WT" if len(keep) == 0 else ",".join(keep)
+
+    dataset[mutation_column] = dataset[mutation_column].apply(simplify_single_mutations)
+
+    return dataset