SCiMS 1.0.0__py3-none-any.whl

scims/__init__.py

@@ -0,0 +1,35 @@
+# __init__.py: initialize SCiMS package
+
+from .utils import (
+    read_metadata,
+    normalize_colname,
+    find_sample_id_column,
+    read_master_file,
+    extract_sample_id
+)
+
+from .helpers import (
+    load_training_data
+)
+
+from .classification import (
+    process_sample_xy,
+    process_sample_zw
+)
+
+from .__main__ import main
+
+__all__ = [
+    'main',
+    'read_metadata',
+    'normalize_colname',
+    'find_sample_id_column',
+    'read_master_file',
+    'extract_sample_id',
+    'load_training_data',
+    'process_sample_xy',
+    'process_sample_zw'
+]
+
+# Package version
+__version__ = '1.1.0'

scims/__main__.py

@@ -0,0 +1,196 @@
+"""
+SCiMS: Sex Calling in Metagenomic Sequencing
+
+This script classifies host sex using metagenomic sequencing data alone. 
+Metagenomic samples are classified into male, female, or uncertain based on 
+coverage ratios of putative sex chromosomes (X/Y or Z/W). It uses a kernel 
+density estimation (KDE) approach, comparing coverage ratios against training data.
+In the XY system, 'male' is heterogametic (XY); in the ZW system, 'female' is 
+heterogametic (ZW).
+
+Author: Hanh Tran
+Version: 1.1.0
+"""
+
+import argparse
+import logging
+import os
+import pandas as pd
+import sys
+
+from scipy.stats import gaussian_kde
+
+from .utils import (
+    read_metadata,
+    find_sample_id_column,
+    extract_sample_id
+)
+from .helpers import load_training_data
+from .process_idxstats import process_idxstats_file
+
+def main():
+    parser = argparse.ArgumentParser(description="SCiMS: Sex Calling in Metagenomic Sequencing")
+    
+    # Mode selection: default is single-sample mode.
+    parser.add_argument('--idxstats_file', dest="idxstats_file", help='Path to a single idxstats file (default mode)')
+    parser.add_argument('--idxstats_folder', dest="idxstats_folder", help='Path to the folder containing idxstats files for multiple-sample mode')
+    
+    parser.add_argument('--scaffolds', dest="scaffold_ids_file", required=True, help='Path to the text file containing scaffold IDs')
+    parser.add_argument('--homogametic_scaffold', dest="homogametic_scaffold", required=True, help='ID of the homogametic scaffold (e.g. X or Z)')
+    parser.add_argument('--heterogametic_scaffold', dest="heterogametic_scaffold", required=True, help='ID of the heterogametic scaffold (e.g. Y or W)')
+    parser.add_argument('--ZW', dest="ZW", action="store_true", help='Switch to ZW system (default is XY)')
+    parser.add_argument('--threshold', dest="threshold", type=float, default=0.5, help='Probability threshold for determining sex')
+    parser.add_argument('--output_dir', dest="output_dir", required=True, help='Path to the output directory')
+    parser.add_argument('--training_data', dest="training_data", help='Path to the training data file', default="training_data_hmp_1000x_normalizedXY.txt")
+    
+    # Optional metadata update (only used in multiple-sample mode)
+    parser.add_argument('--metadata', dest="metadata", help='Path to the metadata file (optional, used in multiple-sample mode)')
+    parser.add_argument('--id_column', dest="id_column", help='User-specified sample ID column name in metadata')
+    
+    # New boolean flag for log output (default is False)
+    parser.add_argument('--log', dest="log", action="store_true", help='If set, a log file is written to the output directory (scims.log)')
+    
+    args = parser.parse_args()
+    
+    # Check metadata parameters early
+    if args.metadata and not args.id_column:
+        print("Error: When providing a metadata file, you must also specify the id column using --id_column.")
+        sys.exit(1)
+    
+    # Setup logging after parsing arguments
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.INFO)
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+    
+    # Clear any existing handlers
+    if logger.hasHandlers():
+        logger.handlers.clear()
+    
+    # Add console handler
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+    
+    # Create output directory if it doesn't exist
+    os.makedirs(args.output_dir, exist_ok=True)
+    
+    # Add file handler if the --log flag is set
+    if args.log:
+        log_file_path = os.path.join(args.output_dir, "scims.log")
+        file_handler = logging.FileHandler(log_file_path, mode='w')
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+        logger.info(f"Log file created at: {log_file_path}")
+    
+    logger.info(" \n=================================================")
+    logger.info("""
+    _|_|_|   _|_|_|  _|_|_|  _|      _|   _|_|_|  
+    _|      _|         _|    _|_|  _|_|   _|        
+    _|_|_|  _|         _|    _|  _|  _|   _|_|_|    
+        _|  _|         _|    _|      _|       _|  
+    _|_|_|   _|_|_|  _|_|_|  _|      _|   _|_|_|    
+    =================================================""")
+    logger.info("SCiMS: Sex Calling in Metagenomic Sequencing")
+    logger.info("Version: 1.1.0")
+    logger.info("=================================================")
+    
+    # Validate mode: either a single file or folder must be provided
+    if args.idxstats_folder:
+        mode = "multiple"
+    elif args.idxstats_file:
+        mode = "single"
+    else:
+        logger.error("You must specify either --idxstats_file for single-sample mode or --idxstats_folder for multiple-sample mode.")
+        sys.exit(1)
+    
+    # Load scaffold IDs
+    try:
+        with open(args.scaffold_ids_file, 'r') as sf:
+            scaffold_ids = [line.strip() for line in sf if line.strip()]
+    except Exception as e:
+        logger.error(f"Failed to read scaffold IDs: {e}")
+        sys.exit(1)
+    
+    # Load training data and build KDE models
+    try:
+        training_data = load_training_data(args.training_data)
+    except Exception as e:
+        logger.error(f"Failed to load training data: {e}")
+        sys.exit(1)
+    
+    if args.ZW:
+        # Use ZW system columns from training data
+        male_rows = training_data[training_data['actual_sex_zw'] == 'male']
+        female_rows = training_data[training_data['actual_sex_zw'] == 'female']
+        male_data = male_rows[['Rz', 'Rw']].dropna().values.T
+        female_data = female_rows[['Rz', 'Rw']].dropna().values.T
+    else:
+        # Default to XY system
+        male_rows = training_data[training_data['actual_sex'] == 'male']
+        female_rows = training_data[training_data['actual_sex'] == 'female']
+        male_data = male_rows[['Rx', 'Ry']].dropna().values.T
+        female_data = female_rows[['Rx', 'Ry']].dropna().values.T
+    
+    kde_male_joint = gaussian_kde(male_data)
+    kde_female_joint = gaussian_kde(female_data)
+    
+    if mode == "multiple":
+        try:
+            folder_files = [os.path.join(args.idxstats_folder, f) 
+                            for f in os.listdir(args.idxstats_folder) if f.endswith(".idxstats")]
+        except Exception as e:
+            logger.error(f"Error reading idxstats folder: {e}")
+            sys.exit(1)
+            
+        if not folder_files:
+            logger.error("No idxstats files found in the provided folder.")
+            sys.exit(1)
+        
+        all_results = []  # To collect results for optional metadata merging
+        for idxstats_file in folder_files:
+            result = process_idxstats_file(idxstats_file, scaffold_ids, args, kde_male_joint, kde_female_joint)
+            all_results.append(result)
+            # Build output dictionary using consistent keys
+            out_dict = {
+                "SCiMS_ID": result.get("SCiMS_ID"),
+                "SCiMS_predicted_sex": result.get("SCiMS_sex"),
+                "SCiMS_male_post_prob": result.get("SCiMS_male_post_prob"),
+                "SCiMS_female_post_prob": result.get("SCiMS_female_post_prob")
+            }
+            base_name = os.path.basename(idxstats_file).split('.')[0]
+            output_file = os.path.join(args.output_dir, f"{base_name}_results.txt")
+            pd.DataFrame([out_dict]).to_csv(output_file, sep='\t', index=False)
+            logger.info(f"Results written to {output_file}")
+        
+        # Merge metadata once after processing all files
+        if args.metadata:
+            try:
+                results_df = pd.DataFrame(all_results)
+                metadata = read_metadata(args.metadata)
+                sample_id_col = find_sample_id_column(metadata, args.id_column)
+                merged_df = pd.merge(metadata, results_df, left_on=sample_id_col, right_on='SCiMS_ID', how='left')
+                merged_df.drop(columns=['SCiMS_ID'], inplace=True)
+                metadata_basename = os.path.basename(args.metadata).split('.')[0]
+                metadata_file = os.path.join(args.output_dir, f"{metadata_basename}_scims_updated.txt")
+                merged_df.to_csv(metadata_file, sep='\t', index=False)
+                logger.info(f"Updated metadata with classification results written to {metadata_file}")
+            except Exception as e:
+                logger.error(f"Error updating metadata: {e}")
+                sys.exit(1)
+    else:
+        # Single-sample mode: process one file and write the filtered result
+        result = process_idxstats_file(args.idxstats_file, scaffold_ids, args, kde_male_joint, kde_female_joint)
+        out_dict = {
+            "SCiMS_ID": result.get("SCiMS_ID"),
+            "SCiMS_predicted_sex": result.get("SCiMS_sex"),
+            "SCiMS_male_post_prob": result.get("SCiMS_male_post_prob"),
+            "SCiMS_female_post_prob": result.get("SCiMS_female_post_prob")
+        }
+        results_df = pd.DataFrame([out_dict])
+        base_name = os.path.basename(args.idxstats_file).split('.')[0]
+        output_file = os.path.join(args.output_dir, f"{base_name}_results.txt")
+        results_df.to_csv(output_file, sep='\t', index=False)
+        logger.info(f"Results written to {output_file}")
+    
+if __name__ == "__main__":
+    main()

scims/classification.py

@@ -0,0 +1,135 @@
+# classification.py : core logic for classification process
+
+from scipy.stats import gaussian_kde
+import logging
+import numpy as np
+import pandas as pd
+from .helpers import (
+    compute_coverage_ratio_rx,
+    compute_joint_posterior,
+    determine_sex_with_joint_posteriors,
+    load_training_data
+) 
+
+def process_sample_xy(
+        idxstats: pd.DataFrame,
+        x_id: str,
+        y_id: str,
+        male_kde: gaussian_kde,
+        female_kde: gaussian_kde,
+        threshold: float
+) -> dict:
+    """
+    Given an idxstats DataFrame, XY chromosome IDs (x_id, y_id), and KDE models for male/female,
+    computes coverage ratios (Rx, Ry) and determines sex using joint posteriors.
+    """
+    # Indices of X/Y
+    x_index = idxstats.index.get_loc(x_id) if x_id in idxstats.index else None
+    y_index = idxstats.index.get_loc(y_id) if y_id in idxstats.index else None
+    
+    # Idenfity which rows are autosomes (excluding X and Y)
+    autosome_indices = [
+        i for i in range(len(idxstats))
+        if i != x_index and (y_index is None or i != y_index)]
+    
+    # Coverage ratio for X vs autosomes (Rx)
+    Rx = compute_coverage_ratio_rx(idxstats, autosome_indices, x_index)
+
+    # Coverage ratio for Y vs X+Y (Ry)
+    x_count = idxstats.loc[x_id].iloc[1] if x_id in idxstats.index else 0    # column 1 = mapped reads column 0 = chromosome length
+    y_count = idxstats.loc[y_id].iloc[1] if y_id in idxstats.index else 0
+    x_length = idxstats.loc[x_id].iloc[0]
+    y_length = idxstats.loc[y_id].iloc[0]
+    total_count = idxstats.iloc[:, 1].sum()
+    total_xy = x_count + y_count
+
+    if total_xy == 0:
+        Ry = np.nan
+        logging.warning(f"No reads mapped to X or Y in {x_id} or {y_id}. Skipping Ry ratio calculation.")
+    else:
+        factor = x_length / y_length # This factor is used to take into account that X and Y are different lengths, so we need to normalize the coverage ratio by the length of the chromosomes so that the coverage ratio is comparable between the two chromosomes and not just because of artifactual differences in length
+        Ry = (y_count / total_xy) * factor
+
+    # Compute posteriors
+    P_male = 0.5
+    P_female = 0.5
+
+    P_male_posterior, P_female_posterior = compute_joint_posterior(Rx, Ry, male_kde, female_kde, P_male, P_female)
+    
+    inferred_sex = determine_sex_with_joint_posteriors(P_male_posterior, P_female_posterior, threshold)
+
+    return {
+        'Rx': Rx,
+        'Ry': Ry,
+        'Total reads mapped': total_count,
+        'Reads mapped to X': x_count,
+        'Reads mapped to Y': y_count,
+        'Posterior probability of being male': np.round(P_male_posterior, 3),
+        'Posterior probability of being female': np.round(P_female_posterior, 3),
+        'SCiMS predicted sex': inferred_sex
+    }
+
+
+def process_sample_zw(
+        idxstats: pd.DataFrame,
+        z_id: str,
+        w_id: str,
+        male_kde: gaussian_kde,
+        female_kde: gaussian_kde,
+        threshold: float
+) -> dict:
+    """
+    Given an idxstats DataFrame, ZW chromosome IDs (z_id, w_id), and KDE models for male/female,
+    computes coverage ratios (Rz, Rw) and determines sex using joint posteriors.
+    """
+    z_index = idxstats.index.get_loc(z_id) if z_id in idxstats.index else None
+    w_index = idxstats.index.get_loc(w_id) if w_id in idxstats.index else None
+
+    # Identify which rows are autosomes (excluding Z and W)
+    autosome_indices = [
+        i for i in range(len(idxstats))
+        if i != z_index and (w_index is None or i != w_index)
+    ]
+
+    # Coverage ratio for Z vs autosomes (Rz)
+    Rz = compute_coverage_ratio_rx(idxstats, autosome_indices, z_index)
+
+    # Coverage ratio for W vs Z+W (Rw)
+    z_count = idxstats.loc[z_id].iloc[1] if z_id in idxstats.index else 0
+    z_length = idxstats.loc[z_id].iloc[0]
+    #print(z_length)
+    w_count = idxstats.loc[w_id].iloc[1] if w_id in idxstats.index else 0
+    w_length = idxstats.loc[w_id].iloc[0]
+    #print(w_length)
+    total_count = idxstats.iloc[:, 1].sum()
+    total_zw = z_count + w_count
+
+    if total_zw == 0:
+        Rw = np.nan
+        logging.warning(f"No reads mapped to Z or W in {z_id} or {w_id}. Skipping Rw ratio calculation.")
+    else:
+        #Rw = (w_count / total_zw) * (z_length / w_length)
+        #print(Rw)
+        factor = z_length / w_length # This factor is used to take into account that Z and W are different lengths, so we need to normalize the coverage ratio by the length of the chromosomes so that the coverage ratio is comparable between the two chromosomes and not just because of artifactual differences in length
+        #print(factor)
+        Rw = (w_count / total_zw) * factor
+        #print(Rw)
+
+    # Compute posteriors
+    P_male = 0.5
+    P_female = 0.5
+
+    P_male_posterior, P_female_posterior = compute_joint_posterior(Rz, Rw, male_kde, female_kde, P_male, P_female)
+
+    inferred_sex = determine_sex_with_joint_posteriors(P_male_posterior, P_female_posterior, threshold)
+
+    return {
+        'Rz': Rz,
+        'Rw': Rw,
+        'Total reads mapped': total_count,
+        'Reads mapped to Z': z_count,
+        'Reads mapped to W': w_count,
+        'Posterior probability of being male': np.round(P_male_posterior, 3),
+        'Posterior probability of being female': np.round(P_female_posterior, 3),
+        'SCiMS predicted sex': inferred_sex
+    }