smftools 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

smftools/informatics/direct_smf.py

@@ -0,0 +1,137 @@
+## direct_smf
+
+def direct_smf(fasta, output_directory, mod_list, model_dir, model, thresholds, input_data_path, split_dir, barcode_kit, mapping_threshold, experiment_name, bam_suffix, batch_size, basecall, barcode_both_ends, trim, device, make_bigwigs, skip_unclassified, delete_batch_hdfs, threads):
+    """
+    Processes sequencing data from a direct methylation detection Nanopore SMF experiment to an AnnData object.
+
+    Parameters:
+        fasta (str): File path to the reference genome to align to.
+        output_directory (str): A file path to the directory to output all the analyses.
+        mod_list (list): A list of strings of the modification types to use in the analysis.
+        model_dir (str): a string representing the file path to the dorado basecalling model directory.
+        model (str): a string representing the dorado basecalling model.
+        thresholds (list): A list of floats to pass for call thresholds.
+        input_data_path (str): a string representing the file path to the experiment directory containing the input sequencing files.
+        split_dir (str): A string representing the file path to the directory to split the BAMs into.
+        barcode_kit (str): A string representing the barcoding kit used in the experiment.
+        mapping_threshold (float): A value in between 0 and 1 to threshold the minimal fraction of aligned reads which map to the reference region. References with values above the threshold are included in the output adata.
+        experiment_name (str): A string to provide an experiment name to the output adata file.
+        bam_suffix (str): A suffix to add to the bam file.
+        batch_size (int): An integer number of TSV files to analyze in memory at once while loading the final adata object.
+        basecall (bool): Whether to basecall
+        barcode_both_ends (bool): Whether to require a barcode detection on both ends for demultiplexing.
+        trim (bool): Whether to trim barcodes, adapters, and primers from read ends
+        device (str): Device to use for basecalling. auto, metal, cpu, cuda
+        make_bigwigs (bool): Whether to make bigwigs
+        skip_unclassified (bool): Whether to skip unclassified reads when extracting mods and loading anndata
+        delete_batch_hdfs (bool): Whether to delete intermediate hdf5 files.
+        threads (int): cpu threads available for processing.
+
+    Returns:
+        final_adata_path (str): Path to the final adata object   
+        sorted_output (str): Path to the aligned, sorted BAM
+    """
+    from .helpers import align_and_sort_BAM, aligned_BAM_to_bed, extract_mods, get_chromosome_lengths, make_modbed, modcall, modkit_extract_to_adata, modQC, demux_and_index_BAM, make_dirs, bam_qc, run_multiqc
+    import os
+
+    if basecall:
+        model_basename = os.path.basename(model)
+        model_basename = model_basename.replace('.', '_')
+        mod_string = "_".join(mod_list)
+        bam=f"{output_directory}/{model_basename}_{mod_string}_calls"
+    else:
+        bam_base=os.path.basename(input_data_path).split('.bam')[0]
+        bam=os.path.join(output_directory, bam_base)
+    aligned_BAM=f"{bam}_aligned"
+    aligned_sorted_BAM=f"{aligned_BAM}_sorted"
+
+    if barcode_both_ends:
+        split_dir = split_dir + '_both_ends_barcoded'
+    else:
+        split_dir = split_dir + '_at_least_one_end_barcoded'
+
+    mod_bed_dir=f"{split_dir}/split_mod_beds"
+    mod_tsv_dir=f"{split_dir}/split_mod_tsvs"
+    bam_qc_dir = f"{split_dir}/bam_qc"
+
+    aligned_sorted_output = aligned_sorted_BAM + bam_suffix
+    mod_map = {'6mA': '6mA', '5mC_5hmC': '5mC'}
+    mods = [mod_map[mod] for mod in mod_list]
+
+    # Make a FAI and .chrom.names file for the fasta
+    get_chromosome_lengths(fasta)
+
+    os.chdir(output_directory)
+
+    # 1) Basecall using dorado
+    if basecall:
+        modcall_output = bam + bam_suffix
+        if os.path.exists(modcall_output):
+            print(modcall_output + ' already exists. Using existing basecalled BAM.')
+        else:
+            modcall(model_dir, model, input_data_path, barcode_kit, mod_list, bam, bam_suffix, barcode_both_ends, trim, device)
+    else:
+        modcall_output = input_data_path
+
+    # 2) Align the BAM to the reference FASTA. Also make an index and a bed file of mapped reads
+    aligned_output = aligned_BAM + bam_suffix
+    sorted_output = aligned_sorted_BAM + bam_suffix
+    if os.path.exists(aligned_output) and os.path.exists(sorted_output):
+        print(sorted_output + ' already exists. Using existing aligned/sorted BAM.')
+    else:
+        align_and_sort_BAM(fasta, modcall_output, bam_suffix, output_directory, make_bigwigs, threads)
+
+    # Make beds and provide basic histograms
+    bed_dir = os.path.join(output_directory, 'beds')
+    if os.path.isdir(bed_dir):
+        print(bed_dir + ' already exists. Skipping BAM -> BED conversion for ' + sorted_output)
+    else:
+        aligned_BAM_to_bed(aligned_output, output_directory, fasta, make_bigwigs, threads)
+
+    # 3) Split the aligned and sorted BAM files by barcode (BC Tag) into the split_BAM directory
+    if os.path.isdir(split_dir):
+        print(split_dir + ' already exists. Using existing demultiplexed BAMs.')
+        bam_files = os.listdir(split_dir)
+        bam_files = [os.path.join(split_dir, file) for file in bam_files if '.bam' in file and '.bai' not in file and 'unclassified' not in file]
+        bam_files.sort()
+    else:
+        make_dirs([split_dir])
+        bam_files = demux_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix, barcode_kit, barcode_both_ends, trim, fasta, make_bigwigs, threads)
+        # split_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix, output_directory, converted_FASTA) # deprecated, just use dorado demux
+
+    # Make beds and provide basic histograms
+    bed_dir = os.path.join(split_dir, 'beds')
+    if os.path.isdir(bed_dir):
+        print(bed_dir + ' already exists. Skipping BAM -> BED conversion for demultiplexed bams')
+    else:
+        for bam in bam_files:
+            aligned_BAM_to_bed(bam, split_dir, fasta, make_bigwigs, threads)
+
+    # 4) Samtools QC metrics on split BAM files
+    if os.path.isdir(bam_qc_dir):
+        print(bam_qc_dir + ' already exists. Using existing BAM QC calculations.')
+    else:
+        make_dirs([bam_qc_dir])
+        bam_qc(bam_files, bam_qc_dir, threads, modality='direct')
+
+    # 5) Using nanopore modkit to work with modified BAM files ###
+    if os.path.isdir(mod_bed_dir):
+        print(mod_bed_dir + ' already exists, skipping making modbeds')
+    else:
+        make_dirs([mod_bed_dir])  
+        modQC(aligned_sorted_output, thresholds) # get QC metrics for mod calls
+        make_modbed(aligned_sorted_output, thresholds, mod_bed_dir) # Generate bed files of position methylation summaries for every sample
+
+    # multiqc ###
+    if os.path.isdir(f"{split_dir}/multiqc"):
+        print(f"{split_dir}/multiqc" + ' already exists, skipping multiqc')
+    else:
+        run_multiqc(split_dir, f"{split_dir}/multiqc")
+
+    make_dirs([mod_tsv_dir])  
+    extract_mods(thresholds, mod_tsv_dir, split_dir, bam_suffix, skip_unclassified, threads) # Extract methylations calls for split BAM files into split TSV files
+
+    #6 Load the modification data from TSVs into an adata object
+    final_adata, final_adata_path = modkit_extract_to_adata(fasta, split_dir, mapping_threshold, experiment_name, mods, batch_size, mod_tsv_dir, delete_batch_hdfs, threads)
+
+    return final_adata, final_adata_path, sorted_output, bam_files

smftools/informatics/fast5_to_pod5.py

@@ -0,0 +1,21 @@
+# fast5_to_pod5
+
+def fast5_to_pod5(fast5_dir, output_pod5='FAST5s_to_POD5.pod5'):
+    """
+    Convert Nanopore FAST5 files to POD5 file
+
+    Parameters:
+        fast5_dir (str): String representing the file path to a directory containing all FAST5 files to convert into a single POD5 output.
+        output_pod5 (str): The name of the output POD5.
+    
+    Returns:
+        None
+    
+    """
+    import subprocess
+    from pathlib import Path
+
+    if Path(fast5_dir).is_file():
+        subprocess.run(["pod5", "convert", "fast5", fast5_dir, "--output", output_pod5])
+    elif Path(fast5_dir).is_dir():
+        subprocess.run(["pod5", "convert", "fast5", f".{fast5_dir}*.fast5", "--output", output_pod5])

smftools/informatics/helpers/LoadExperimentConfig.py

@@ -0,0 +1,75 @@
+## LoadExperimentConfig
+
+class LoadExperimentConfig:
+    """
+    Loads in the experiment configuration csv and saves global variables with experiment configuration parameters.
+    Parameters:
+        experiment_config (str): A string representing the file path to the experiment configuration csv file.
+    
+    Attributes:
+        var_dict (dict): A dictionary containing experiment configuration parameters.
+
+    Example:
+        >>> import pandas as pd
+        >>> from io import StringIO
+        >>> csv_data = '''variable,value,type
+        ... mapping_threshold,0.05,float
+        ... batch_size,4,int
+        ... testing_bool,True,bool
+        ... strands,"[bottom, top]",list
+        ... split_dir,split_bams,string
+        ... pod5_dir,None,string
+        ... pod5_dir,,string
+        ... '''
+        >>> csv_file = StringIO(csv_data)
+        >>> df = pd.read_csv(csv_file)
+        >>> df.to_csv('test_config.csv', index=False)
+        >>> config_loader = LoadExperimentConfig('test_config.csv')
+        >>> config_loader.var_dict['mapping_threshold']
+        0.05
+        >>> config_loader.var_dict['batch_size']
+        4
+        >>> config_loader.var_dict['testing_bool']
+        True
+        >>> config_loader.var_dict['strands']
+        ['bottom', 'top']
+        >>> config_loader.var_dict['split_dir']
+        'split_bams'
+        >>> config_loader.var_dict['pod5_dir'] is None
+        True
+        >>> config_loader.var_dict['pod5_dir'] is None
+        True
+    """
+    def __init__(self, experiment_config):
+        import pandas as pd
+        print(f"Loading experiment config from {experiment_config}")
+        # Read the CSV into a pandas DataFrame
+        df = pd.read_csv(experiment_config)
+        # Initialize an empty dictionary to store variables
+        var_dict = {}
+        # Iterate through each row in the DataFrame
+        for _, row in df.iterrows():
+            var_name = str(row['variable'])
+            value = row['value']
+            dtype = row['type']
+            # Handle empty and None values
+            if pd.isna(value) or value in ['None', '']:
+                value = None
+            else:
+                # Handle different data types
+                if dtype == 'list':
+                    # Convert the string representation of a list to an actual list
+                    value = value.strip('()[]').replace(', ', ',').split(',')
+                elif dtype == 'int':
+                    value = int(value)
+                elif dtype == 'float':
+                    value = float(value)
+                elif dtype == 'bool':
+                    value = value.lower() == 'true'
+                elif dtype == 'string':
+                    value = str(value)
+            # Store the variable in the dictionary
+            var_dict[var_name] = value
+        # Save the dictionary as an attribute of the class
+        self.var_dict = var_dict
+

smftools/informatics/helpers/__init__.py

@@ -0,0 +1,74 @@
+from .align_and_sort_BAM import align_and_sort_BAM
+from .aligned_BAM_to_bed import aligned_BAM_to_bed
+from .bam_qc import bam_qc
+from .bed_to_bigwig import bed_to_bigwig
+from .binarize_converted_base_identities import binarize_converted_base_identities
+from .canoncall import canoncall
+from .complement_base_list import complement_base_list
+from .converted_BAM_to_adata_II import converted_BAM_to_adata_II
+from .concatenate_fastqs_to_bam import concatenate_fastqs_to_bam
+from .count_aligned_reads import count_aligned_reads
+from .demux_and_index_BAM import demux_and_index_BAM
+from .extract_base_identities import extract_base_identities
+from .extract_mods import extract_mods
+from .extract_read_features_from_bam import extract_read_features_from_bam
+from .extract_read_lengths_from_bed import extract_read_lengths_from_bed
+from .extract_readnames_from_BAM import extract_readnames_from_BAM
+from .find_conversion_sites import find_conversion_sites
+from .generate_converted_FASTA import convert_FASTA_record, generate_converted_FASTA
+from .get_chromosome_lengths import get_chromosome_lengths
+from .get_native_references import get_native_references
+from .index_fasta import index_fasta
+from .LoadExperimentConfig import LoadExperimentConfig
+from .make_dirs import make_dirs
+from .make_modbed import make_modbed
+from .modcall import modcall
+from .modkit_extract_to_adata import modkit_extract_to_adata
+from .modQC import modQC
+from .one_hot_encode import one_hot_encode
+from .ohe_batching import ohe_batching
+from .one_hot_decode import one_hot_decode
+from .ohe_layers_decode import ohe_layers_decode
+from .plot_read_length_and_coverage_histograms import plot_read_length_and_coverage_histograms
+from .run_multiqc import run_multiqc
+from .separate_bam_by_bc import separate_bam_by_bc
+from .split_and_index_BAM import split_and_index_BAM
+
+__all__ = [
+    "align_and_sort_BAM",
+    "aligned_BAM_to_bed",
+    "bam_qc",
+    "bed_to_bigwig",
+    "binarize_converted_base_identities",
+    "canoncall",
+    "complement_base_list",
+    "converted_BAM_to_adata_II",
+    "concatenate_fastqs_to_bam",
+    "count_aligned_reads",
+    "demux_and_index_BAM",
+    "extract_base_identities",
+    "extract_mods",
+    "extract_read_features_from_bam",
+    "extract_read_lengths_from_bed",
+    "extract_readnames_from_BAM",
+    "find_conversion_sites",
+    "convert_FASTA_record",
+    "generate_converted_FASTA",
+    "get_chromosome_lengths",
+    "get_native_references",
+    "index_fasta",
+    "LoadExperimentConfig",
+    "make_dirs",
+    "make_modbed",
+    "modcall",
+    "modkit_extract_to_adata",
+    "modQC",
+    "one_hot_encode",
+    "ohe_batching",
+    "one_hot_decode",
+    "ohe_layers_decode",
+    "plot_read_length_and_coverage_histograms",
+    "run_multiqc",
+    "separate_bam_by_bc",
+    "split_and_index_BAM"
+]

smftools/informatics/helpers/align_and_sort_BAM.py

@@ -0,0 +1,59 @@
+## align_and_sort_BAM
+
+def align_and_sort_BAM(fasta, input, bam_suffix='.bam', output_directory='aligned_outputs', make_bigwigs=False, threads=None):
+    """
+    A wrapper for running dorado aligner and samtools functions
+    
+    Parameters:
+        fasta (str): File path to the reference genome to align to.
+        input (str): File path to the basecalled file to align. Works for .bam and .fastq files
+        bam_suffix (str): The suffix to use for the BAM file.
+        output_directory (str): A file path to the directory to output all the analyses.
+        make_bigwigs (bool): Whether to make bigwigs
+        threads (int): Number of additional threads to use
+
+    Returns:
+        None
+            The function writes out files for: 1) An aligned BAM, 2) and aligned_sorted BAM, 3) an index file for the aligned_sorted BAM, 4) A bed file for the aligned_sorted BAM, 5) A text file containing read names in the aligned_sorted BAM
+    """
+    import subprocess
+    import os
+
+    input_basename = os.path.basename(input)
+    input_suffix = '.' + input_basename.split('.')[1]
+
+    output_path_minus_suffix = os.path.join(output_directory, input_basename.split(input_suffix)[0])
+    
+    aligned_BAM=f"{output_path_minus_suffix}_aligned"
+    aligned_sorted_BAM=f"{aligned_BAM}_sorted"
+    aligned_output = aligned_BAM + bam_suffix
+    aligned_sorted_output = aligned_sorted_BAM + bam_suffix
+
+    if threads:
+        threads = str(threads)
+    else:
+        pass
+    
+    # Run dorado aligner
+    print(f"Aligning BAM to Reference: {input}")
+    if threads:
+        alignment_command = ["dorado", "aligner", "-t", threads, '--mm2-opts', "-N 1", fasta, input]
+    else:
+        alignment_command = ["dorado", "aligner", '--mm2-opts', "-N 1", fasta, input]
+    subprocess.run(alignment_command, stdout=open(aligned_output, "w"))
+
+    # Sort the BAM on positional coordinates
+    print(f"Sorting BAM: {aligned_output}")
+    if threads:
+        sort_command = ["samtools", "sort", "-@", threads, "-o", aligned_sorted_output, aligned_output]
+    else:
+        sort_command = ["samtools", "sort", "-o", aligned_sorted_output, aligned_output]
+    subprocess.run(sort_command)
+
+    # Create a BAM index file
+    print(f"Indexing BAM: {aligned_sorted_output}")
+    if threads:
+        index_command = ["samtools", "index", "-@", threads, aligned_sorted_output]
+    else:
+        index_command = ["samtools", "index", aligned_sorted_output]
+    subprocess.run(index_command)

smftools/informatics/helpers/aligned_BAM_to_bed.py

@@ -0,0 +1,74 @@
+def aligned_BAM_to_bed(aligned_BAM, out_dir, fasta, make_bigwigs, threads=None):
+    """
+    Takes an aligned BAM as input and writes a BED file of reads as output.
+    Bed columns are: Record name, start position, end position, read length, read name.
+
+    Parameters:
+        aligned_BAM (str): Path to an input aligned_BAM to extract to a BED file.
+        out_dir (str): Directory to output files.
+        fasta (str): File path to the reference genome.
+        make_bigwigs (bool): Whether to generate bigwig files.
+        threads (int): Number of threads to use.
+
+    Returns:
+        None
+    """
+    import subprocess
+    import os
+    import concurrent.futures
+    from concurrent.futures import ProcessPoolExecutor
+    from .bed_to_bigwig import bed_to_bigwig
+    from . import make_dirs
+    from .plot_read_length_and_coverage_histograms import plot_read_length_and_coverage_histograms
+
+    threads = threads or os.cpu_count()  # Use max available cores if not specified
+
+    # Create necessary directories
+    plotting_dir = os.path.join(out_dir, "bed_cov_histograms")
+    bed_dir = os.path.join(out_dir, "beds")
+    make_dirs([plotting_dir, bed_dir])
+
+    bed_output = os.path.join(bed_dir, os.path.basename(aligned_BAM).replace(".bam", "_bed.bed"))
+
+    print(f"Creating BED from BAM: {aligned_BAM} using {threads} threads...")
+
+    # Convert BAM to BED format
+    with open(bed_output, "w") as output_file:
+        samtools_view = subprocess.Popen(["samtools", "view", "-@", str(threads), aligned_BAM], stdout=subprocess.PIPE)
+        awk_process = subprocess.Popen(
+            ["awk", '{print $3 "\t" $4 "\t" $4+length($10)-1 "\t" length($10)-1 "\t" $1}'],
+            stdin=samtools_view.stdout,
+            stdout=output_file
+        )
+
+    samtools_view.stdout.close()
+    awk_process.wait()
+    samtools_view.wait()
+
+    print(f"BED file created: {bed_output}")
+
+    def split_bed(bed):
+        """Splits BED into aligned and unaligned reads."""
+        aligned = bed.replace(".bed", "_aligned.bed")
+        unaligned = bed.replace(".bed", "_unaligned.bed")
+
+        with open(bed, "r") as infile, open(aligned, "w") as aligned_out, open(unaligned, "w") as unaligned_out:
+            for line in infile:
+                (unaligned_out if line.startswith("*") else aligned_out).write(line)
+
+        os.remove(bed)
+        return aligned
+
+    print(f"Splitting BED: {bed_output}")
+    aligned_bed = split_bed(bed_output)
+
+    with ProcessPoolExecutor() as executor:  # Use processes instead of threads
+        futures = []
+        futures.append(executor.submit(plot_read_length_and_coverage_histograms, aligned_bed, plotting_dir))
+        if make_bigwigs:
+            futures.append(executor.submit(bed_to_bigwig, fasta, aligned_bed))
+
+        # Wait for all tasks to complete
+        concurrent.futures.wait(futures)
+
+    print("Processing completed successfully.")