smftools 0.1.0__py3-none-any.whl

smftools/informatics/helpers/modkit_extract_to_adata.py

@@ -0,0 +1,355 @@
+## modkit_extract_to_adata
+from .. import readwrite
+from .get_native_references import get_native_references
+from .count_aligned_reads import count_aligned_reads
+from .extract_base_identities import extract_base_identities
+from .one_hot_encode import one_hot_encode
+import pandas as pd
+import anndata as ad
+import os
+import gc
+import math
+import numpy as np
+
+def modkit_extract_to_adata(fasta, bam, mapping_threshold, experiment_name, mods, batch_size):
+    """
+    
+    """
+    ###################################################
+    ### Get input tsv file names into a sorted list ###
+    # List all files in the directory
+    files = os.listdir(os.getcwd())
+    # get current working directory
+    cwd = os.getcwd()
+    # Filter file names that contain the search string in their filename and keep them in a list
+    tsvs = [tsv for tsv in files if 'extract.tsv' in tsv]
+    # Sort file list by names and print the list of file names
+    tsvs.sort()
+    print(f'{len(tsvs)} sample tsv files found: {tsvs}')
+    print(f'sample bam file found: {bam}')
+
+    # Get all references within the FASTA and indicate the length and identity of the record sequence
+    max_reference_length = 0
+    reference_dict = get_native_references(fasta)
+    for record in reference_dict.keys():
+        if reference_dict[record][0] > max_reference_length:
+            max_reference_length = reference_dict[record][0]
+
+    print(f'{readwrite.time_string()}: Max reference length in dataset: {max_reference_length}')
+    batches = math.ceil(len(tsvs) / batch_size) # Number of batches to process
+    print('{0}: Processing input tsvs in {1} batches of {2} tsvs '.format(readwrite.time_string(), batches, batch_size))
+
+    # look at aligned read proportions in the bam
+    aligned_reads_count, unaligned_reads_count, record_counts = count_aligned_reads(bam)
+    print('{} percent of reads in bam aligned successfully'.format(aligned_reads_count*100 / (aligned_reads_count+unaligned_reads_count)))
+    records_to_analyze = []
+    # Iterate over references and decide which to use in the analysis based on the mapping_threshold
+    for record in record_counts:
+        print('{0} reads mapped to reference record {1}. This is {2} percent of all mapped reads'.format(record_counts[record][0], record, record_counts[record][1]*100))
+        if record_counts[record][1] >= mapping_threshold:
+            records_to_analyze.append(record)
+    print(f'Records to analyze: {records_to_analyze}')
+    # Iterate over records to analyze and return a dictionary keyed by the reference name that points to another dictionary keyed by read names that map to that reference. This internal dictionary points to a one-hot encoding of the mapped read
+    record_seq_dict = {}
+    for record in records_to_analyze:
+        current_reference_length = reference_dict[record][0]
+        delta_max_length = max_reference_length - current_reference_length
+        sequence = reference_dict[record][1] + 'N'*delta_max_length
+        # Get a dictionary of positional base identities keyed by read id
+        base_identities = extract_base_identities(bam, record, current_reference_length, max_reference_length)
+        # One hot encode the sequence string of the reads
+        one_hot_reads = {read_name: one_hot_encode(seq) for read_name, seq in base_identities.items()}
+        record_seq_dict[record] = (one_hot_reads, sequence)
+
+    ###################################################
+
+    ###################################################
+    # Begin iterating over batches
+    for batch in range(batches):
+        print('{0}: Processing tsvs for batch {1} '.format(readwrite.time_string(), batch))
+        # For the final batch, just take the remaining tsv files
+        if batch == batches - 1:
+            tsv_batch = tsvs
+        # For all other batches, take the next batch of tsvs out of the file queue.    
+        else:
+            tsv_batch = tsvs[:batch_size]
+            tsvs = tsvs[batch_size:]
+        print('{0}: tsvs in batch {1} '.format(readwrite.time_string(), tsv_batch))
+    ###################################################
+
+        ###################################################
+        ### Add the tsvs as dataframes to a dictionary (dict_total) keyed by integer index. Also make modification specific dictionaries and strand specific dictionaries.
+        # Initialize dictionaries and place them in a list
+        dict_total, dict_a, dict_a_bottom, dict_a_top, dict_c, dict_c_bottom, dict_c_top, dict_combined_bottom, dict_combined_top = {},{},{},{},{},{},{},{},{}
+        dict_list = [dict_total, dict_a, dict_a_bottom, dict_a_top, dict_c, dict_c_bottom, dict_c_top, dict_combined_bottom, dict_combined_top]
+
+        # Give names to represent each dictionary in the list
+        sample_types = ['total', 'm6A', 'm6A_bottom_strand', 'm6A_top_strand', '5mC', '5mC_bottom_strand', '5mC_top_strand', 'combined_bottom_strand', 'combined_top_strand']
+
+        # Give indices of dictionaries to skip for analysis and final dictionary saving.
+        dict_to_skip = [0, 1, 4]
+        combined_dicts = [7, 8]
+        A_stranded_dicts = [2, 3]
+        C_stranded_dicts = [5, 6]
+        dict_to_skip = dict_to_skip + combined_dicts + A_stranded_dicts + C_stranded_dicts
+        dict_to_skip = set(dict_to_skip)
+
+        # Load the dict_total dictionary with all of the tsv files as dataframes.
+        for i, tsv in enumerate(tsv_batch):
+            print('{0}: Loading sample tsv {1} into dataframe'.format(readwrite.time_string(), tsv))
+            temp_df = pd.read_csv(tsv, sep='\t', header=0)
+            for record in records_to_analyze:
+                if record not in dict_total.keys():
+                    dict_total[record] = {}
+                # Only keep the reads aligned to the chromosomes of interest
+                print('{0}: Filtering sample dataframe to keep chromosome of interest'.format(readwrite.time_string()))
+                dict_total[record][i] = temp_df[temp_df['chrom'] == record]
+                # Only keep the read positions that fall within the region of interest
+                print('{0}: Filtering sample dataframe to keep positions falling within region of interest'.format(readwrite.time_string()))
+                current_reference_length = reference_dict[record][0]
+                dict_total[record][i] = dict_total[record][i][(current_reference_length > dict_total[record][i]['ref_position']) & (dict_total[record][i]['ref_position']>= 0)]
+
+        # Iterate over dict_total of all the tsv files and extract the modification specific and strand specific dataframes into dictionaries
+        for record in dict_total.keys():
+            for i in dict_total[record].keys():
+                if '6mA' in mods:
+                    # Remove Adenine stranded dicts from the dicts to skip set
+                    dict_to_skip.difference_update(A_stranded_dicts)
+
+                    if record not in dict_a.keys() and record not in dict_a_bottom.keys() and record not in dict_a_top.keys():
+                        dict_a[record], dict_a_bottom[record], dict_a_top[record] = {}, {}, {}
+
+                    # get a dictionary of dataframes that only contain methylated adenine positions
+                    dict_a[record][i] = dict_total[record][i][dict_total[record][i]['modified_primary_base'] == 'A']
+                    print('{}: Successfully created a methyl-adenine dictionary for '.format(readwrite.time_string()) + str(i))
+                    # Stratify the adenine dictionary into two strand specific dictionaries.
+                    dict_a_bottom[record][i] = dict_a[record][i][dict_a[record][i]['ref_strand'] == '-']
+                    print('{}: Successfully created a minus strand methyl-adenine dictionary for '.format(readwrite.time_string()) + str(i))
+                    dict_a_top[record][i] = dict_a[record][i][dict_a[record][i]['ref_strand'] == '+']
+                    print('{}: Successfully created a plus strand methyl-adenine dictionary for '.format(readwrite.time_string()) + str(i))
+
+                if '5mC' in mods:
+                    # Remove Cytosine stranded dicts from the dicts to skip set
+                    dict_to_skip.difference_update(C_stranded_dicts)
+
+                    if record not in dict_c.keys() and record not in dict_c_bottom.keys() and record not in dict_c_top.keys():
+                        dict_c[record], dict_c_bottom[record], dict_c_top[record] = {}, {}, {}
+
+                    # get a dictionary of dataframes that only contain methylated cytosine positions
+                    dict_c[record][i] = dict_total[record][i][dict_total[record][i]['modified_primary_base'] == 'C']
+                    print('{}: Successfully created a methyl-cytosine dictionary for '.format(readwrite.time_string()) + str(i))
+                    # Stratify the cytosine dictionary into two strand specific dictionaries.
+                    dict_c_bottom[record][i] = dict_c[record][i][dict_c[record][i]['ref_strand'] == '-']
+                    print('{}: Successfully created a minus strand methyl-cytosine dictionary for '.format(readwrite.time_string()) + str(i))
+                    dict_c_top[record][i] = dict_c[record][i][dict_c[record][i]['ref_strand'] == '+']
+                    print('{}: Successfully created a plus strand methyl-cytosine dictionary for '.format(readwrite.time_string()) + str(i))
+                    # In the strand specific dictionaries, only keep positions that are informative for GpC SMF
+                
+                if '6mA' in mods and '5mC' in mods:
+                    # Remove combined stranded dicts from the dicts to skip set
+                    dict_to_skip.difference_update(combined_dicts)                
+                    # Initialize the sample keys for the combined dictionaries
+
+                    if record not in dict_combined_bottom.keys() and record not in dict_combined_top.keys():
+                        dict_combined_bottom[record], dict_combined_top[record]= {}, {}
+
+                    print('{}: Successfully created a minus strand combined methylation dictionary for '.format(readwrite.time_string()) + str(i))
+                    dict_combined_bottom[record][i] = []
+                    print('{}: Successfully created a plus strand combined methylation dictionary for '.format(readwrite.time_string()) + str(i))
+                    dict_combined_top[record][i] = []
+
+        # Iterate over the stranded modification dictionaries and replace the dataframes with a dictionary of read names pointing to a list of values from the dataframe
+        for i, dict_type in enumerate(dict_list):
+            # Only iterate over stranded dictionaries
+            if i not in dict_to_skip:
+                print('{0}: Extracting methylation states for {1} dictionary'.format(readwrite.time_string(), sample_types[i]))
+                for record in dict_type.keys():
+                    # Get the dictionary for the modification type of interest from the reference mapping of interest
+                    dict = dict_type[record]
+                    print('{0}: Extracting methylation states for {1} dictionary'.format(readwrite.time_string(), record))
+                    # For each sample in a stranded dictionary
+                    for sample in dict.keys():
+                        print('{0}: Extracting {1} dictionary from record {2} for sample {3}'.format(readwrite.time_string(), sample_types[i], record, sample))
+                        # Load the combined bottom strand dictionary after all the individual dictionaries have been made for the sample
+                        if i == 7:
+                            # Load the minus strand dictionaries for each sample into temporary variables
+                            temp_a_dict = dict_list[2][record][sample].copy()
+                            temp_c_dict = dict_list[5][record][sample].copy()
+                            dict[sample] = {}
+                            # Iterate over the reads present in the merge of both dictionaries
+                            for read in set(temp_a_dict) | set(temp_c_dict):
+                                # Add the arrays element-wise if the read is present in both dictionaries
+                                if read in temp_a_dict and read in temp_c_dict:
+                                    dict[sample][read] = np.nansum([temp_a_dict[read], temp_c_dict[read]], axis=0)
+                                # If the read is present in only one dictionary, copy its value
+                                elif read in temp_a_dict:
+                                    dict[sample][read] = temp_a_dict[read]
+                                else:
+                                    dict[sample][read] = temp_c_dict[read]
+                    # Load the combined top strand dictionary after all the individual dictionaries have been made for the sample
+                        elif i == 8:
+                        # Load the plus strand dictionaries for each sample into temporary variables
+                            temp_a_dict = dict_list[3][record][sample].copy()
+                            temp_c_dict = dict_list[6][record][sample].copy()
+                            dict[sample] = {}
+                            # Iterate over the reads present in the merge of both dictionaries
+                            for read in set(temp_a_dict) | set(temp_c_dict):
+                                # Add the arrays element-wise if the read is present in both dictionaries
+                                if read in temp_a_dict and read in temp_c_dict:
+                                    dict[sample][read] = np.nansum([temp_a_dict[read], temp_c_dict[read]], axis=0)
+                                # If the read is present in only one dictionary, copy its value
+                                elif read in temp_a_dict:
+                                    dict[sample][read] = temp_a_dict[read]
+                                else:
+                                    dict[sample][read] = temp_c_dict[read]
+                        # For all other dictionaries
+                        else:
+                            # extract the dataframe from the dictionary into a temporary variable
+                            temp_df = dict[sample]
+                            # reassign the dictionary pointer to a nested dictionary.
+                            dict[sample] = {}
+                            # # Iterate through rows in the temp DataFrame
+                            for index, row in temp_df.iterrows():
+                                read = row['read_id'] # read name
+                                position = row['ref_position']  # positional coordinate
+                                probability = row['call_prob'] # Get the probability of the given call
+                                # if the call_code is modified change methylated value to the probability of methylation
+                                if (row['call_code'] in ['a', 'h', 'm']): 
+                                    methylated = probability
+                                # If the call code is canonical, change the methylated value to 1 - the probability of canonical
+                                elif (row['call_code'] in ['-']):
+                                    methylated = 1 - probability
+
+                                # If the current read is not in the dictionary yet, initalize the dictionary with a nan filled numpy array of proper size.
+                                if read not in dict[sample]:
+                                    dict[sample][read] = np.full(max_reference_length, np.nan) 
+                                else:
+                                    pass
+                                # add the positional methylation state to the numpy array
+                                dict[sample][read][position-1] = methylated
+
+        # Save the sample files in the batch as gzipped hdf5 files
+        print('{0}: Converting batch {1} dictionaries to anndata objects'.format(readwrite.time_string(), batch))
+        for i, dict_type in enumerate(dict_list):
+            if i not in dict_to_skip:
+                # Initialize an hdf5 file for the current modified strand
+                adata = None
+                print('{0}: Converting {1} dictionary to an anndata object'.format(readwrite.time_string(), sample_types[i]))
+                for record in dict_type.keys():
+                    # Get the dictionary for the modification type of interest from the reference mapping of interest
+                    dict = dict_type[record]
+                    for sample in dict.keys():
+                        print('{0}: Converting {1} dictionary for sample {2} to an anndata object'.format(readwrite.time_string(), sample_types[i], sample))
+                        sample = int(sample)
+                        final_sample_index = sample + (batch * batch_size)
+                        print('{0}: Final sample index for sample: {1}'.format(readwrite.time_string(), final_sample_index))
+                        print('{0}: Converting {1} dictionary for sample {2} to a dataframe'.format(readwrite.time_string(), sample_types[i], final_sample_index))
+                        temp_df = pd.DataFrame.from_dict(dict[sample], orient='index')
+                        sorted_index = sorted(temp_df.index)
+                        temp_df = temp_df.reindex(sorted_index)
+                        X = temp_df.values
+                        one_hot_encodings = record_seq_dict[record][0]
+                        read_names = list(one_hot_encodings.keys())
+                        sequence_length = one_hot_encodings[read_names[0]].shape[0]
+                        dict_A, dict_C, dict_G, dict_T, dict_N = {}, {}, {}, {}, {}
+                        # Loop through each read name and its corresponding one-hot array
+                        print('{0}: Extracting one hot encodings into dictionaries'.format(readwrite.time_string()))
+                        for read_name, one_hot_array in one_hot_encodings.items():
+                            dict_A[read_name] = one_hot_array[:, 0]
+                            dict_C[read_name] = one_hot_array[:, 1]
+                            dict_G[read_name] = one_hot_array[:, 2]
+                            dict_T[read_name] = one_hot_array[:, 3]
+                            dict_N[read_name] = one_hot_array[:, 4]
+                        # Load dfs with data from the dictionaries
+                        print('{0}: Loading dataframes from one hot encoded dictionaries'.format(readwrite.time_string()))
+                        df_A = pd.DataFrame.from_dict(dict_A, orient='index').reindex(sorted_index)
+                        df_C = pd.DataFrame.from_dict(dict_C, orient='index').reindex(sorted_index)
+                        df_G = pd.DataFrame.from_dict(dict_G, orient='index').reindex(sorted_index)
+                        df_T = pd.DataFrame.from_dict(dict_T, orient='index').reindex(sorted_index)
+                        df_N = pd.DataFrame.from_dict(dict_N, orient='index').reindex(sorted_index)
+
+                        ohe_df_map = {0: df_A, 1: df_C, 2: df_G, 3: df_T, 4: df_N}
+
+                        print('{0}: Loading {1} dataframe for sample {2} into a temp anndata object'.format(readwrite.time_string(), sample_types[i], final_sample_index))
+                        temp_adata = ad.AnnData(X, dtype=X.dtype)
+                        print('{0}: Adding read names and position ids to {1} anndata for sample {2}'.format(readwrite.time_string(), sample_types[i], final_sample_index))
+                        temp_adata.obs_names = temp_df.index
+                        temp_adata.obs_names = temp_adata.obs_names.astype(str)
+                        temp_adata.var_names = temp_df.columns
+                        temp_adata.var_names = temp_adata.var_names.astype(str)
+                        print('{0}: Adding final sample id to {1} anndata for sample {2}'.format(readwrite.time_string(), sample_types[i], final_sample_index))
+                        temp_adata.obs['Sample'] = [str(final_sample_index)] * len(temp_adata)
+                        dataset, strand = sample_types[i].split('_')[:2]
+                        temp_adata.obs['Strand'] = [strand] * len(temp_adata)
+                        temp_adata.obs['Dataset'] = [dataset] * len(temp_adata)
+                        temp_adata.obs['Reference'] = [f'{record}_{dataset}_{strand}'] * len(temp_adata)
+                        temp_adata.obs['Reference_chromosome'] = [f'{record}'] * len(temp_adata)
+
+                        for j, base in enumerate(['A', 'C', 'G', 'T', 'N']):
+                            temp_adata.layers[f'{base}_binary_encoding'] = ohe_df_map[j].values
+
+                        # If final adata object already has a sample loaded, concatenate the current sample into the existing adata object 
+                        if adata:
+                            print('{0}: Concatenating {1} anndata object for sample {2}'.format(readwrite.time_string(), sample_types[i], final_sample_index))
+                            adata = ad.concat([adata, temp_adata], join='outer', index_unique=None)
+                        else:
+                            print('{0}: Initializing {1} anndata object for sample {2}'.format(readwrite.time_string(), sample_types[i], final_sample_index))
+                            adata = temp_adata
+
+                print('{0}: Writing {1} anndata out as a gzipped hdf5 file'.format(readwrite.time_string(), sample_types[i]))
+                adata.write_h5ad('{0}_{1}_{2}_SMF_binarized_sample_hdf5.h5ad.gz'.format(readwrite.date_string(), batch, sample_types[i]), compression='gzip')
+
+        # Delete the batch dictionaries from memory
+        del dict_list
+        gc.collect()
+    
+    # Iterate over all of the batched hdf5 files and concatenate them.
+    files = os.listdir(os.getcwd())
+    # Name the final output file
+    final_hdf = '{0}_{1}_final_experiment_hdf5.h5ad.gz'.format(readwrite.date_string(), experiment_name)
+    # Filter file names that contain the search string in their filename and keep them in a list
+    hdfs = [hdf for hdf in files if 'hdf5.h5ad' in hdf and hdf != final_hdf]
+    # Sort file list by names and print the list of file names
+    hdfs.sort()
+    print('{0} sample files found: {1}'.format(len(hdfs), hdfs))
+    final_adata = None
+    for hdf in hdfs:
+        print('{0}: Reading in {1} hdf5 file'.format(readwrite.time_string(), hdf))
+        temp_adata = ad.read_h5ad(hdf)
+        if final_adata:
+            print('{0}: Concatenating final adata object with {1} hdf5 file'.format(readwrite.time_string(), hdf))
+            final_adata = ad.concat([final_adata, temp_adata], join='outer', index_unique=None)
+        else:
+            print('{0}: Initializing final adata object with {1} hdf5 file'.format(readwrite.time_string(), hdf))
+            final_adata = temp_adata
+    print('{0}: Writing final concatenated hdf5 file'.format(readwrite.time_string()))
+
+    for record in records_to_analyze:
+        # Add FASTA sequence to the object
+        sequence = record_seq_dict[record][1]
+        final_adata.uns[f'{record}_FASTA_sequence'] = sequence
+        final_adata.var[f'{record}_FASTA_sequence_base'] = list(sequence)
+
+        # Add consensus sequence of samples mapped to the record to the object
+        record_subset = final_adata[final_adata.obs['Reference_chromosome'] == record].copy()
+        layer_map, layer_counts = {}, []
+        for i, layer in enumerate(record_subset.layers):
+            layer_map[i] = layer.split('_')[0]
+            layer_counts.append(np.sum(record_subset.layers[layer], axis=0))
+        count_array = np.array(layer_counts)
+        nucleotide_indexes = np.argmax(count_array, axis=0)
+        consensus_sequence_list = [layer_map[i] for i in nucleotide_indexes]
+        final_adata.var[f'{record}_consensus_across_samples'] = consensus_sequence_list
+
+    final_adata.write_h5ad(final_hdf, compression='gzip')
+
+    # Delete the individual h5ad files and only keep the final concatenated file
+    files = os.listdir(os.getcwd())
+    hdfs_to_delete = [hdf for hdf in files if 'hdf5.h5ad' in hdf and hdf != final_hdf]
+    # Iterate over the files and delete them
+    for hdf in hdfs_to_delete:
+        try:
+            os.remove(hdf)
+            print(f"Deleted file: {hdf}")
+        except OSError as e:
+            print(f"Error deleting file {hdf}: {e}")

smftools/informatics/helpers/one_hot_encode.py

@@ -0,0 +1,14 @@
+# one_hot_encode
+from .. import readwrite
+
+# String encodings
+def one_hot_encode(sequence):
+    """
+    Input: A sequence string of a read.
+    Output: One hot encoding of the sequence string.
+    """
+    mapping = {'A': 0, 'C': 1, 'G': 2, 'T': 3, 'N': 4}
+    one_hot_matrix = np.zeros((len(sequence), 5), dtype=int)
+    for i, nucleotide in enumerate(sequence):
+        one_hot_matrix[i, mapping[nucleotide]] = 1
+    return one_hot_matrix

smftools/informatics/helpers/separate_bam_by_bc.py

@@ -0,0 +1,28 @@
+## separate_bam_by_bc
+import pysam
+
+# General
+def separate_bam_by_bc(input_bam, output_prefix):
+    """
+    Input: Takes a single BAM input. Also takes an output prefix to append to the output file.
+    Output: Splits the BAM based on the BC SAM tag value.
+    """
+    # Open the input BAM file for reading
+    with pysam.AlignmentFile(input_bam, "rb") as bam:
+        # Create a dictionary to store output BAM files
+        output_files = {}
+        # Iterate over each read in the BAM file
+        for read in bam:
+            try:
+                # Get the barcode tag value
+                bc_tag = read.get_tag("BC", with_value_type=True)[0].split('barcode')[1]
+                # Open the output BAM file corresponding to the barcode
+                if bc_tag not in output_files:
+                    output_files[bc_tag] = pysam.AlignmentFile(f"{output_prefix}_{bc_tag}.bam", "wb", header=bam.header)
+                # Write the read to the corresponding output BAM file
+                output_files[bc_tag].write(read)
+            except KeyError:
+                 print(f"BC tag not present for read: {read.query_name}")
+    # Close all output BAM files
+    for output_file in output_files.values():
+        output_file.close()

smftools/informatics/helpers/split_and_index_BAM.py

@@ -0,0 +1,21 @@
+## split_and_index_BAM
+from .. import readwrite
+import os
+import subprocess
+import glob
+from .separate_bam_by_bc import separate_bam_by_bc
+
+def split_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix):
+    """
+    A wrapper function for splitting BAMS and indexing them
+    """
+    os.chdir(split_dir)
+    aligned_sorted_output = aligned_sorted_BAM + bam_suffix
+    file_prefix = readwrite.datestring()
+    separate_bam_by_bc(aligned_sorted_output, file_prefix)
+    # Make a BAM index file for the BAMs in that directory
+    bam_pattern = '*' + bam_suffix
+    bam_files = glob.glob(os.path.join(split_dir, bam_pattern))
+    for input_file in bam_files:
+        subprocess.run(["samtools", "index", input_file])
+        print(f"Indexed {input_file}")

smftools/informatics/pod5_conversion.py

@@ -0,0 +1,26 @@
+## pod5_conversion
+from .helpers import align_BAM, canoncall, converted_BAM_to_adata, generate_converted_FASTA, split_and_index_BAM
+import subprocess
+
+def pod5_conversion(fasta, output_directory, conversion_types, strands, model, pod5_dir, split_dir, barcode_kit, mapping_threshold, experiment_name, bam_suffix):
+    """
+    Converts a POD5 file from a nanopore conversion SMF experiment to an adata object
+    """
+    bam=f"{output_directory}/HAC_basecalls"
+    aligned_BAM=f"{bam}_aligned"
+    aligned_sorted_BAM=f"{aligned_BAM}_sorted"
+    # 1) Convert FASTA file
+    converted_FASTA=fasta.split('.fa')[0]+'_converted.fasta'
+    generate_converted_FASTA(fasta, conversion_types, strands, converted_FASTA)
+
+    # 2) Basecall from the input POD5 to generate a singular output BAM
+    canoncall(model, pod5_dir, barcode_kit, bam, bam_suffix)
+
+    # 3) Align the BAM to the converted reference FASTA and sort the bam on positional coordinates. Also make an index and a bed file of mapped reads
+    align_BAM(converted_FASTA, bam, bam_suffix)
+
+    ### 4) Split the aligned and sorted BAM files by barcode (BC Tag) into the split_BAM directory###
+    split_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix)
+
+    # 5) Take the converted BAM and load it into an adata object. 
+    converted_BAM_to_adata(converted_FASTA, split_dir, mapping_threshold, experiment_name, conversion_types, bam_suffix)

smftools/informatics/pod5_direct.py

@@ -0,0 +1,29 @@
+## pod5_direct
+from .helpers import align_BAM, extract_mods, make_modbed, modcall, modkit_extract_to_adata, modQC, split_and_index_BAM
+
+def pod5_direct(fasta, output_directory, mod_list, model, thresholds, pod5_dir, split_dir, barcode_kit, mapping_threshold, experiment_name, bam_suffix, batch_size):
+    """
+    
+    """
+    bam=f"{output_directory}/HAC_mod_calls"
+    aligned_BAM=f"{bam}_aligned"
+    aligned_sorted_BAM=f"{aligned_BAM}_sorted"
+    mod_bed_dir=f"{output_directory}/split_mod_beds"
+    mod_tsv_dir=f"{output_directory}/split_mod_tsvs"
+
+    aligned_sorted_output = aligned_sorted_BAM + bam_suffix
+    mod_map = {'6mA': '6mA', '5mC_5hmC': '5mC'}
+    mods = [mod_map[mod] for mod in mod_list]
+
+    # 1) Basecall using dorado
+    modcall(model, pod5_dir, barcode_kit, mod_list, bam, bam_suffix)
+    # 2) Align the BAM to the converted reference FASTA. Also make an index and a bed file of mapped reads
+    align_BAM(fasta, bam, bam_suffix)
+    # 3) Split the aligned and sorted BAM files by barcode (BC Tag) into the split_BAM directory
+    split_and_index_BAM(aligned_sorted_BAM, split_dir, bam_suffix)
+    # 4) Using nanopore modkit to work with modified BAM files ###
+    modQC(aligned_sorted_output, thresholds) # get QC metrics for mod calls
+    make_modbed(aligned_sorted_output, thresholds, mod_bed_dir) # Generate bed files of position methylation summaries for every sample
+    extract_mods(thresholds, mod_tsv_dir, split_dir, bam_suffix) # Extract methylations calls for split BAM files into split TSV files
+    #5 Load the modification data from TSVs into an adata object
+    modkit_extract_to_adata(fasta, aligned_sorted_output, mapping_threshold, experiment_name, mods, batch_size)

smftools/informatics/pod5_to_adata.py

@@ -0,0 +1,17 @@
+## pod5_to_adata
+from .helpers import load_experiment_config
+from.pod5_direct import pod5_direct
+from.pod5_conversion import pod5_conversion
+
+def pod5_to_adata(config_path, ):
+    """
+    
+    """
+    # Load experiment config parameters into global variables
+    load_experiment_config(config_path)
+    if smf_modality == 'conversion':
+        (fasta, output_directory, conversion_types, strands, model, pod5_dir, split_dir, barcode_kit, mapping_threshold, experiment_name, bam_suffix)
+    elif smf_modality == 'direct':
+        pod5_direct(fasta, output_directory, mod_list, model, thresholds, pod5_dir, split_dir, barcode_kit, mapping_threshold, experiment_name, bam_suffix, batch_size)
+    else:
+        print("Error")

smftools/informatics/readwrite.py

@@ -0,0 +1,109 @@
+## readwrite ##
+
+# Basic I/O
+import os
+# Datetime
+from datetime import datetime
+# Data structures and basic operations
+import math
+import numpy as np
+import pandas as pd
+import anndata as ad
+import scipy.sparse as sp
+
+# Runtime warnings
+import warnings
+warnings.filterwarnings('ignore', category=UserWarning, module='anndata')
+warnings.filterwarnings('ignore', category=FutureWarning, module='anndata')
+
+######################################################################################################
+## Datetime functionality
+def date_string():
+    """
+    Each time this is called, it returns the current date string
+    """
+    current_date = datetime.now()
+    date_string = current_date.strftime("%Y%m%d")
+    date_string = date_string[2:]
+    return date_string
+
+def time_string():
+    """
+    Each time this is called, it returns the current time string
+    """
+    current_time = datetime.now()
+    return current_time.strftime("%H:%M:%S")
+######################################################################################################
+
+######################################################################################################
+## Numpy, Pandas, Anndata functionality
+def adata_to_df(adata, layer=None):
+    """
+    Input: An adata object with a specified layer.
+    Output: A dataframe for the specific layer.
+    """
+    # Extract the data matrix from the given layer
+    if layer:
+        data_matrix = adata.layers[layer]
+    else:
+        data_matrix = adata.X
+    # Extract observation (read) annotations
+    obs_df = adata.obs
+    # Extract variable (position) annotations
+    var_df = adata.var
+    # Convert data matrix and annotations to pandas DataFrames
+    df = pd.DataFrame(data_matrix, index=obs_df.index, columns=var_df.index)
+    return df
+
+def save_matrix(matrix, save_name):
+    """
+    Input: A numpy matrix and a save_name
+    Output: A txt file representation of the data matrix
+    """
+    np.savetxt(f'{save_name}.txt', matrix)
+
+def concatenate_h5ads(output_file, file_suffix='h5ad.gz', delete_inputs=True):
+    """
+    Concatenate all h5ad files in a directory and delete them after the final adata is written out.
+    Input: an output file path relative to the directory in which the function is called
+    """
+    # List all files in the directory
+    files = os.listdir(os.getcwd())
+    # get current working directory
+    cwd = os.getcwd()
+    suffix = file_suffix
+    # Filter file names that contain the search string in their filename and keep them in a list
+    hdfs = [hdf for hdf in files if suffix in hdf]
+    # Sort file list by names and print the list of file names
+    hdfs.sort()
+    print('{0} sample files found: {1}'.format(len(hdfs), hdfs))
+    # Iterate over all of the hdf5 files and concatenate them.
+    final_adata = None
+    for hdf in hdfs:
+        print('{0}: Reading in {1} hdf5 file'.format(time_string(), hdf))
+        temp_adata = ad.read_h5ad(hdf)
+        if final_adata:
+            print('{0}: Concatenating final adata object with {1} hdf5 file'.format(time_string(), hdf))
+            final_adata = ad.concat([final_adata, temp_adata], join='outer', index_unique=None)
+        else:
+            print('{0}: Initializing final adata object with {1} hdf5 file'.format(time_string(), hdf))
+            final_adata = temp_adata
+    print('{0}: Writing final concatenated hdf5 file'.format(time_string()))
+    final_adata.write_h5ad(output_file, compression='gzip')
+
+    # Delete the individual h5ad files and only keep the final concatenated file
+    if delete_inputs:
+        files = os.listdir(os.getcwd())
+        hdfs = [hdf for hdf in files if suffix in hdf]
+        if output_file in hdfs:
+            hdfs.remove(output_file)
+            # Iterate over the files and delete them
+            for hdf in hdfs:
+                try:
+                    os.remove(hdf)
+                    print(f"Deleted file: {hdf}")
+                except OSError as e:
+                    print(f"Error deleting file {hdf}: {e}")
+    else:
+        print('Keeping input files')
+######################################################################################################

smftools/preprocessing/__init__.py

@@ -0,0 +1,35 @@
+from .append_C_context import append_C_context
+from .binarize_on_Youden import binarize_on_Youden
+from .binary_layers_to_ohe import binary_layers_to_ohe
+from .calculate_complexity import calculate_complexity
+from .calculate_converted_read_methylation_stats import calculate_converted_read_methylation_stats
+from .calculate_coverage import calculate_coverage
+from .calculate_pairwise_hamming_distances import calculate_pairwise_hamming_distances
+from .calculate_position_Youden import calculate_position_Youden
+from .calculate_read_length_stats import calculate_read_length_stats
+from .clean_NaN import clean_NaN
+from .filter_converted_reads_on_methylation import filter_converted_reads_on_methylation
+from .filter_reads_on_length import filter_reads_on_length
+from .invert_adata import invert_adata
+from .mark_duplicates import mark_duplicates
+from .min_non_diagonal import min_non_diagonal
+from .remove_duplicates import remove_duplicates
+
+__all__ = [
+    "append_C_context",
+    "binarize_on_Youden",
+    "binary_layers_to_ohe",
+    "calculate_complexity",
+    "calculate_converted_read_methylation_stats",
+    "calculate_coverage",    
+    "calculate_pairwise_hamming_distances",
+    "calculate_position_Youden",
+    "calculate_read_length_stats",
+    "clean_NaN",   
+    "filter_converted_reads_on_methylation",
+    "filter_reads_on_length",
+    "invert_adata",
+    "mark_duplicates",
+    "min_non_diagonal",
+    "remove_duplicates"
+]