smftools 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

smftools/informatics/helpers/binarize_converted_base_identities.py

@@ -5,7 +5,7 @@ def binarize_converted_base_identities(base_identities, strand, modification_typ
     Binarizes conversion SMF data within a sequence string
 
     Parameters:
-        base_identities (dict): A dictionary returned by extract_base_identity_at_coordinates.
+        base_identities (dict): A dictionary returned by extract_base_identities. Keyed by read name. Points to a list of base identities.
         strand (str): A string indicating which strand was converted in the experiment (options are 'top' and 'bottom').
         modification_type (str): A string indicating the modification type of interest (options are '5mC' and '6mA').
     
@@ -27,5 +27,5 @@ def binarize_converted_base_identities(base_identities, strand, modification_typ
             elif modification_type == '6mA':
                 binarized_base_identities[key] = [1 if x == 'T' else 0 if x == 'C' else np.nan for x in base_identities[key]]
         else:
-            pass
+            print(f"{strand} not recognized")
     return binarized_base_identities

smftools/informatics/helpers/canoncall.py

@@ -19,5 +19,7 @@ def canoncall(model, pod5_dir, barcode_kit, bam, bam_suffix):
     import subprocess
     output = bam + bam_suffix
     command = ["dorado", "basecaller", model, pod5_dir, "--kit-name", barcode_kit, "-Y"]
+    command_string = " ".join(command)
+    print(f"Running {command_string}\n to generate {output}")
     with open(output, "w") as outfile:
         subprocess.run(command, stdout=outfile)

smftools/informatics/helpers/complement_base_list.py

@@ -0,0 +1,21 @@
+# complement_base_list
+
+def complement_base_list(sequence):
+    """
+    Takes a list of DNA base identities and returns their complement.
+
+    Parameters:
+        sequence (list): A list of DNA bases (e.g., ['A', 'C', 'G', 'T']).
+
+    Returns:
+        complement (list): A list of complementary DNA bases.
+    """
+    complement_mapping = {
+        'A': 'T',
+        'T': 'A',
+        'C': 'G',
+        'G': 'C',
+        'N': 'N'  # Handling ambiguous bases like 'N'
+    }
+
+    return [complement_mapping[base] for base in sequence]

smftools/informatics/helpers/concatenate_fastqs_to_bam.py

@@ -0,0 +1,54 @@
+# concatenate_fastqs_to_bam
+
+def concatenate_fastqs_to_bam(fastq_files, output_bam, barcode_tag='BC', gzip_suffix='.gz'):
+    """
+    Concatenate multiple demultiplexed FASTQ (.fastq or .fq) files into an unaligned BAM and add the FASTQ barcode suffix to the BC tag.
+
+    Parameters:
+        fastq_files (list): List of paths to demultiplexed FASTQ files.
+        output_bam (str): Path to the output BAM file.
+        barcode_tag (str): The SAM tag for storing the barcode (default: 'BC').
+        gzip_suffix (str): Suffix to use for input gzip files (Defaul: '.gz')
+
+    Returns:
+        None
+    """
+    import os
+    import pysam
+    import gzip
+    from Bio import SeqIO
+    from tqdm import tqdm
+
+    n_fastqs = len(fastq_files)
+
+    with pysam.AlignmentFile(output_bam, "wb", header={"HD": {"VN": "1.0"}, "SQ": []}) as bam_out:
+        for fastq_file in tqdm(fastq_files, desc="Processing FASTQ files"):
+            # Extract barcode from the FASTQ filename (handles .fq, .fastq, .fq.gz, and .fastq.gz extensions)
+            base_name = os.path.basename(fastq_file)
+            if n_fastqs > 1:
+                if base_name.endswith('.fastq.gz'):
+                    barcode = base_name.split('_')[-1].replace(f'.fastq{gzip_suffix}', '')
+                elif base_name.endswith('.fq.gz'):
+                    barcode = base_name.split('_')[-1].replace(f'.fq{gzip_suffix}', '')
+                elif base_name.endswith('.fastq'):
+                    barcode = base_name.split('_')[-1].replace('.fastq', '')
+                elif base_name.endswith('.fq'):
+                    barcode = base_name.split('_')[-1].replace('.fq', '')
+                else:
+                    raise ValueError(f"Unexpected file extension for {fastq_file}. Only .fq, .fastq, .fq{gzip_suffix}, and .fastq{gzip_suffix} are supported.")
+            
+            # Read the FASTQ file (handle gzipped and non-gzipped files)
+            open_func = gzip.open if fastq_file.endswith(gzip_suffix) else open
+            with open_func(fastq_file, 'rt') as fq_in:
+                for record in SeqIO.parse(fq_in, 'fastq'):
+                    # Create an unaligned BAM entry for each FASTQ record
+                    aln = pysam.AlignedSegment()
+                    aln.query_name = record.id
+                    aln.query_sequence = str(record.seq)
+                    aln.flag = 4  # Unmapped
+                    aln.query_qualities = pysam.qualitystring_to_array(record.letter_annotations["phred_quality"])
+                    if n_fastqs > 1:
+                        # Add the barcode to the BC tag
+                        aln.set_tag(barcode_tag, barcode)
+                    # Write to BAM file
+                    bam_out.write(aln)

smftools/informatics/helpers/converted_BAM_to_adata.py

@@ -21,144 +21,213 @@ def converted_BAM_to_adata(converted_FASTA, split_dir, mapping_threshold, experi
     from .find_conversion_sites import find_conversion_sites
     from .count_aligned_reads import count_aligned_reads
     from .extract_base_identities import extract_base_identities
-    from .one_hot_encode import one_hot_encode
+    from .make_dirs import make_dirs
+    from .ohe_batching import ohe_batching
     import pandas as pd
     import numpy as np
     import anndata as ad
     import os
-
+    from tqdm import tqdm
+    import gc
+    
+    ##########################################################################################
+    ## Get file paths and make necessary directories. ##
     # Get all of the input BAM files
     files = os.listdir(split_dir)
-    # Change directory to the BAM directory
-    os.chdir(split_dir)
+    # Make output dir
+    parent_dir = os.path.dirname(split_dir)
+    h5_dir = os.path.join(parent_dir, 'h5ads')
+    tmp_dir = os.path.join(parent_dir, 'tmp')
+    make_dirs([h5_dir, tmp_dir])
     # Filter file names that contain the search string in their filename and keep them in a list
     bams = [bam for bam in files if bam_suffix in bam and '.bai' not in bam]
     # Sort file list by names and print the list of file names
     bams.sort()
+    bam_path_list = [os.path.join(split_dir, bam) for bam in bams]
     print(f'Found the following BAMS: {bams}')
     final_adata = None
+    ##########################################################################################
+
+    ##########################################################################################
 
-    # Make a dictionary, keyed by modification type, that points to another dictionary of unconverted_record_ids. This points to a list of: 1) record length, 2) top strand conversion coordinates, 3) bottom strand conversion coordinates, 4) record sequence
+    ## need to fix this section
+    # Make a dictionary, keyed by modification type, that points to another dictionary of unconverted_record_ids. This points to a list of: 1) record length, 2) top strand conversion coordinates, 3) bottom strand conversion coordinates, 4) sequence string unconverted , 5) Complement sequence unconverted
     modification_dict = {}
+    # Init a dict to be keyed by FASTA record that points to the sequence string of the unconverted record
+    record_FASTA_dict = {}
     # While populating the dictionary, also extract the longest sequence record in the input references
     max_reference_length = 0
     for conversion_type in conversion_types:
+        # Points to a list containing: 1) sequence length of the record, 2) top strand coordinate list, 3) bottom strand coorinate list, 4) sequence string unconverted , 5) Complement sequence unconverted
         modification_dict[conversion_type] = find_conversion_sites(converted_FASTA, conversion_type, conversion_types)
+        # Get the max reference length
         for record in modification_dict[conversion_type].keys():
             if modification_dict[conversion_type][record][0] > max_reference_length:
                 max_reference_length = modification_dict[conversion_type][record][0]
 
-    # Init a dict to be keyed by FASTA record that points to the sequence string of the unconverted record
-    record_FASTA_dict = {}
+            mod_type, strand = record.split('_')[-2:]
 
-    # Iterate over the experiment BAM files
-    for bam_index, bam in enumerate(bams):
-        # Give each bam a sample name
-        sample = bam.split(sep=bam_suffix)[0]
+            chromosome = record.split('_{0}_{1}'.format(mod_type, strand))[0]
+            unconverted_chromosome_name = f'{chromosome}_{conversion_types[0]}_top'
+            current_reference_length = modification_dict[mod_type][unconverted_chromosome_name][0]
+            delta_max_length = max_reference_length - current_reference_length
+            sequence = modification_dict[mod_type][unconverted_chromosome_name][3] + 'N'*delta_max_length
+            complement = modification_dict[mod_type][unconverted_chromosome_name][4] + 'N'*delta_max_length
+            record_FASTA_dict[record] = [sequence, complement, chromosome, unconverted_chromosome_name, current_reference_length, delta_max_length, conversion_type, strand]
+    ##########################################################################################
+
+    ##########################################################################################
+    bam_alignment_stats_dict = {}
+    records_to_analyze = []
+    for bam_index, bam in enumerate(bam_path_list):
+        bam_alignment_stats_dict[bam_index] = {}
         # look at aligned read proportions in the bam
         aligned_reads_count, unaligned_reads_count, record_counts = count_aligned_reads(bam)
         percent_aligned = aligned_reads_count*100 / (aligned_reads_count+unaligned_reads_count)
-        print(f'{percent_aligned} percent of total reads in {bam} aligned successfully')
-        records_to_analyze = []
+        print(f'{percent_aligned} percent of total reads in {bams[bam_index]} aligned successfully')
+        bam_alignment_stats_dict[bam_index]['Total'] = (aligned_reads_count, percent_aligned)
         # Iterate over converted reference strands and decide which to use in the analysis based on the mapping_threshold
         for record in record_counts:
             print(f'{record_counts[record][0]} reads mapped to reference record {record}. This is {record_counts[record][1]*100} percent of all mapped reads in the sample.')
             if record_counts[record][1] >= mapping_threshold:
                 records_to_analyze.append(record)
-        print(f'Records to analyze: {records_to_analyze}')
-        # Iterate over records to analyze (ie all conversions detected)
+                bam_alignment_stats_dict[bam_index]
+                bam_alignment_stats_dict[bam_index][record] = (record_counts[record][0], record_counts[record][1]*100)
+    records_to_analyze = set(records_to_analyze)
+    ##########################################################################################
+
+    ##########################################################################################
+    # One hot encode read sequences and write them out into the tmp_dir as h5ad files. 
+    # Save the file paths in the bam_record_ohe_files dict.
+    bam_record_ohe_files = {}
+
+    # Iterate over split bams
+    for bam_index, bam in enumerate(bam_path_list):
+        # Iterate over references to process
         for record in records_to_analyze:
-            mod_type, strand = record.split('_')[-2:]
-            if strand == 'top':
-                strand_index = 1
-            elif strand == 'bottom':
-                strand_index = 2
-
-            chromosome = record.split('_{0}_{1}'.format(mod_type, strand))[0]
-            unconverted_chromosome_name = f'{chromosome}_{conversion_types[0]}_top'
-            positions = modification_dict[mod_type][unconverted_chromosome_name][strand_index]
-            current_reference_length = modification_dict[mod_type][unconverted_chromosome_name][0]
-            delta_max_length = max_reference_length - current_reference_length
-            sequence = modification_dict[mod_type][unconverted_chromosome_name][3] + 'N'*delta_max_length
-            record_FASTA_dict[f'{record}'] = sequence
-            print(f'Chromosome: {chromosome}\nUnconverted Sequence: {sequence}')
+            unconverted_record_name = "_".join(record.split('_')[:-2]) + '_unconverted_top'
+            sample = bams[bam_index].split(sep=bam_suffix)[0]
+            chromosome = record_FASTA_dict[unconverted_record_name][2]
+            current_reference_length = record_FASTA_dict[unconverted_record_name][4]
+            mod_type = record_FASTA_dict[unconverted_record_name][6]
+            strand = record_FASTA_dict[unconverted_record_name][7]
+            
+            # Extract the base identities of reads aligned to the record
+            fwd_base_identities, rev_base_identities = extract_base_identities(bam, record, range(current_reference_length), max_reference_length)
 
-            # Get a dictionary of positional identities keyed by read id
-            print(f'Extracting base identities of target positions')
-            target_base_identities = extract_base_identities(bam, record, positions, max_reference_length) 
             # binarize the dictionary of positional identities
-            print(f'Binarizing base identities of target positions')
-            binarized_base_identities = binarize_converted_base_identities(target_base_identities, strand, mod_type) 
+            print(f'Binarizing base identities')
+            fwd_binarized_base_identities = binarize_converted_base_identities(fwd_base_identities, strand, mod_type) 
+            rev_binarized_base_identities = binarize_converted_base_identities(rev_base_identities, strand, mod_type)
+            merged_binarized_base_identities = {**fwd_binarized_base_identities, **rev_binarized_base_identities}
             # converts the base identity dictionary to a dataframe.
-            binarized_base_identities_df = pd.DataFrame.from_dict(binarized_base_identities, orient='index') 
+            binarized_base_identities_df = pd.DataFrame.from_dict(merged_binarized_base_identities, orient='index') 
             sorted_index = sorted(binarized_base_identities_df.index)
             binarized_base_identities_df = binarized_base_identities_df.reindex(sorted_index)
-            # Get the sequence string of every read
-            print(f'Extracting base identities of all positions in each read')
-            all_base_identities = extract_base_identities(bam, record, range(current_reference_length), max_reference_length)
-            # One hot encode the sequence string of the reads
-            print(f'One hot encoding base identities of all positions in each read')
-            one_hot_reads = {read_name: one_hot_encode(seq) for read_name, seq in all_base_identities.items()}
-
-            # Initialize empty DataFrames for each base
-            read_names = list(one_hot_reads.keys())
-            sequence_length = one_hot_reads[read_names[0]].shape[0]
-            df_A = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
-            df_C = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
-            df_G = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
-            df_T = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
-            df_N = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
-
-            # Iterate through the dictionary and populate the DataFrames
-            for read_name, one_hot_array in one_hot_reads.items():
-                df_A.loc[read_name] = one_hot_array[:, 0]
-                df_C.loc[read_name] = one_hot_array[:, 1]
-                df_G.loc[read_name] = one_hot_array[:, 2]
-                df_T.loc[read_name] = one_hot_array[:, 3]
-                df_N.loc[read_name] = one_hot_array[:, 4]
-
-            ohe_df_map = {0: df_A, 1: df_C, 2: df_G, 3: df_T, 4: df_N}   
 
             # Load an anndata object with the sample data
             X = binarized_base_identities_df.values
             adata = ad.AnnData(X, dtype=X.dtype)
-            adata.obs_names = binarized_base_identities_df.index
-            adata.obs_names = adata.obs_names.astype(str)
-            adata.var_names = binarized_base_identities_df.columns
-            adata.var_names = adata.var_names.astype(str)
-            adata.obs['Sample'] = [sample] * len(adata)
-            adata.obs['Strand'] = [strand] * len(adata)
-            adata.obs['Dataset'] = [mod_type] * len(adata)
-            adata.obs['Reference'] = [record] * len(adata)
-            adata.obs['Reference_chromosome'] = [chromosome] * len(adata)
-            
-            for j, base in enumerate(['A', 'C', 'G', 'T', 'N']):
-                adata.layers[f'{base}_binary_encoding'] = ohe_df_map[j].values 
+            if adata.shape[0] > 0:
+                adata.obs_names = binarized_base_identities_df.index.astype(str)
+                adata.var_names = binarized_base_identities_df.columns.astype(str)
+                adata.obs['Sample'] = [sample] * len(adata)
+                adata.obs['Strand'] = [strand] * len(adata)
+                adata.obs['Dataset'] = [mod_type] * len(adata)
+                adata.obs['Reference'] = [record] * len(adata)
+                adata.obs['Reference_chromosome'] = [chromosome] * len(adata)
+
+                read_mapping_direction = []
+                for read_id in adata.obs_names:
+                    if read_id in fwd_base_identities.keys():
+                        read_mapping_direction.append('fwd')
+                    elif read_id in rev_base_identities.keys():
+                        read_mapping_direction.append('rev')
+                    else:
+                        read_mapping_direction.append('unk')
+
+                adata.obs['Read_mapping_direction'] = read_mapping_direction
+
+                # One hot encode the sequence string of the reads
+                fwd_ohe_files = ohe_batching(fwd_base_identities, tmp_dir, record, f"{bam_index}_fwd",batch_size=100000)
+                rev_ohe_files = ohe_batching(rev_base_identities, tmp_dir, record, f"{bam_index}_rev",batch_size=100000)
+                bam_record_ohe_files[f'{bam_index}_{record}'] = fwd_ohe_files + rev_ohe_files
+                del fwd_base_identities, rev_base_identities
+
+                one_hot_reads = {}
+                n_rows_OHE = 5
+                for ohe_file in tqdm(bam_record_ohe_files[f'{bam_index}_{record}'], desc="Reading in OHE reads"):
+                    tmp_ohe_dict = ad.read_h5ad(ohe_file).uns
+                    one_hot_reads.update(tmp_ohe_dict)
+                    del tmp_ohe_dict
+
+                read_names = list(one_hot_reads.keys())
+                dict_A, dict_C, dict_G, dict_T, dict_N = {}, {}, {}, {}, {}
+
+                sequence_length = one_hot_reads[read_names[0]].reshape(n_rows_OHE, -1).shape[1]
+                df_A = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
+                df_C = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
+                df_G = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
+                df_T = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
+                df_N = pd.DataFrame(0, index=sorted_index, columns=range(sequence_length))
+
+                for read_name, one_hot_array in one_hot_reads.items():
+                    one_hot_array = one_hot_array.reshape(n_rows_OHE, -1)
+                    dict_A[read_name] = one_hot_array[0, :]
+                    dict_C[read_name] = one_hot_array[1, :]
+                    dict_G[read_name] = one_hot_array[2, :]
+                    dict_T[read_name] = one_hot_array[3, :]
+                    dict_N[read_name] = one_hot_array[4, :]
+
+                del one_hot_reads
+                gc.collect()
+
+                for j, read_name in tqdm(enumerate(sorted_index), desc='Loading dataframes of OHE reads', total=len(sorted_index)):
+                    df_A.iloc[j] = dict_A[read_name]
+                    df_C.iloc[j] = dict_C[read_name]
+                    df_G.iloc[j] = dict_G[read_name]
+                    df_T.iloc[j] = dict_T[read_name]
+                    df_N.iloc[j] = dict_N[read_name]
+
+                del dict_A, dict_C, dict_G, dict_T, dict_N
+                gc.collect()
+
+                ohe_df_map = {0: df_A, 1: df_C, 2: df_G, 3: df_T, 4: df_N}
+                
+                for j, base in enumerate(['A', 'C', 'G', 'T', 'N']):
+                    adata.layers[f'{base}_binary_encoding'] = ohe_df_map[j].values 
+                    ohe_df_map[j] = None # Reassign pointer for memory usage purposes
+
+                if final_adata:
+                    if adata.shape[0] > 0:
+                        final_adata = ad.concat([final_adata, adata], join='outer', index_unique=None)
+                    else:
+                        print(f"{sample} did not have any mapped reads on {record}, omiting from final adata")
+                else:
+                    if adata.shape[0] > 0:
+                        final_adata = adata
+                    else:
+                        print(f"{sample} did not have any mapped reads on {record}, omiting from final adata")
 
-            if final_adata:
-                final_adata = ad.concat([final_adata, adata], join='outer', index_unique=None)
             else:
-                final_adata = adata
-
-    for record in record_FASTA_dict.keys():
-        chromosome = record.split('_')[0]
-        sequence = record_FASTA_dict[record]
+                print(f"{sample} did not have any mapped reads on {record}, omiting from final adata")
+
+    # Set obs columns to type 'category'
+    for col in final_adata.obs.columns:
+        final_adata.obs[col] = final_adata.obs[col].astype('category')
+
+    for record in records_to_analyze:
+        unconverted_record_name = "_".join(record.split('_')[:-2]) + '_unconverted_top'
+        sequence = record_FASTA_dict[unconverted_record_name][0]
+        complement = record_FASTA_dict[unconverted_record_name][1]
+        chromosome = record_FASTA_dict[unconverted_record_name][2]
+        final_adata.var[f'{chromosome}_unconverted_top_strand_FASTA_base'] = list(sequence)
+        final_adata.var[f'{chromosome}_unconverted_bottom_strand_FASTA_base'] = list(complement)
         final_adata.uns[f'{record}_FASTA_sequence'] = sequence
-        final_adata.var[f'{record}_FASTA_sequence'] = list(sequence)
-
-        # May need to remove the bottom for conversion SMF
-        record_subset = final_adata[final_adata.obs['Reference'] == record].copy()
-        layer_map, layer_counts = {}, []
-        for i, layer in enumerate(record_subset.layers):
-            layer_map[i] = layer.split('_')[0]
-            layer_counts.append(np.sum(record_subset.layers[layer], axis=0))
-        count_array = np.array(layer_counts)
-        nucleotide_indexes = np.argmax(count_array, axis=0)
-        consensus_sequence_list = [layer_map[i] for i in nucleotide_indexes]
-        final_adata.var[f'{record}_consensus_across_samples'] = consensus_sequence_list
 
     ######################################################################################################
 
     ######################################################################################################
     ## Export the final adata object
-    final_adata.write_h5ad('{0}_{1}.h5ad.gz'.format(readwrite.date_string(), experiment_name), compression='gzip')
+    final_output = os.path.join(h5_dir, f'{readwrite.date_string()}_{experiment_name}.h5ad.gz')
+    final_adata.write_h5ad(final_output, compression='gzip')

smftools/informatics/helpers/count_aligned_reads.py

@@ -16,24 +16,28 @@ def count_aligned_reads(bam_file):
     """
     from .. import readwrite
     import pysam
+    from tqdm import tqdm
+    from collections import defaultdict
+
     print('{0}: Counting aligned reads in BAM > {1}'.format(readwrite.time_string(), bam_file))
     aligned_reads_count = 0
     unaligned_reads_count = 0
     # Make a dictionary, keyed by the reference_name of reference chromosome that points to an integer number of read counts mapped to the chromosome, as well as the proportion of mapped reads in that chromosome
-    record_counts = {}
+    record_counts = defaultdict(int)
+
     with pysam.AlignmentFile(bam_file, "rb") as bam:
+        total_reads = bam.mapped + bam.unmapped
         # Iterate over reads to get the total mapped read counts and the reads that map to each reference
-        for read in bam:
-            if read.is_unmapped: 
+        for read in tqdm(bam, desc='Counting aligned reads in BAM', total=total_reads):
+            if read.is_unmapped:
                 unaligned_reads_count += 1
-            else: 
+            else:
                 aligned_reads_count += 1
-                if read.reference_name in record_counts:
-                    record_counts[read.reference_name] += 1
-                else:
-                    record_counts[read.reference_name] = 1
+                record_counts[read.reference_name] += 1  # Automatically increments if key exists, adds if not
+
         # reformat the dictionary to contain read counts mapped to the reference, as well as the proportion of mapped reads in reference
         for reference in record_counts:
             proportion_mapped_reads_in_record = record_counts[reference] / aligned_reads_count
             record_counts[reference] = (record_counts[reference], proportion_mapped_reads_in_record)
-    return aligned_reads_count, unaligned_reads_count, record_counts
+
+    return aligned_reads_count, unaligned_reads_count, dict(record_counts)

smftools/informatics/helpers/extract_base_identities.py

@@ -3,7 +3,7 @@
 # General
 def extract_base_identities(bam_file, chromosome, positions, max_reference_length):
     """
-    Extracts the base identities from every position within the read that has a reference coordinate
+    Extracts the base identities from every position within the mapped reads that have a reference coordinate
 
     Parameters:
         bam (str): File path to the BAM file to align (excluding the file suffix).
@@ -12,32 +12,46 @@ def extract_base_identities(bam_file, chromosome, positions, max_reference_lengt
         max_reference_length (int): The maximum length of a record in the reference set.
 
     Returns:
-        base_identities (dict): A dictionary, keyed by read name, that points to a list of base identities. If the read does not contain that position, fill the list at that index with a N value.
-    
+        fwd_base_identities (dict): A dictionary, keyed by read name, that points to a list of base identities from forward mapped reads. If the read does not contain that position, fill the list at that index with a N value.
+        rev_base_identities (dict): A dictionary, keyed by read name, that points to a list of base identities from reverse mapped reads. If the read does not contain that position, fill the list at that index with a N value.
     """
     from .. import readwrite
     import pysam
+    from tqdm import tqdm
+    
     positions = set(positions)
     # Initialize a base identity dictionary that will hold key-value pairs that are: key (read-name) and value (list of base identities at positions of interest)
-    base_identities = {}
+    fwd_base_identities = {}
+    rev_base_identities = {}
     # Open the postion sorted BAM file
     print('{0}: Reading BAM file: {1}'.format(readwrite.time_string(), bam_file))
     with pysam.AlignmentFile(bam_file, "rb") as bam:
         # Iterate over every read in the bam that comes from the chromosome of interest
         print('{0}: Iterating over reads in bam'.format(readwrite.time_string()))
-        for read in bam.fetch(chromosome): 
-            if read.query_name in base_identities:
-                pass
-                #print('Duplicate read found in BAM for read {}. Skipping duplicate'.format(read.query_name))
-            else:          
-                # Initialize the read key in the base_identities dictionary by pointing to a N filled list of length reference_length
-                base_identities[read.query_name] = ['N'] * max_reference_length
-                # Iterate over a list of tuples for the given read. The tuples contain the 0-indexed position relative to the read start, as well the 0-based index relative to the reference.
-                for read_position, reference_position in read.get_aligned_pairs():
-                    # If the aligned read's reference coordinate is in the positions set and if the read position was successfully mapped
-                    if reference_position in positions and read_position:
-                        # get the base_identity in the read corresponding to that position
-                        base_identity = read.query_sequence[read_position]
-                        # Add the base identity to array
-                        base_identities[read.query_name][reference_position] = base_identity
-    return base_identities
+        total_reads = bam.mapped
+        for read in tqdm(bam.fetch(chromosome), desc='Extracting base identities from reads in BAM', total=total_reads):  
+            # Only iterate over mapped reads
+            if read.is_mapped:
+                # Get sequence of read. PySam reports fwd mapped reads as the true read sequence. Pysam reports rev mapped reads as the reverse complement of the read.
+                query_sequence = read.query_sequence
+                # If the read aligned as a reverse complement, mark that the read is reversed
+                if read.is_reverse:
+                    # Initialize the read key in a temp base_identities dictionary by pointing to a N filled list of length reference_length.
+                    rev_base_identities[read.query_name] = ['N'] * max_reference_length
+                    # Iterate over a list of tuples for the given read. The tuples contain the 0-indexed position relative to the read.query_sequence start, as well the 0-based index relative to the reference.
+                    for read_position, reference_position in read.get_aligned_pairs(matches_only=True):
+                        # If the aligned read's reference coordinate is in the positions set and if the read position was successfully mapped
+                        if reference_position in positions and read_position:
+                            # get the base_identity in the read corresponding to that position
+                            rev_base_identities[read.query_name][reference_position] = query_sequence[read_position]
+                else:
+                    # Initialize the read key in a temp base_identities dictionary by pointing to a N filled list of length reference_length.
+                    fwd_base_identities[read.query_name] = ['N'] * max_reference_length
+                    # Iterate over a list of tuples for the given read. The tuples contain the 0-indexed position relative to the read.query_sequence start, as well the 0-based index relative to the reference.
+                    for read_position, reference_position in read.get_aligned_pairs(matches_only=True):
+                        # If the aligned read's reference coordinate is in the positions set and if the read position was successfully mapped
+                        if reference_position in positions and read_position:
+                            # get the base_identity in the read corresponding to that position
+                            fwd_base_identities[read.query_name][reference_position] = query_sequence[read_position]
+
+    return fwd_base_identities, rev_base_identities

smftools/informatics/helpers/extract_readnames_from_BAM.py

@@ -0,0 +1,22 @@
+# extract_readnames_from_BAM
+
+def extract_readnames_from_BAM(aligned_BAM):
+    """
+    Takes a BAM and writes out a txt file containing read names from the BAM
+
+    Parameters:
+        aligned_BAM (str): Path to an input aligned_BAM to extract read names from.
+
+    Returns:
+        None
+
+    """
+    import subprocess
+    # Make a text file of reads for the BAM
+    txt_output = aligned_BAM.split('.bam')[0] + '_read_names.txt'
+    samtools_view = subprocess.Popen(["samtools", "view", aligned_BAM], stdout=subprocess.PIPE)
+    with open(txt_output, "w") as output_file:
+        cut_process = subprocess.Popen(["cut", "-f1"], stdin=samtools_view.stdout, stdout=output_file)   
+    samtools_view.stdout.close()
+    cut_process.wait()
+    samtools_view.wait()

smftools/informatics/helpers/find_conversion_sites.py

@@ -6,34 +6,35 @@ def find_conversion_sites(fasta_file, modification_type, conversion_types):
     If searching for adenine conversions, it will find coordinates of all adenines.
 
     Parameters:
-        fasta_file (str): A string representing the file path to the unconverted reference FASTA.
+        fasta_file (str): A string representing the file path to the converted reference FASTA.
         modification_type (str): A string representing the modification type of interest (options are '5mC' and '6mA').
         conversion_types (list): A list of strings of the conversion types to use in the analysis. Used here to pass the unconverted record name.
 
     Returns: 
-        record_dict (dict): A dictionary keyed by unconverted record ids contained within the FASTA. Points to a list containing: 1) sequence length of the record, 2) top strand coordinate list, 3) bottom strand coorinate list, 4) sequence string
+        record_dict (dict): A dictionary keyed by unconverted record ids contained within the FASTA. Points to a list containing: 1) sequence length of the record, 2) top strand coordinate list, 3) bottom strand coorinate list, 4) sequence string, 5) Complement sequence
     """
     from .. import readwrite
     from Bio import SeqIO
     from Bio.SeqRecord import SeqRecord
     from Bio.Seq import Seq
     
-    print('{0}: Finding positions of interest in reference FASTA > {1}'.format(readwrite.time_string(), fasta_file))
+    #print('{0}: Finding positions of interest in reference FASTA: {1}'.format(readwrite.time_string(), fasta_file))
     # Initialize lists to hold top and bottom strand positional coordinates of interest
     top_strand_coordinates = []
     bottom_strand_coordinates = []
     unconverted = conversion_types[0]
     record_dict = {}
-    print('{0}: Opening FASTA file {1}'.format(readwrite.time_string(), fasta_file))
+    #print('{0}: Opening FASTA file {1}'.format(readwrite.time_string(), fasta_file))
     # Open the FASTA record as read only
     with open(fasta_file, "r") as f:
         # Iterate over records in the FASTA
         for record in SeqIO.parse(f, "fasta"):
             # Only iterate over the unconverted records for the reference
             if unconverted in record.id:
-                print('{0}: Iterating over record {1} in FASTA file {2}'.format(readwrite.time_string(), record, fasta_file))
+                #print('{0}: Iterating over record {1} in FASTA file {2}'.format(readwrite.time_string(), record, fasta_file))
                 # Extract the sequence string of the record
                 sequence = str(record.seq).upper()
+                complement = str(record.seq.complement()).upper()
                 sequence_length = len(sequence)
                 if modification_type == '5mC':
                     # Iterate over the sequence string from the record
@@ -42,7 +43,7 @@ def find_conversion_sites(fasta_file, modification_type, conversion_types):
                             top_strand_coordinates.append(i)  # 0-indexed coordinate
                         if sequence[i] == 'G':
                             bottom_strand_coordinates.append(i)  # 0-indexed coordinate      
-                    print('{0}: Returning zero-indexed top and bottom strand FASTA coordinates for all cytosines'.format(readwrite.time_string()))
+                    #print('{0}: Returning zero-indexed top and bottom strand FASTA coordinates for all cytosines'.format(readwrite.time_string()))
                 elif modification_type == '6mA':
                     # Iterate over the sequence string from the record
                     for i in range(0, len(sequence)):
@@ -50,10 +51,11 @@ def find_conversion_sites(fasta_file, modification_type, conversion_types):
                             top_strand_coordinates.append(i)  # 0-indexed coordinate
                         if sequence[i] == 'T':
                             bottom_strand_coordinates.append(i)  # 0-indexed coordinate      
-                    print('{0}: Returning zero-indexed top and bottom strand FASTA coordinates for adenines of interest'.format(readwrite.time_string()))          
+                    #print('{0}: Returning zero-indexed top and bottom strand FASTA coordinates for adenines of interest'.format(readwrite.time_string()))          
                 else:
-                    print('modification_type not found. Please try 5mC or 6mA')    
-                record_dict[record.id] = [sequence_length, top_strand_coordinates, bottom_strand_coordinates, sequence]
+                    #print('modification_type not found. Please try 5mC or 6mA') 
+                    pass   
+                record_dict[record.id] = [sequence_length, top_strand_coordinates, bottom_strand_coordinates, sequence, complement]
             else:
                 pass  
     return record_dict