smftools 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

smftools/preprocessing/calculate_converted_read_methylation_stats.py

@@ -3,13 +3,17 @@
 ## Conversion SMF Specific 
 # Read methylation QC
 
-def calculate_converted_read_methylation_stats(adata, obs_column='Reference'):
+def calculate_converted_read_methylation_stats(adata, reference_column, sample_names_col, output_directory, show_methylation_histogram=False, save_methylation_histogram=False):
     """
-    Adds methylation statistics for each read. Indicates whether the read GpC methylation exceeded other_C methylation (background false positives)
+    Adds methylation statistics for each read. Indicates whether the read GpC methylation exceeded other_C methylation (background false positives).
 
     Parameters:
-        adata (AnnData): An AnnData object
-        obs_column (str): observation category of interest
+        adata (AnnData): An adata object
+        reference_column (str): String representing the name of the Reference column to use
+        sample_names_col (str): String representing the name of the sample name column to use
+        output_directory (str): String representing the output directory to make and write out the histograms.
+        show_methylation_histogram (bool): Whether to display the histograms.
+        save_methylation_histogram (bool): Whether to save the histograms.
 
     Returns:
         None 
@@ -17,16 +21,21 @@ def calculate_converted_read_methylation_stats(adata, obs_column='Reference'):
     import numpy as np
     import anndata as ad
     import pandas as pd
+    import matplotlib.pyplot as plt
+    from .. import readwrite
+
+    references = set(adata.obs[reference_column])
+    sample_names = set(adata.obs[sample_names_col])
+
+    site_types = ['GpC_site', 'CpG_site', 'ambiguous_GpC_CpG_site', 'other_C']
 
-    site_types = ['GpC_site', 'CpG_site', 'ambiguous_GpC_site', 'ambiguous_CpG_site', 'other_C']
-    categories = adata.obs[obs_column].cat.categories
     for site_type in site_types:
         adata.obs[f'{site_type}_row_methylation_sums'] = pd.Series(0, index=adata.obs_names, dtype=int)
         adata.obs[f'{site_type}_row_methylation_means'] = pd.Series(np.nan, index=adata.obs_names, dtype=float)
         adata.obs[f'number_valid_{site_type}_in_read'] = pd.Series(0, index=adata.obs_names, dtype=int)
         adata.obs[f'fraction_valid_{site_type}_in_range'] = pd.Series(np.nan, index=adata.obs_names, dtype=float)
-    for cat in categories:
-        cat_subset = adata[adata.obs[obs_column] == cat].copy()
+    for cat in references:
+        cat_subset = adata[adata.obs[reference_column] == cat].copy()
         for site_type in site_types:
             print(f'Iterating over {cat}_{site_type}')
             observation_matrix = cat_subset.obsm[f'{cat}_{site_type}']
@@ -42,4 +51,46 @@ def calculate_converted_read_methylation_stats(adata, obs_column='Reference'):
             adata.obs.update(temp_obs_data)
     # Indicate whether the read-level GpC methylation rate exceeds the false methylation rate of the read
     pass_array = np.array(adata.obs[f'GpC_site_row_methylation_means'] > adata.obs[f'other_C_row_methylation_means'])
-    adata.obs['GpC_above_other_C'] = pd.Series(pass_array, index=adata.obs.index, dtype=bool)
+    adata.obs['GpC_above_other_C'] = pd.Series(pass_array, index=adata.obs.index, dtype=bool)
+
+    adata.uns['methylation_dict'] = {}
+    n_bins = 50
+    site_types_to_analyze = ['GpC_site', 'CpG_site', 'ambiguous_GpC_CpG_site', 'other_C']
+
+    for reference in references:
+        reference_adata = adata[adata.obs[reference_column] == reference].copy()
+        split_reference = reference.split('_')[0][1:]
+        for sample in sample_names:
+            sample_adata = reference_adata[reference_adata.obs[sample_names_col] == sample].copy()
+            for site_type in site_types_to_analyze:
+                methylation_data = sample_adata.obs[f'{site_type}_row_methylation_means']
+                max_meth = np.max(sample_adata.obs[f'{site_type}_row_methylation_sums'])
+                if not np.isnan(max_meth):
+                    n_bins = int(max_meth // 2)
+                else:
+                    n_bins = 1
+                mean = np.mean(methylation_data)
+                median = np.median(methylation_data)
+                stdev = np.std(methylation_data)
+                adata.uns['methylation_dict'][f'{reference}_{sample}_{site_type}'] = [mean, median, stdev]
+                if show_methylation_histogram or save_methylation_histogram:
+                    fig, ax = plt.subplots(figsize=(6, 4))
+                    count, bins, patches =  plt.hist(methylation_data, bins=n_bins, weights=np.ones(len(methylation_data)) / len(methylation_data), alpha=0.7, color='blue', edgecolor='black')
+                    plt.axvline(median, color='red', linestyle='dashed', linewidth=1)
+                    plt.text(median + stdev, max(count)*0.8, f'Median: {median:.2f}', color='red')
+                    plt.axvline(median - stdev, color='green', linestyle='dashed', linewidth=1, label=f'Stdev: {stdev:.2f}')
+                    plt.axvline(median + stdev, color='green', linestyle='dashed', linewidth=1)
+                    plt.text(median + stdev + 0.05, max(count) / 3, f'+1 Stdev: {stdev:.2f}', color='green')
+                    plt.xlabel('Fraction methylated')
+                    plt.ylabel('Proportion')
+                    title = f'Distribution of {methylation_data.shape[0]} read {site_type} methylation means \nfor {sample} sample on {split_reference} after filtering'
+                    plt.title(title, pad=20)
+                    plt.xlim(-0.05, 1.05)  # Set x-axis range from 0 to 1
+                    ax.spines['right'].set_visible(False)
+                    ax.spines['top'].set_visible(False)
+                    save_name = output_directory + f'/{readwrite.date_string()} {title}'
+                    if save_methylation_histogram:
+                        plt.savefig(save_name, bbox_inches='tight', pad_inches=0.1)
+                        plt.close()
+                    else:
+                        plt.show()

smftools/preprocessing/calculate_coverage.py

@@ -7,7 +7,7 @@ def calculate_coverage(adata, obs_column='Reference', position_nan_threshold=0.0
     Parameters:
         adata (AnnData): An AnnData object
         obs_column (str): Observation column value to subset on prior to calculating position statistics for that category.
-        position_nan_threshold (float): A minimal threshold of coverage to call the position as valid.
+        position_nan_threshold (float): A minimal fractional threshold of coverage within the obs_column category to call the position as valid.
 
     Returns:
         None
@@ -21,7 +21,7 @@ def calculate_coverage(adata, obs_column='Reference', position_nan_threshold=0.0
     # Loop over categories
     for cat in categories:
         # Look at positional information for each reference
-        temp_cat_adata = adata[adata.obs[obs_column] == cat]
+        temp_cat_adata = adata[adata.obs[obs_column] == cat].copy()
         # Look at read coverage on the given category strand
         cat_valid_coverage = np.sum(~np.isnan(temp_cat_adata.X), axis=0)
         cat_invalid_coverage = np.sum(np.isnan(temp_cat_adata.X), axis=0)

smftools/preprocessing/calculate_pairwise_hamming_distances.py

@@ -13,7 +13,7 @@ def calculate_pairwise_hamming_distances(arrays):
 
     """
     import numpy as np
-    import tqdm
+    from tqdm import tqdm
     from scipy.spatial.distance import hamming
     num_arrays = len(arrays)
     # Initialize an empty distance matrix

smftools/preprocessing/calculate_read_length_stats.py

@@ -1,12 +1,17 @@
 ## calculate_read_length_stats
 
 # Read length QC
-def calculate_read_length_stats(adata):
+def calculate_read_length_stats(adata, reference_column, sample_names_col, output_directory, show_read_length_histogram=False, save_read_length_histogram=False):
     """
     Append first valid position in a read and last valid position in the read. From this determine and append the read length. 
 
     Parameters:
         adata (AnnData): An adata object
+        reference_column (str): String representing the name of the Reference column to use
+        sample_names_col (str): String representing the name of the sample name column to use
+        output_directory (str): String representing the output directory to make and write out the histograms.
+        show_read_length_histogram (bool): Whether to display the histograms.
+        save_read_length_histogram (bool): Whether to save the histograms.
     
     Returns:
         upper_bound (int): last valid position in the dataset
@@ -15,8 +20,17 @@ def calculate_read_length_stats(adata):
     import numpy as np
     import anndata as ad
     import pandas as pd
-    ## Add basic observation-level (read-level) metadata to the object: first valid position in a read and last valid position in the read. From this determine the read length. Save two new variable which hold the first and last valid positions in the entire dataset
+    import matplotlib.pyplot as plt
+    from .. import readwrite
+    from .make_dirs import make_dirs
+
+    make_dirs([output_directory])
 
+    references = set(adata.obs[reference_column])
+    sample_names = set(adata.obs[sample_names_col])
+
+    ## Add basic observation-level (read-level) metadata to the object: first valid position in a read and last valid position in the read. From this determine the read length. Save two new variable which hold the first and last valid positions in the entire dataset
+    print('calculating read length stats')
     # Add some basic observation-level (read-level) metadata to the anndata object
     read_first_valid_position = np.array([int(adata.var_names[i]) for i in np.argmax(~np.isnan(adata.X), axis=1)])
     read_last_valid_position = np.array([int(adata.var_names[i]) for i in (adata.X.shape[1] - 1 - np.argmax(~np.isnan(adata.X[:, ::-1]), axis=1))])
@@ -29,4 +43,44 @@ def calculate_read_length_stats(adata):
     # Define variables to hold the first and last valid position in the dataset
     upper_bound = int(np.nanmax(adata.obs['last_valid_position']))
     lower_bound = int(np.nanmin(adata.obs['first_valid_position']))
+
+# Add an unstructured element to the anndata object which points to a dictionary of read lengths keyed by reference and sample name. Points to a tuple containing (mean, median, stdev) of the read lengths of the sample for the given reference strand
+
+    ## Plot histogram of read length data and save the median and stdev of the read lengths for each sample.
+    adata.uns['read_length_dict'] = {}
+
+    for reference in references:
+        temp_reference_adata = adata[adata.obs[reference_column] == reference].copy()
+        split_reference = reference.split('_')[0][1:]
+        for sample in sample_names:
+            temp_sample_adata = temp_reference_adata[temp_reference_adata.obs[sample_names_col] == sample].copy()
+            temp_data = temp_sample_adata.obs['read_length']
+            max_length = np.max(temp_data)
+            mean = np.mean(temp_data)
+            median = np.median(temp_data)
+            stdev = np.std(temp_data)
+            adata.uns['read_length_dict'][f'{reference}_{sample}'] = [mean, median, stdev]
+            if not np.isnan(max_length):
+                n_bins = int(max_length // 100)
+            else:
+                n_bins = 1
+            if show_read_length_histogram or save_read_length_histogram:
+                plt.figure(figsize=(10, 6))
+                plt.text(median + 0.5, max(plt.hist(temp_data, bins=n_bins)[0]) / 2, f'Median: {median:.2f}', color='red')
+                plt.hist(temp_data, bins=n_bins, alpha=0.7, color='blue', edgecolor='black')
+                plt.xlabel('Read Length')
+                plt.ylabel('Count')
+                title = f'Read length distribution of {temp_sample_adata.shape[0]} total reads from {sample} sample on {split_reference} allele'
+                plt.title(title)
+                # Add a vertical line at the median
+                plt.axvline(median, color='red', linestyle='dashed', linewidth=1)
+                # Annotate the median
+                plt.xlim(lower_bound - 100, upper_bound + 100) 
+                if save_read_length_histogram:
+                    save_name = output_directory + f'/{readwrite.date_string()} {title}'
+                    plt.savefig(save_name, bbox_inches='tight', pad_inches=0.1)
+                    plt.close()
+                else:
+                    plt.show()
+
     return upper_bound, lower_bound

smftools/preprocessing/clean_NaN.py

@@ -1,7 +1,5 @@
 ## clean_NaN
-from ..readwrite import adata_to_df
 
-# NaN handling
 def clean_NaN(adata, layer=None):
     """
     Append layers to adata that contain NaN cleaning strategies.
@@ -16,6 +14,8 @@ def clean_NaN(adata, layer=None):
     import numpy as np
     import anndata as ad
     import pandas as pd
+    from ..readwrite import adata_to_df
+
     # Fill NaN with closest SMF value
     df = adata_to_df(adata, layer=layer)
     df = df.ffill(axis=1).bfill(axis=1)

smftools/preprocessing/filter_converted_reads_on_methylation.py

@@ -11,7 +11,7 @@ def filter_converted_reads_on_methylation(adata, valid_SMF_site_threshold=0.8, m
         valid_SMF_site_threshold (float): A minimum proportion of valid SMF sites that must be present in the read. Default is 0.8
         min_SMF_threshold (float): A minimum read methylation level. Default is 0.025
     Returns:
-        None
+        adata (AnnData): The filtered adata object.
     """
     import numpy as np
     import anndata as ad
@@ -24,4 +24,6 @@ def filter_converted_reads_on_methylation(adata, valid_SMF_site_threshold=0.8, m
         # Keep reads with SMF methylation over background methylation.
         adata = adata[adata.obs['GpC_above_other_C'] == True].copy()
         # Keep reads over a defined methylation threshold
-        adata = adata[adata.obs['GpC_site_row_methylation_means'] > min_SMF_threshold].copy()
+        adata = adata[adata.obs['GpC_site_row_methylation_means'] > min_SMF_threshold].copy()
+        
+    return adata

smftools/preprocessing/filter_reads_on_length.py

@@ -10,7 +10,7 @@ def filter_reads_on_length(adata, filter_on_coordinates=False, min_read_length=2
         min_read_length (int): The minimum read length to keep in the filtered dataset. Default is 2700.
 
     Returns:
-        None
+        adata (AnnData): The filtered adata object
     Input: Adata object. a list of lower and upper bound (set to False or None if not wanted), and a minimum read length integer.
  
     """
@@ -36,4 +36,6 @@ def filter_reads_on_length(adata, filter_on_coordinates=False, min_read_length=2
 
     if min_read_length:
         print(f'Subsetting adata to keep reads longer than {min_read_length}')
-        adata = adata[adata.obs['read_length'] > min_read_length].copy()
+        adata = adata[adata.obs['read_length'] > min_read_length].copy()
+
+    return adata

smftools/preprocessing/invert_adata.py

@@ -13,6 +13,7 @@ def invert_adata(adata):
     """
     import numpy as np
     import anndata as ad
+    print('Inverting adata')
     # Reassign var_names with new names
     old_var_names = adata.var_names.astype(int).to_numpy()
     new_var_names = np.sort(old_var_names)[::-1].astype(str)

smftools/preprocessing/load_sample_sheet.py

@@ -0,0 +1,24 @@
+# load_sample_sheet
+
+def load_sample_sheet(adata, sample_sheet_path, mapping_key_column):
+    """
+    Loads a sample sheet csv and uses one of the columns to map sample information into the AnnData object.
+
+    Parameters:
+        adata (AnnData): The Anndata object to append sample information to.
+        sample_sheet_path (str):
+        mapping_key_column (str):
+
+    Returns:
+        None
+    """
+    import pandas as pd
+    import anndata as ad
+    df = pd.read_csv(sample_sheet_path)
+    key_column = mapping_key_column
+    df[key_column] = df[key_column].astype(str)
+    value_columns = [column for column in df.columns if column != key_column]
+    mapping_dict = df.set_index(key_column)[value_columns].to_dict(orient='index')
+    for column in value_columns:
+        column_map = {key: value[column] for key, value in mapping_dict.items()}
+        adata.obs[column] = adata.obs[key_column].map(column_map)

smftools/preprocessing/make_dirs.py

@@ -0,0 +1,21 @@
+## make_dirs
+
+# General
+def make_dirs(directories):
+    """
+    Takes a list of file paths and makes new directories if the directory does not already exist.
+
+    Parameters:
+        directories (list): A list of directories to make
+    
+    Returns:
+        None
+    """
+    import os
+
+    for directory in directories:
+        if not os.path.isdir(directory):
+            os.mkdir(directory)
+            print(f"Directory '{directory}' created successfully.")
+        else:
+            print(f"Directory '{directory}' already exists.")

smftools/preprocessing/mark_duplicates.py

@@ -1,6 +1,6 @@
 ## mark_duplicates
 
-def mark_duplicates(adata, layers, obs_column='Reference', sample_col='Sample_names'):
+def mark_duplicates(adata, layers, obs_column='Reference', sample_col='Sample_names', hamming_distance_thresholds={}):
     """
     Marks duplicates in the adata object.
 
@@ -9,6 +9,7 @@ def mark_duplicates(adata, layers, obs_column='Reference', sample_col='Sample_na
         layers (list): A list of strings representing the layers to use.
         obs_column (str): A string representing the obs column name to first subset on. Default is 'Reference'.
         sample_col (str):L A string representing the obs column name to second subset on. Default is 'Sample_names'.
+        hamming_distance_thresholds (dict): A dictionary keyed by obs_column categories that points to a float corresponding to the distance threshold to apply. Default is an empty dict.
     
     Returns:
         None
@@ -48,22 +49,32 @@ def mark_duplicates(adata, layers, obs_column='Reference', sample_col='Sample_na
             distance_df = pd.DataFrame(distance_matrix, index=read_names, columns=read_names)
             # Save the distance dataframe into an unstructured component of the adata object
             adata.uns[f'Pairwise_Hamming_distance_within_{cat}_{sample}'] = distance_df
-            # Calculate the minimum non-self distance for every read in the reference and sample
-            min_distance_values = min_non_diagonal(distance_matrix)
-            min_distance_df = pd.DataFrame({'Nearest_neighbor_Hamming_distance': min_distance_values}, index=read_names)
-            adata.obs.update(min_distance_df)
-            # Generate a histogram of minimum non-self distances for each read
-            min_distance_bins = plt.hist(min_distance_values, bins=n_reads//4)
-            # Normalize the max value in any histogram bin to 1
-            normalized_min_distance_counts = min_distance_bins[0] / np.max(min_distance_bins[0])
-            # Extract the bin index of peak centers in the histogram
-            peak_centers, _ = find_peaks(normalized_min_distance_counts, prominence=0.2, distance=5)
-            first_peak_index = peak_centers[0]
-            offset_index = first_peak_index-1
-            # Use the distance corresponding to the first peak as the threshold distance in graph construction
-            first_peak_distance = min_distance_bins[1][first_peak_index]
-            offset_distance = min_distance_bins[1][offset_index]
-            adata.uns[f'Hamming_distance_threshold_for_{cat}_{sample}'] = offset_distance
+            if n_reads > 1:
+                # Calculate the minimum non-self distance for every read in the reference and sample
+                min_distance_values = min_non_diagonal(distance_matrix)
+                min_distance_df = pd.DataFrame({'Nearest_neighbor_Hamming_distance': min_distance_values}, index=read_names)
+                adata.obs.update(min_distance_df)
+                # Generate a histogram of minimum non-self distances for each read
+                if n_reads > 3:
+                    n_bins = n_reads // 4
+                else:
+                    n_bins = 1
+                min_distance_bins = plt.hist(min_distance_values, bins=n_bins)
+                if cat in hamming_distance_thresholds: 
+                    adata.uns[f'Hamming_distance_threshold_for_{cat}_{sample}'] = hamming_distance_thresholds[cat]
+                else: # eventually this should be written to use known PCR duplicate controls for thresholding.
+                    # Normalize the max value in any histogram bin to 1
+                    normalized_min_distance_counts = min_distance_bins[0] / np.max(min_distance_bins[0])
+                    # Extract the bin index of peak centers in the histogram
+                    peak_centers, _ = find_peaks(normalized_min_distance_counts, prominence=0.2, distance=5)
+                    first_peak_index = peak_centers[0]
+                    offset_index = first_peak_index-1
+                    # Use the distance corresponding to the first peak as the threshold distance in graph construction
+                    first_peak_distance = min_distance_bins[1][first_peak_index]
+                    offset_distance = min_distance_bins[1][offset_index]
+                    adata.uns[f'Hamming_distance_threshold_for_{cat}_{sample}'] = offset_distance
+            else:
+                adata.uns[f'Hamming_distance_threshold_for_{cat}_{sample}'] = 0
 
     ## Detect likely duplicate reads and mark them in the adata object.
     adata.obs['Marked_duplicate'] = pd.Series(False, index=adata.obs_names, dtype=bool)
@@ -91,7 +102,11 @@ def mark_duplicates(adata, layers, obs_column='Reference', sample_col='Sample_na
             clusters = [list(cluster) for cluster in clusters]
             # Get the number of clusters
             cluster_count = len(clusters)
-            adata.uns[f'Hamming_distance_clusters_within_{cat}_{sample}'] = [cluster_count, n_reads, cluster_count / n_reads, clusters]
+            if n_reads > 0:
+                fraction_unique = cluster_count / n_reads
+            else:
+                fraction_unique = 0
+            adata.uns[f'Hamming_distance_clusters_within_{cat}_{sample}'] = [cluster_count, n_reads, fraction_unique, clusters]
             # Update the adata object
             read_cluster_map = {}
             read_duplicate_map = {}
@@ -116,4 +131,4 @@ def mark_duplicates(adata, layers, obs_column='Reference', sample_col='Sample_na
             adata.obs.update(df_combined)
             adata.obs['Marked_duplicate'] = adata.obs['Marked_duplicate'].astype(bool)
             adata.obs['Unique_in_final_read_set'] = adata.obs['Unique_in_final_read_set'].astype(bool)
-            print(f'Hamming clusters for {sample} on {cat}\nThreshold: {first_peak_distance}\nNumber clusters: {cluster_count}\nNumber reads: {n_reads}\nFraction unique: {cluster_count / n_reads}')   
+            print(f'Hamming clusters for {sample} on {cat}\nThreshold: {distance_threshold}\nNumber clusters: {cluster_count}\nNumber reads: {n_reads}\nFraction unique: {fraction_unique}')   

smftools/preprocessing/recipes.py

@@ -0,0 +1,125 @@
+# recipes
+
+def recipe_1_Kissiov_and_McKenna_2025(adata, sample_sheet_path, output_directory, mapping_key_column='Sample', reference_column = 'Reference', sample_names_col='Sample_names', invert=False):
+    """
+    The first part of the preprocessing workflow applied to the smf.inform.pod_to_adata() output derived from Kissiov_and_McKenna_2025.
+
+    Performs the following tasks:
+    1) Loads a sample CSV to append metadata mappings to the adata object.
+    2) Appends a boolean indicating whether each position in var_names is within a given reference.
+    3) Appends the cytosine context to each position from each reference.
+    4) Calculate read level methylation statistics.
+    5) Optionally inverts the adata to flip the position coordinate orientation.
+    6) Calculates read length statistics (start position, end position, read length)
+    7) Returns a dictionary to pass the variable namespace to the parent scope.
+
+    Parameters:
+        adata (AnnData): The AnnData object to use as input.
+        sample_sheet_path (str): String representing the path to the sample sheet csv containing the sample metadata.
+        output_directory (str): String representing the path to the output directory for plots.
+        mapping_key_column (str): The column name to use as the mapping keys for applying the sample sheet metadata.
+        reference_column (str): The name of the reference column to use.
+        sample_names_col (str): The name of the sample name column to use.
+        invert (bool): Whether to invert the positional coordinates of the adata object.
+
+    Returns:
+        variables (dict): A dictionary of variables to append to the parent scope.
+    """
+    import anndata as ad
+    import pandas as pd
+    import numpy as np
+    from .load_sample_sheet import load_sample_sheet
+    from .calculate_coverage import calculate_coverage
+    from .append_C_context import append_C_context
+    from .calculate_converted_read_methylation_stats import calculate_converted_read_methylation_stats
+    from .invert_adata import invert_adata
+    from .calculate_read_length_stats import calculate_read_length_stats
+
+    # Clean up some of the Reference metadata and save variable names that point to sets of values in the column.
+    adata.obs[reference_column] = adata.obs[reference_column].astype('category')
+    references = adata.obs[reference_column].cat.categories
+    split_references = [(reference, reference.split('_')[0][1:]) for reference in references]
+    reference_mapping = {k: v for k, v in split_references}
+    adata.obs[f'{reference_column}_short'] = adata.obs[reference_column].map(reference_mapping)
+    short_references = set(adata.obs[f'{reference_column}_short'])
+    binary_layers = adata.layers.keys()
+
+    # load sample sheet metadata
+    load_sample_sheet(adata, sample_sheet_path, mapping_key_column)
+
+    # hold sample names set
+    adata.obs[sample_names_col] = adata.obs[sample_names_col].astype('category')
+    sample_names = adata.obs[sample_names_col].cat.categories
+
+    # Add position level metadata
+    calculate_coverage(adata, obs_column=reference_column)
+    adata.var['SNP_position'] = (adata.var[f'N_{reference_column}_with_position'] > 0) & (adata.var[f'N_{reference_column}_with_position'] < len(references)).astype(bool)
+
+    # Append cytosine context to the reference positions based on the conversion strand.
+    append_C_context(adata, obs_column=reference_column, use_consensus=False)
+
+    # Calculate read level methylation statistics. Assess if GpC methylation level is above other_C methylation level as a QC.
+    calculate_converted_read_methylation_stats(adata, reference_column, sample_names_col, output_directory, show_methylation_histogram=False, save_methylation_histogram=False)
+
+    # Invert the adata object (ie flip the strand orientation for visualization)
+    if invert:
+        invert_adata(adata)
+    else:
+        pass
+
+    # Calculate read length statistics, with options to display or save the read length histograms
+    upper_bound, lower_bound = calculate_read_length_stats(adata, reference_column, sample_names_col, output_directory, show_read_length_histogram=False, save_read_length_histogram=False)
+
+    variables = {
+        "short_references": short_references,
+        "binary_layers": binary_layers,
+        "sample_names": sample_names,
+        "upper_bound": upper_bound,
+        "lower_bound": lower_bound,
+        "references": references
+    }
+    return variables
+
+def recipe_2_Kissiov_and_McKenna_2025(adata, output_directory, binary_layers, hamming_distance_thresholds={}, reference_column = 'Reference', sample_names_col='Sample_names'):
+    """
+    The second part of the preprocessing workflow applied to the adata that has already been preprocessed by recipe_1_Kissiov_and_McKenna_2025.
+
+    Performs the following tasks:
+    1) Adds new layers containing NaN replaced variants of adata.X (fill_closest, nan0_0minus1, nan1_12).
+    2) Marks putative PCR duplicates using pairwise hamming distance metrics.
+    3) Performs a complexity analysis of the library based on the PCR duplicate detection rate.
+    4) Removes PCR duplicates from the adata.
+    5) Returns two adata object: one for the filtered adata and one for the duplicate adata.
+
+    Parameters:
+        adata (AnnData): The AnnData object to use as input.
+        output_directory (str): String representing the path to the output directory for plots.
+        binary_layers (list): A list of layers to used for the binary encoding of read sequences. Used for duplicate detection.
+        hamming_distance_thresholds (dict): A dictionary keyed by obs_column categories that points to a float corresponding to the distance threshold to apply. Default is an empty dict.
+        reference_column (str): The name of the reference column to use.
+        sample_names_col (str): The name of the sample name column to use.
+
+    Returns:
+        filtered_adata (AnnData): An AnnData object containing the filtered reads
+        duplicates (AnnData): An AnnData object containing the duplicate reads
+    """
+    import anndata as ad
+    import pandas as pd
+    import numpy as np
+    from .clean_NaN import clean_NaN
+    from .mark_duplicates import mark_duplicates
+    from .calculate_complexity import calculate_complexity
+    from .remove_duplicates import remove_duplicates
+
+    # NaN replacement strategies stored in additional layers. Having layer=None uses adata.X
+    clean_NaN(adata, layer=None)
+
+    # Duplicate detection using pairwise hamming distance across reads
+    mark_duplicates(adata, binary_layers, obs_column=reference_column, sample_col=sample_names_col, hamming_distance_thresholds=hamming_distance_thresholds)
+
+    # Complexity analysis using the marked duplicates and the lander-watermann algorithm
+    calculate_complexity(adata, output_directory, obs_column=reference_column, sample_col=sample_names_col, plot=True, save_plot=False)
+
+    # Remove duplicate reads and store the duplicate reads in a new AnnData object named duplicates.
+    filtered_adata, duplicates = remove_duplicates(adata)
+    return filtered_adata, duplicates

smftools/preprocessing/remove_duplicates.py

@@ -8,11 +8,14 @@ def remove_duplicates(adata):
         adata (Anndata): An adata object.
 
     Returns:
-        None 
+        filtered_adata (AnnData): An AnnData object of the filtered reads 
+        duplicates (AnnData): An AnnData object of the duplicate reads 
     """
     import anndata as ad
 
     initial_size = adata.shape[0]
-    adata = adata[adata.obs['Unique_in_final_read_set'] == True].copy()
-    final_size = adata.shape[0]
-    print(f'Removed {initial_size-final_size} reads from the dataset')
+    filtered_adata = adata[adata.obs['Unique_in_final_read_set'] == True].copy()
+    final_size = filtered_adata.shape[0]
+    print(f'Removed {initial_size-final_size} reads from the dataset')
+    duplicates = adata[adata.obs['Unique_in_final_read_set'] == False].copy()
+    return filtered_adata, duplicates

smftools/tools/apply_HMM.py

@@ -0,0 +1 @@
+# apply_HMM

smftools/tools/read_HMM.py

@@ -0,0 +1 @@
+# read_HMM

smftools/tools/subset_adata.py

@@ -0,0 +1,32 @@
+# subset_adata
+
+def subset_adata(adata, obs_columns):
+    """
+    Subsets an AnnData object based on categorical values in specified `.obs` columns.
+    
+    Parameters:
+        adata (AnnData): The AnnData object to subset.
+        obs_columns (list of str): List of `.obs` column names to subset by. The order matters.
+
+    Returns:
+        dict: A dictionary where keys are tuples of category values and values are corresponding AnnData subsets.
+    """
+
+    def subset_recursive(adata_subset, columns):
+        if not columns:
+            return {(): adata_subset}
+        
+        current_column = columns[0]
+        categories = adata_subset.obs[current_column].cat.categories
+        
+        subsets = {}
+        for cat in categories:
+            subset = adata_subset[adata_subset.obs[current_column] == cat]
+            subsets.update(subset_recursive(subset, columns[1:]))
+        
+        return subsets
+    
+    # Start the recursive subset process
+    subsets_dict = subset_recursive(adata, obs_columns)
+    
+    return subsets_dict

smftools/tools/train_HMM.py

@@ -0,0 +1,43 @@
+# train_HMM
+
+def train_HMM(adata, model_name='trained_HMM', save_hmm=False):
+    """
+
+    Parameters:
+        adata (AnnData): Input AnnData object
+        model_name (str): Name of the model
+        save_hmm (bool): Whether to save the model
+    
+    """
+    import numpy as np
+    import anndata as ad
+    from pomegranate.distributions import Categorical
+    from pomegranate.hmm import DenseHMM
+
+    bound = Categorical([[0.95, 0.05]])
+    unbound = Categorical([[0.05, 0.95]])
+
+    edges = [[0.9, 0.1], [0.1, 0.9]]
+    starts = [0.5, 0.5]
+    ends = [0.5, 0.5]
+
+    model = DenseHMM([bound, unbound], edges=edges, starts=starts, ends=ends, max_iter=5, verbose=True)
+
+    # define training sets and labels
+    # Determine the number of reads to sample
+    n_sample = round(0.7 * adata.X.shape[0])
+    # Generate random indices
+    np.random.seed(0)
+    random_indices = np.random.choice(adata.shape[0], size=n_sample, replace=False)
+    # Subset the AnnData object using the random indices
+    training_adata_subsampled = adata[random_indices, :]
+    training_sequences = training_adata_subsampled.X
+
+    # Train the HMM without labeled data
+    model.fit(training_sequences, algorithm='baum-welch')
+
+    if save_hmm:
+        # Save the model to a file
+        model_json = model.to_json()
+        with open(f'{model_name}.json', 'w') as f:
+                f.write(model_json)