smftools 0.1.1__py3-none-any.whl → 0.1.6__py3-none-any.whl

smftools/preprocessing/binary_layers_to_ohe.py

@@ -1,30 +0,0 @@
-## binary_layers_to_ohe
-
-## Conversion SMF Specific 
-def binary_layers_to_ohe(adata, layers, stack='hstack'):
-    """
-    Parameters:
-        adata (AnnData): Anndata object.
-        layers (list): a list of strings. Each string represents a layer in the adata object. The layer should encode a binary matrix
-        stack (str): Dimension to stack the one-hot-encoding. Options include 'hstack' and 'vstack'. Default is 'hstack', since this is more efficient.
-    
-    Returns:
-        ohe_dict (dict): A dictionary keyed by obs_name that points to a stacked (hstack or vstack) one-hot encoding of the binary layers
-    Input: An adata object and a list of layers containing a binary encoding.
-    """
-    import numpy as np
-    import anndata as ad
-    # Extract the layers
-    layers = [adata.layers[layer_name] for layer_name in layers]
-    n_reads = layers[0].shape[0]
-    ohe_dict = {}
-    for i in range(n_reads):
-        read_ohe = []
-        for layer in layers:
-            read_ohe.append(layer[i])
-        read_name = adata.obs_names[i]
-        if stack == 'hstack':
-            ohe_dict[read_name] = np.hstack(read_ohe)
-        elif stack == 'vstack':
-            ohe_dict[read_name] = np.vstack(read_ohe)
-    return ohe_dict

smftools/preprocessing/calculate_complexity.py

@@ -1,71 +0,0 @@
-## calculate_complexity
-
-def calculate_complexity(adata, obs_column='Reference', sample_col='Sample_names', plot=True, save_plot=False, output_directory=''):
-    """
-    A complexity analysis of the library.
-
-    Parameters:
-        adata (AnnData): An adata object with mark_duplicates already run.
-        obs_column (str): String of the obs column to iterate over.
-        sample_col (str): String of the sample column to iterate over.
-        plot (bool): Whether to plot the complexity model.
-        save_plot (bool): Whether to save the complexity model.
-        output_directory (str): String representing the path to the output directory.
-
-    Returns:
-        None
-
-    """
-    import numpy as np
-    import pandas as pd
-    from scipy.optimize import curve_fit
-
-    def lander_waterman(x, C0):
-        return C0 * (1 - np.exp(-x / C0))
-
-    def count_unique_reads(reads, depth):
-        subsample = np.random.choice(reads, depth, replace=False)
-        return len(np.unique(subsample))
-
-    categories = adata.obs[obs_column].cat.categories 
-    sample_names = adata.obs[sample_col].cat.categories 
-
-    for cat in categories:
-        for sample in sample_names:
-            unique_reads, total_reads = adata.uns[f'Hamming_distance_clusters_within_{cat}_{sample}'][0:2]
-            reads = np.concatenate((np.arange(unique_reads), np.random.choice(unique_reads, total_reads - unique_reads, replace=True)))
-            # Subsampling depths
-            subsampling_depths = [total_reads // (i+1) for i in range(10)]
-            # Arrays to store results
-            subsampled_total_reads = []
-            subsampled_unique_reads = []
-            # Perform subsampling
-            for depth in subsampling_depths:
-                unique_count = count_unique_reads(reads, depth)
-                subsampled_total_reads.append(depth)
-                subsampled_unique_reads.append(unique_count)
-            # Fit the Lander-Waterman model to the data
-            popt, _ = curve_fit(lander_waterman, subsampled_total_reads, subsampled_unique_reads)
-            # Generate data for the complexity curve
-            x_data = np.linspace(0, 5000, 100)
-            y_data = lander_waterman(x_data, *popt)
-            adata.uns[f'Library_complexity_{sample}_on_{cat}'] = popt[0]
-            if plot:
-                import matplotlib.pyplot as plt
-                # Plot the complexity curve
-                plt.figure(figsize=(6, 4))
-                plt.plot(total_reads, unique_reads, 'o', label='Observed unique reads')
-                plt.plot(x_data, y_data, '-', label=f'Lander-Waterman fit\nEstimated C0 = {popt[0]:.2f}')
-                plt.xlabel('Total number of reads')
-                plt.ylabel('Number of unique reads')
-                title = f'Library Complexity Analysis for {sample} on {cat}'
-                plt.title(title)
-                plt.legend()
-                plt.grid(True)
-                if save_plot:
-                    date_string = date_string()
-                    save_name = output_directory + f'/{date_string}_{title}'
-                    plt.savefig(save_name, bbox_inches='tight', pad_inches=0.1)
-                    plt.close()
-                else:
-                    plt.show()

smftools/preprocessing/calculate_converted_read_methylation_stats.py

@@ -1,45 +0,0 @@
-## calculate_converted_read_methylation_stats
-
-## Conversion SMF Specific 
-# Read methylation QC
-
-def calculate_converted_read_methylation_stats(adata, obs_column='Reference'):
-    """
-    Adds methylation statistics for each read. Indicates whether the read GpC methylation exceeded other_C methylation (background false positives)
-
-    Parameters:
-        adata (AnnData): An AnnData object
-        obs_column (str): observation category of interest
-
-    Returns:
-        None 
-    """
-    import numpy as np
-    import anndata as ad
-    import pandas as pd
-
-    site_types = ['GpC_site', 'CpG_site', 'ambiguous_GpC_site', 'ambiguous_CpG_site', 'other_C']
-    categories = adata.obs[obs_column].cat.categories
-    for site_type in site_types:
-        adata.obs[f'{site_type}_row_methylation_sums'] = pd.Series(0, index=adata.obs_names, dtype=int)
-        adata.obs[f'{site_type}_row_methylation_means'] = pd.Series(np.nan, index=adata.obs_names, dtype=float)
-        adata.obs[f'number_valid_{site_type}_in_read'] = pd.Series(0, index=adata.obs_names, dtype=int)
-        adata.obs[f'fraction_valid_{site_type}_in_range'] = pd.Series(np.nan, index=adata.obs_names, dtype=float)
-    for cat in categories:
-        cat_subset = adata[adata.obs[obs_column] == cat].copy()
-        for site_type in site_types:
-            print(f'Iterating over {cat}_{site_type}')
-            observation_matrix = cat_subset.obsm[f'{cat}_{site_type}']
-            number_valid_positions_in_read = np.nansum(~np.isnan(observation_matrix), axis=1)
-            row_methylation_sums = np.nansum(observation_matrix, axis=1)
-            number_valid_positions_in_read[number_valid_positions_in_read == 0] = 1
-            fraction_valid_positions_in_range = number_valid_positions_in_read / np.max(number_valid_positions_in_read)
-            row_methylation_means = np.divide(row_methylation_sums, number_valid_positions_in_read)
-            temp_obs_data = pd.DataFrame({f'number_valid_{site_type}_in_read': number_valid_positions_in_read,
-                                        f'fraction_valid_{site_type}_in_range': fraction_valid_positions_in_range,
-                                        f'{site_type}_row_methylation_sums': row_methylation_sums,
-                                        f'{site_type}_row_methylation_means': row_methylation_means}, index=cat_subset.obs.index)
-            adata.obs.update(temp_obs_data)
-    # Indicate whether the read-level GpC methylation rate exceeds the false methylation rate of the read
-    pass_array = np.array(adata.obs[f'GpC_site_row_methylation_means'] > adata.obs[f'other_C_row_methylation_means'])
-    adata.obs['GpC_above_other_C'] = pd.Series(pass_array, index=adata.obs.index, dtype=bool)

smftools/preprocessing/calculate_coverage.py

@@ -1,41 +0,0 @@
-## calculate_coverage
-
-def calculate_coverage(adata, obs_column='Reference', position_nan_threshold=0.05):
-    """
-    Append position level metadata regarding whether the position is informative within the given observation category.
-
-    Parameters:
-        adata (AnnData): An AnnData object
-        obs_column (str): Observation column value to subset on prior to calculating position statistics for that category.
-        position_nan_threshold (float): A minimal threshold of coverage to call the position as valid.
-
-    Returns:
-        None
-    """
-    import numpy as np
-    import anndata as ad
-    import pandas as pd
-
-    categories = adata.obs[obs_column].cat.categories
-    n_categories_with_position = np.zeros(adata.shape[1])
-    # Loop over categories
-    for cat in categories:
-        # Look at positional information for each reference
-        temp_cat_adata = adata[adata.obs[obs_column] == cat]
-        # Look at read coverage on the given category strand
-        cat_valid_coverage = np.sum(~np.isnan(temp_cat_adata.X), axis=0)
-        cat_invalid_coverage = np.sum(np.isnan(temp_cat_adata.X), axis=0)
-        cat_valid_fraction = cat_valid_coverage / (cat_valid_coverage + cat_invalid_coverage)
-        # Append metadata for category to the anndata object
-        adata.var[f'{cat}_valid_fraction'] = pd.Series(cat_valid_fraction, index=adata.var.index)
-        # Characterize if the position is in the given category or not
-        conditions = [
-            (adata.var[f'{cat}_valid_fraction'] >= position_nan_threshold),
-            (adata.var[f'{cat}_valid_fraction'] < position_nan_threshold)
-        ]
-        choices = [True, False]
-        adata.var[f'position_in_{cat}'] = np.select(conditions, choices, default=False)
-        n_categories_with_position += np.array(adata.var[f'position_in_{cat}'])
-
-    # Final array with the sum at each position of the number of categories covering that position
-    adata.var[f'N_{obs_column}_with_position'] = n_categories_with_position.astype(int)

smftools/preprocessing/calculate_pairwise_hamming_distances.py

@@ -1,27 +0,0 @@
-## calculate_pairwise_hamming_distances
-
-## Conversion SMF Specific 
-def calculate_pairwise_hamming_distances(arrays):
-    """
-    Calculate the pairwise Hamming distances for a list of h-stacked ndarrays.
-
-    Parameters:
-        arrays (str): A list of ndarrays.
-
-    Returns:
-        distance_matrix (ndarray): a 2D array containing the pairwise Hamming distances between all arrays.
-
-    """
-    import numpy as np
-    import tqdm
-    from scipy.spatial.distance import hamming
-    num_arrays = len(arrays)
-    # Initialize an empty distance matrix
-    distance_matrix = np.zeros((num_arrays, num_arrays))
-    # Calculate pairwise distances with progress bar
-    for i in tqdm(range(num_arrays), desc="Calculating Hamming Distances"):
-        for j in range(i + 1, num_arrays):
-            distance = hamming(arrays[i], arrays[j])
-            distance_matrix[i, j] = distance
-            distance_matrix[j, i] = distance
-    return distance_matrix

smftools/preprocessing/calculate_position_Youden.py

@@ -1,104 +0,0 @@
-## calculate_position_Youden
-
-## Calculating and applying position level thresholds for methylation calls to binarize the SMF data
-def calculate_position_Youden(adata, positive_control_sample, negative_control_sample, J_threshold=0.4, obs_column='Reference', save=False, output_directory=''):
-    """
-    Adds new variable metadata to each position indicating whether the position provides reliable SMF methylation calls. Also outputs plots of the positional ROC curves.
-
-    Parameters:
-        adata (AnnData): An AnnData object.
-        positive_control_sample (str): string representing the sample name corresponding to the Plus MTase control sample.
-        negative_control_sample (str): string representing the sample name corresponding to the Minus MTase control sample.
-        J_threshold (float): A float indicating the J-statistic used to indicate whether a position passes QC for methylation calls.
-        obs_column (str): The category to iterate over.
-        save (bool): Whether to save the ROC plots.
-        output_directory (str): String representing the path to the output directory to output the ROC curves.
-    
-    Returns: 
-        None
-    """
-    import numpy as np
-    import pandas as pd
-    import anndata as ad
-    import matplotlib.pyplot as plt
-    from sklearn.metrics import roc_curve, roc_auc_score
-
-    control_samples = [positive_control_sample, negative_control_sample]
-    categories = adata.obs[obs_column].cat.categories 
-    # Iterate over each category in the specified obs_column
-    for cat in categories:
-        # Subset to keep only reads associated with the category
-        cat_subset = adata[adata.obs[obs_column] == cat].copy()
-        # Iterate over positive and negative control samples
-        for control in control_samples:
-            # Initialize a dictionary for the given control sample. This will be keyed by dataset and position to point to a tuple of coordinate position and an array of methylation probabilities
-            adata.uns[f'{cat}_position_methylation_dict_{control}'] = {}
-            # get the current control subset on the given category
-            filtered_obs = cat_subset.obs[cat_subset.obs['Sample_names'].str.contains(control, na=False, regex=True)]
-            control_subset = cat_subset[filtered_obs.index].copy()
-            # Iterate through every position in the control subset
-            for position in range(control_subset.shape[1]):
-                # Get the coordinate name associated with that position
-                coordinate = control_subset.var_names[position]
-                # Get the array of methlyation probabilities for each read in the subset at that position
-                position_data = control_subset.X[:, position]
-                # Get the indexes of everywhere that is not a nan value
-                nan_mask = ~np.isnan(position_data)
-                # Keep only the methlyation data that has real values
-                position_data = position_data[nan_mask]
-                # Get the position data coverage
-                position_coverage = len(position_data)
-                # Get fraction coverage
-                fraction_coverage = position_coverage / control_subset.shape[0]
-                # Save the position and the position methylation data for the control subset
-                adata.uns[f'{cat}_position_methylation_dict_{control}'][f'{position}'] = (position, position_data, fraction_coverage)
-
-    for cat in categories:
-        fig, ax = plt.subplots(figsize=(6, 4))
-        plt.plot([0, 1], [0, 1], linestyle='--', color='gray')
-        plt.xlabel('False Positive Rate')
-        plt.ylabel('True Positive Rate')
-        ax.spines['right'].set_visible(False)
-        ax.spines['top'].set_visible(False)
-        n_passed_positions = 0
-        n_total_positions = 0
-        # Initialize a list that will hold the positional thresholds for the category
-        probability_thresholding_list = [(np.nan, np.nan)] * adata.shape[1]
-        for i, key in enumerate(adata.uns[f'{cat}_position_methylation_dict_{positive_control_sample}'].keys()):
-            position = int(adata.uns[f'{cat}_position_methylation_dict_{positive_control_sample}'][key][0])
-            positive_position_array = adata.uns[f'{cat}_position_methylation_dict_{positive_control_sample}'][key][1]
-            fraction_coverage = adata.uns[f'{cat}_position_methylation_dict_{positive_control_sample}'][key][2]
-            if fraction_coverage > 0.2:
-                try:
-                    negative_position_array = adata.uns[f'{cat}_position_methylation_dict_{negative_control_sample}'][key][1] 
-                    # Combine the negative and positive control data
-                    data = np.concatenate([negative_position_array, positive_position_array])
-                    labels = np.array([0] * len(negative_position_array) + [1] * len(positive_position_array))
-                    # Calculate the ROC curve
-                    fpr, tpr, thresholds = roc_curve(labels, data)
-                    # Calculate Youden's J statistic
-                    J = tpr - fpr
-                    optimal_idx = np.argmax(J)
-                    optimal_threshold = thresholds[optimal_idx]
-                    max_J = np.max(J)
-                    data_tuple = (optimal_threshold, max_J)
-                    probability_thresholding_list[position] = data_tuple
-                    n_total_positions += 1
-                    if max_J > J_threshold:
-                        n_passed_positions += 1
-                        plt.plot(fpr, tpr, label='ROC curve')
-                except:
-                    probability_thresholding_list[position] = (0.8, np.nan)
-        title = f'ROC Curve for {n_passed_positions} positions with J-stat greater than {J_threshold}\n out of {n_total_positions} total positions on {cat}'
-        plt.title(title)
-        date_string = date_string()
-        save_name = output_directory + f'/{date_string} {title}'
-        if save:
-            plt.savefig(save_name)
-            plt.close()
-        else:
-            plt.show()   
-
-        adata.var[f'{cat}_position_methylation_thresholding_Youden_stats'] = probability_thresholding_list
-        J_max_list = [probability_thresholding_list[i][1] for i in range(adata.shape[1])]
-        adata.var[f'{cat}_position_passed_QC'] = [True if i > J_threshold else False for i in J_max_list]

smftools/preprocessing/calculate_read_length_stats.py

@@ -1,32 +0,0 @@
-## calculate_read_length_stats
-
-# Read length QC
-def calculate_read_length_stats(adata):
-    """
-    Append first valid position in a read and last valid position in the read. From this determine and append the read length. 
-
-    Parameters:
-        adata (AnnData): An adata object
-    
-    Returns:
-        upper_bound (int): last valid position in the dataset
-        lower_bound (int): first valid position in the dataset
-    """
-    import numpy as np
-    import anndata as ad
-    import pandas as pd
-    ## Add basic observation-level (read-level) metadata to the object: first valid position in a read and last valid position in the read. From this determine the read length. Save two new variable which hold the first and last valid positions in the entire dataset
-
-    # Add some basic observation-level (read-level) metadata to the anndata object
-    read_first_valid_position = np.array([int(adata.var_names[i]) for i in np.argmax(~np.isnan(adata.X), axis=1)])
-    read_last_valid_position = np.array([int(adata.var_names[i]) for i in (adata.X.shape[1] - 1 - np.argmax(~np.isnan(adata.X[:, ::-1]), axis=1))])
-    read_length = read_last_valid_position - read_first_valid_position + np.ones(len(read_first_valid_position))
-
-    adata.obs['first_valid_position'] = pd.Series(read_first_valid_position, index=adata.obs.index, dtype=int)
-    adata.obs['last_valid_position'] = pd.Series(read_last_valid_position, index=adata.obs.index, dtype=int)
-    adata.obs['read_length'] = pd.Series(read_length, index=adata.obs.index, dtype=int)
-
-    # Define variables to hold the first and last valid position in the dataset
-    upper_bound = int(np.nanmax(adata.obs['last_valid_position']))
-    lower_bound = int(np.nanmin(adata.obs['first_valid_position']))
-    return upper_bound, lower_bound

smftools/preprocessing/clean_NaN.py

@@ -1,38 +0,0 @@
-## clean_NaN
-from ..readwrite import adata_to_df
-
-# NaN handling
-def clean_NaN(adata, layer=None):
-    """
-    Append layers to adata that contain NaN cleaning strategies.
-
-    Parameters:
-        adata (AnnData): an adata object
-        layer (str): string representing the layer to fill NaN values in
-
-    Returns:
-        None
-    """
-    import numpy as np
-    import anndata as ad
-    import pandas as pd
-    # Fill NaN with closest SMF value
-    df = adata_to_df(adata, layer=layer)
-    df = df.ffill(axis=1).bfill(axis=1)
-    adata.layers['fill_nans_closest'] = df.values
-
-    # Replace NaN values with 0, and 0 with minus 1
-    old_value, new_value = [0, -1]
-    df = adata_to_df(adata, layer=layer)
-    df = df.replace(old_value, new_value)
-    old_value, new_value = [np.nan, 0]
-    df = df.replace(old_value, new_value)
-    adata.layers['nan0_0minus1'] = df.values
-
-    # Replace NaN values with 1, and 1 with 2
-    old_value, new_value = [1, 2]
-    df = adata_to_df(adata, layer=layer)
-    df = df.replace(old_value, new_value)
-    old_value, new_value = [np.nan, 1]
-    df = df.replace(old_value, new_value)
-    adata.layers['nan1_12'] = df.values

smftools/preprocessing/filter_converted_reads_on_methylation.py

@@ -1,27 +0,0 @@
-## filter_converted_reads_on_methylation
-
-## Conversion SMF Specific 
-# Read methylation QC
-def filter_converted_reads_on_methylation(adata, valid_SMF_site_threshold=0.8, min_SMF_threshold=0.025):
-    """
-    Filter adata object using minimum thresholds for valid SMF site fraction in read, as well as minimum methylation content in read.
-
-    Parameters:
-        adata (AnnData): An adata object.
-        valid_SMF_site_threshold (float): A minimum proportion of valid SMF sites that must be present in the read. Default is 0.8
-        min_SMF_threshold (float): A minimum read methylation level. Default is 0.025
-    Returns:
-        None
-    """
-    import numpy as np
-    import anndata as ad
-    import pandas as pd
-
-    if valid_SMF_site_threshold:
-        # Keep reads that have over a given valid GpC site content
-        adata = adata[adata.obs['fraction_valid_GpC_site_in_range'] > valid_SMF_site_threshold].copy()
-    if min_SMF_threshold:
-        # Keep reads with SMF methylation over background methylation.
-        adata = adata[adata.obs['GpC_above_other_C'] == True].copy()
-        # Keep reads over a defined methylation threshold
-        adata = adata[adata.obs['GpC_site_row_methylation_means'] > min_SMF_threshold].copy()

smftools/preprocessing/filter_reads_on_length.py

@@ -1,39 +0,0 @@
-## filter_reads_on_length
-
-def filter_reads_on_length(adata, filter_on_coordinates=False, min_read_length=2700):
-    """
-    Filters the adata object to keep a defined coordinate window, as well as reads that are over a minimum threshold in length.
-
-    Parameters:
-        adata (AnnData): An adata object.
-        filter_on_coordinates (bool | list): If False, skips filtering. Otherwise, provide a list containing integers representing the lower and upper bound coordinates to filter on. Default is False.
-        min_read_length (int): The minimum read length to keep in the filtered dataset. Default is 2700.
-
-    Returns:
-        None
-    Input: Adata object. a list of lower and upper bound (set to False or None if not wanted), and a minimum read length integer.
- 
-    """
-    import numpy as np
-    import anndata as ad
-    import pandas as pd
-    if filter_on_coordinates:
-        lower_bound, upper_bound = filter_on_coordinates
-        # Extract the position information from the adata object as an np array
-        var_names_arr = adata.var_names.astype(int).to_numpy()
-        # Find the upper bound coordinate that is closest to the specified value
-        closest_end_index = np.argmin(np.abs(var_names_arr - upper_bound))
-        upper_bound = int(adata.var_names[closest_end_index])
-        # Find the lower bound coordinate that is closest to the specified value
-        closest_start_index = np.argmin(np.abs(var_names_arr - lower_bound))
-        lower_bound = int(adata.var_names[closest_start_index])
-        # Get a list of positional indexes that encompass the lower and upper bounds of the dataset
-        position_list = list(range(lower_bound, upper_bound + 1))
-        position_list = [str(pos) for pos in position_list]
-        position_set = set(position_list)
-        print(f'Subsetting adata to keep data between coordinates {lower_bound} and {upper_bound}')
-        adata = adata[:, adata.var_names.isin(position_set)].copy()
-
-    if min_read_length:
-        print(f'Subsetting adata to keep reads longer than {min_read_length}')
-        adata = adata[adata.obs['read_length'] > min_read_length].copy()

smftools/preprocessing/invert_adata.py

@@ -1,22 +0,0 @@
-## invert_adata
-
-# Optional inversion of the adata
-def invert_adata(adata):
-    """
-    Inverts the adata object along the variable axis
-
-    Parameters:
-        adata (AnnData): An adata object.
-
-    Returns:
-        None
-    """
-    import numpy as np
-    import anndata as ad
-    # Reassign var_names with new names
-    old_var_names = adata.var_names.astype(int).to_numpy()
-    new_var_names = np.sort(old_var_names)[::-1].astype(str)
-    adata.var['Original_positional_coordinate'] = old_var_names.astype(str)
-    adata.var_names = new_var_names
-    # Sort the AnnData object based on the old var_names
-    adata = adata[:, old_var_names.astype(str)]

smftools/preprocessing/mark_duplicates.py

@@ -1,119 +0,0 @@
-## mark_duplicates
-
-def mark_duplicates(adata, layers, obs_column='Reference', sample_col='Sample_names'):
-    """
-    Marks duplicates in the adata object.
-
-    Parameters:
-        adata (AnnData): An adata object.
-        layers (list): A list of strings representing the layers to use.
-        obs_column (str): A string representing the obs column name to first subset on. Default is 'Reference'.
-        sample_col (str):L A string representing the obs column name to second subset on. Default is 'Sample_names'.
-    
-    Returns:
-        None
-    """
-
-    import numpy as np
-    import pandas as pd
-    import matplotlib.pyplot as plt
-    from scipy.signal import find_peaks
-    import networkx as nx
-    from .binary_layers_to_ohe import binary_layers_to_ohe
-    from .calculate_pairwise_hamming_distances import calculate_pairwise_hamming_distances
-    from .min_non_diagonal import min_non_diagonal
-
-    categories = adata.obs[obs_column].cat.categories 
-    sample_names = adata.obs[sample_col].cat.categories 
-
-    # Calculate the pairwise Hamming distances within each reference/sample set. Determine distance thresholds for each reference/sample pair
-    adata.obs['Nearest_neighbor_Hamming_distance'] = pd.Series(np.nan, index=adata.obs_names, dtype=float)
-    for cat in categories:
-        cat_subset = adata[adata.obs[obs_column] == cat].copy()
-        for sample in sample_names:
-            sample_subset = cat_subset[cat_subset.obs[sample_col] == sample].copy()
-            # Encode sequencing reads as a one-hot-encodings
-            adata.uns[f'{cat}_{sample}_read_OHE_dict'] = binary_layers_to_ohe(sample_subset, layers, stack='hstack')
-            # Unpack the read names and one hot encodings into lists
-            read_names = []
-            ohe_list = []
-            for read_name, ohe in adata.uns[f'{cat}_{sample}_read_OHE_dict'].items():
-                read_names.append(read_name)
-                ohe_list.append(ohe)
-            # Calculate the pairwise hamming distances
-            print(f'Calculating hamming distances for {sample} on {cat} allele')
-            distance_matrix = calculate_pairwise_hamming_distances(ohe_list)
-            n_reads = distance_matrix.shape[0]
-            # Load the hamming matrix into a dataframe with index and column names as the read_names
-            distance_df = pd.DataFrame(distance_matrix, index=read_names, columns=read_names)
-            # Save the distance dataframe into an unstructured component of the adata object
-            adata.uns[f'Pairwise_Hamming_distance_within_{cat}_{sample}'] = distance_df
-            # Calculate the minimum non-self distance for every read in the reference and sample
-            min_distance_values = min_non_diagonal(distance_matrix)
-            min_distance_df = pd.DataFrame({'Nearest_neighbor_Hamming_distance': min_distance_values}, index=read_names)
-            adata.obs.update(min_distance_df)
-            # Generate a histogram of minimum non-self distances for each read
-            min_distance_bins = plt.hist(min_distance_values, bins=n_reads//4)
-            # Normalize the max value in any histogram bin to 1
-            normalized_min_distance_counts = min_distance_bins[0] / np.max(min_distance_bins[0])
-            # Extract the bin index of peak centers in the histogram
-            peak_centers, _ = find_peaks(normalized_min_distance_counts, prominence=0.2, distance=5)
-            first_peak_index = peak_centers[0]
-            offset_index = first_peak_index-1
-            # Use the distance corresponding to the first peak as the threshold distance in graph construction
-            first_peak_distance = min_distance_bins[1][first_peak_index]
-            offset_distance = min_distance_bins[1][offset_index]
-            adata.uns[f'Hamming_distance_threshold_for_{cat}_{sample}'] = offset_distance
-
-    ## Detect likely duplicate reads and mark them in the adata object.
-    adata.obs['Marked_duplicate'] = pd.Series(False, index=adata.obs_names, dtype=bool)
-    adata.obs['Unique_in_final_read_set'] = pd.Series(False, index=adata.obs_names, dtype=bool)
-    adata.obs[f'Hamming_distance_cluster_within_{obs_column}_and_sample'] = pd.Series(-1, index=adata.obs_names, dtype=int)
-
-    for cat in categories:
-        for sample in sample_names:
-            distance_df = adata.uns[f'Pairwise_Hamming_distance_within_{cat}_{sample}']
-            read_names = distance_df.index
-            distance_matrix = distance_df.values
-            n_reads = distance_matrix.shape[0]
-            distance_threshold = adata.uns[f'Hamming_distance_threshold_for_{cat}_{sample}']
-            # Initialize the read distance graph
-            G = nx.Graph()
-            # Add each read as a node to the graph
-            G.add_nodes_from(range(n_reads))
-            # Add edges based on the threshold
-            for i in range(n_reads):
-                for j in range(i + 1, n_reads):
-                    if distance_matrix[i, j] <= distance_threshold:
-                        G.add_edge(i, j)        
-            # Determine distinct clusters using connected components
-            clusters = list(nx.connected_components(G))
-            clusters = [list(cluster) for cluster in clusters]
-            # Get the number of clusters
-            cluster_count = len(clusters)
-            adata.uns[f'Hamming_distance_clusters_within_{cat}_{sample}'] = [cluster_count, n_reads, cluster_count / n_reads, clusters]
-            # Update the adata object
-            read_cluster_map = {}
-            read_duplicate_map = {}
-            read_keep_map = {}
-            for i, cluster in enumerate(clusters):
-                for j, read_index in enumerate(cluster):
-                    read_name = read_names[read_index]
-                    read_cluster_map[read_name] = i
-                    if len(cluster) > 1:
-                        read_duplicate_map[read_name] = True
-                        if j == 0:
-                            read_keep_map[read_name] = True
-                        else:
-                            read_keep_map[read_name] = False
-                    elif len(cluster) == 1:
-                        read_duplicate_map[read_name] = False
-                        read_keep_map[read_name] = True
-            cluster_df = pd.DataFrame.from_dict(read_cluster_map, orient='index', columns=[f'Hamming_distance_cluster_within_{obs_column}_and_sample'], dtype=int)
-            duplicate_df = pd.DataFrame.from_dict(read_duplicate_map, orient='index', columns=['Marked_duplicate'], dtype=bool)
-            keep_df = pd.DataFrame.from_dict(read_keep_map, orient='index', columns=['Unique_in_final_read_set'], dtype=bool)
-            df_combined = pd.concat([cluster_df, duplicate_df, keep_df], axis=1)
-            adata.obs.update(df_combined)
-            adata.obs['Marked_duplicate'] = adata.obs['Marked_duplicate'].astype(bool)
-            adata.obs['Unique_in_final_read_set'] = adata.obs['Unique_in_final_read_set'].astype(bool)
-            print(f'Hamming clusters for {sample} on {cat}\nThreshold: {first_peak_distance}\nNumber clusters: {cluster_count}\nNumber reads: {n_reads}\nFraction unique: {cluster_count / n_reads}')   

smftools/preprocessing/min_non_diagonal.py

@@ -1,25 +0,0 @@
-## min_non_diagonal
-
-def min_non_diagonal(matrix):
-    """
-    Takes a matrix and returns the smallest value from each row with the diagonal masked.
-
-    Parameters:
-        matrix (ndarray): A 2D ndarray.
-
-    Returns:
-        min_values (list): A list of minimum values from each row of the matrix
-    """
-    import numpy as np
-
-    n = matrix.shape[0]
-    min_values = []
-    for i in range(n):
-        # Mask to exclude the diagonal element
-        row_mask = np.ones(n, dtype=bool)
-        row_mask[i] = False
-        # Extract the row excluding the diagonal element
-        row = matrix[i, row_mask]
-        # Find the minimum value in the row
-        min_values.append(np.min(row))
-    return min_values

smftools/preprocessing/remove_duplicates.py

@@ -1,18 +0,0 @@
-# remove_duplicates
-
-def remove_duplicates(adata):
-    """
-    Remove duplicates from the adata object
-
-    Parameters:
-        adata (Anndata): An adata object.
-
-    Returns:
-        None 
-    """
-    import anndata as ad
-
-    initial_size = adata.shape[0]
-    adata = adata[adata.obs['Unique_in_final_read_set'] == True].copy()
-    final_size = adata.shape[0]
-    print(f'Removed {initial_size-final_size} reads from the dataset')