smftools 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

smftools/preprocessing/binarize_on_Youden.py

@@ -0,0 +1,45 @@
+def binarize_on_Youden(adata, obs_column='Reference'):
+    """
+    Binarize SMF values based on position thresholds determined by calculate_position_Youden.
+
+    Parameters:
+        adata (AnnData): The anndata object to binarize. `calculate_position_Youden` must have been run first.
+        obs_column (str): The obs column to stratify on. Needs to match what was passed in `calculate_position_Youden`.
+
+    Modifies:
+        Adds a new layer to `adata.layers['binarized_methylation']` containing the binarized methylation matrix.
+    """
+    import numpy as np
+    import anndata as ad    
+
+    # Initialize an empty matrix to store the binarized methylation values
+    binarized_methylation = np.full_like(adata.X, np.nan, dtype=float)  # Keeps same shape as adata.X
+
+    # Get unique categories
+    categories = adata.obs[obs_column].cat.categories
+
+    for cat in categories:
+        # Select subset for this category
+        cat_mask = adata.obs[obs_column] == cat
+        cat_subset = adata[cat_mask]
+
+        # Extract the probability matrix
+        original_matrix = cat_subset.X.copy()
+
+        # Extract the thresholds for each position efficiently
+        thresholds = np.array(cat_subset.var[f'{cat}_position_methylation_thresholding_Youden_stats'].apply(lambda x: x[0]))
+
+        # Identify NaN values
+        nan_mask = np.isnan(original_matrix)
+
+        # Binarize based on threshold
+        binarized_matrix = (original_matrix > thresholds).astype(float)
+
+        # Restore NaN values
+        binarized_matrix[nan_mask] = np.nan
+
+        # Assign the binarized values back into the preallocated storage
+        binarized_methylation[cat_mask, :] = binarized_matrix
+
+    # Store the binarized matrix in a new layer
+    adata.layers['binarized_methylation'] = binarized_methylation

smftools/preprocessing/binary_layers_to_ohe.py

@@ -0,0 +1,40 @@
+## binary_layers_to_ohe
+
+## Conversion SMF Specific 
+def binary_layers_to_ohe(adata, binary_layers, stack='hstack'):
+    """
+    Parameters:
+        adata (AnnData): Anndata object.
+        binary_layers (list): a list of strings. Each string represents a layer in the adata object. The layer should encode a binary matrix. 
+        stack (str): Dimension to stack the one-hot-encoding. Options include 'hstack' and 'vstack'. Default is 'hstack', since this is more efficient.
+    
+    Returns:
+        ohe_dict (dict): A dictionary keyed by obs_name that points to a stacked (hstack or vstack) one-hot encoding of the binary layers
+    Input: An adata object and a list of layers containing a binary encoding.
+    """
+    import numpy as np
+    import anndata as ad
+
+    # Ensure that the N layer is last!
+    # Grab all binary layers that are not encoding N
+    ACGT_binary_layers = [layer for layer in binary_layers if 'binary' in layer and layer != 'N_binary_encoding']
+    # If there is a binary layer encoding N, hold it in N_binary_layer
+    N_binary_layer = [layer for layer in binary_layers if layer == 'N_binary_encoding']
+    # Add the N_binary_encoding layer to the end of the list of binary layers
+    all_binary_layers = ACGT_binary_layers + N_binary_layer
+    print(f'Found {all_binary_layers} layers in adata')
+
+    # Extract the layers
+    layers = [adata.layers[layer_name] for layer_name in all_binary_layers]
+    n_reads = layers[0].shape[0]
+    ohe_dict = {}
+    for i in range(n_reads):
+        read_ohe = []
+        for layer in layers:
+            read_ohe.append(layer[i])
+        read_name = adata.obs_names[i]
+        if stack == 'hstack':
+            ohe_dict[read_name] = np.hstack(read_ohe)
+        elif stack == 'vstack':
+            ohe_dict[read_name] = np.vstack(read_ohe)
+    return ohe_dict

smftools/preprocessing/calculate_complexity.py

@@ -0,0 +1,72 @@
+## calculate_complexity
+
+def calculate_complexity(adata, output_directory='', obs_column='Reference', sample_col='Sample_names', plot=True, save_plot=False):
+    """
+    A complexity analysis of the library.
+
+    Parameters:
+        adata (AnnData): An adata object with mark_duplicates already run.
+        output_directory (str): String representing the path to the output directory.
+        obs_column (str): String of the obs column to iterate over.
+        sample_col (str): String of the sample column to iterate over.
+        plot (bool): Whether to plot the complexity model.
+        save_plot (bool): Whether to save the complexity model.
+
+    Returns:
+        None
+
+    """
+    import numpy as np
+    import pandas as pd
+    from scipy.optimize import curve_fit
+
+    def lander_waterman(x, C0):
+        return C0 * (1 - np.exp(-x / C0))
+
+    def count_unique_reads(reads, depth):
+        subsample = np.random.choice(reads, depth, replace=False)
+        return len(np.unique(subsample))
+
+    categories = adata.obs[obs_column].cat.categories 
+    sample_names = adata.obs[sample_col].cat.categories 
+
+    for cat in categories:
+        for sample in sample_names:
+            unique_reads = adata.uns[f'Hamming_distance_cluster_count_within_{cat}_{sample}']
+            total_reads = adata.uns[f'total_reads_within_{cat}_{sample}']
+            reads = np.concatenate((np.arange(unique_reads), np.random.choice(unique_reads, total_reads - unique_reads, replace=True)))
+            # Subsampling depths
+            subsampling_depths = [total_reads // (i+1) for i in range(10)]
+            # Arrays to store results
+            subsampled_total_reads = []
+            subsampled_unique_reads = []
+            # Perform subsampling
+            for depth in subsampling_depths:
+                unique_count = count_unique_reads(reads, depth)
+                subsampled_total_reads.append(depth)
+                subsampled_unique_reads.append(unique_count)
+            # Fit the Lander-Waterman model to the data
+            popt, _ = curve_fit(lander_waterman, subsampled_total_reads, subsampled_unique_reads)
+            # Generate data for the complexity curve
+            x_data = np.linspace(0, 5000, 100)
+            y_data = lander_waterman(x_data, *popt)
+            adata.uns[f'Library_complexity_of_{sample}_on_{cat}'] = popt[0]
+            if plot:
+                import matplotlib.pyplot as plt
+                # Plot the complexity curve
+                plt.figure(figsize=(6, 4))
+                plt.plot(total_reads, unique_reads, 'o', label='Observed unique reads')
+                plt.plot(x_data, y_data, '-', label=f'Lander-Waterman fit\nEstimated C0 = {popt[0]:.2f}')
+                plt.xlabel('Total number of reads')
+                plt.ylabel('Number of unique reads')
+                title = f'Library Complexity Analysis for {sample} on {cat}'
+                plt.title(title)
+                plt.legend()
+                plt.grid(True)
+                if save_plot:
+                    date_string = date_string()
+                    save_name = output_directory + f'/{date_string}_{title}'
+                    plt.savefig(save_name, bbox_inches='tight', pad_inches=0.1)
+                    plt.close()
+                else:
+                    plt.show()

smftools/preprocessing/calculate_consensus.py

@@ -0,0 +1,47 @@
+# calculate_consensus
+
+def calculate_consensus(adata, reference, sample=False, reference_column='Reference', sample_column='Sample'):
+    """
+    Takes an input AnnData object, the reference to subset on, and the sample name to subset on to calculate the consensus sequence of the read set.
+
+    Parameters:
+        adata (AnnData): The input adata to append consensus metadata to.
+        reference (str): The name of the reference to subset the adata on.
+        sample (bool | str): If False, uses all samples. If a string is passed, the adata is further subsetted to only analyze that sample.
+        reference_column (str): The name of the reference column (Default is 'Reference')
+        sample_column (str): The name of the sample column (Default is 'Sample)
+
+    Returns:
+        None
+    
+    """
+    import numpy as np
+
+    # Subset the adata on the refernce of interest. Optionally, subset additionally on a sample of interest.
+    record_subset = adata[adata.obs[reference_column] == reference].copy()
+    if sample:
+        record_subset = record_subset[record_subset.obs[sample_column] == sample].copy()
+    else:
+        pass
+
+    # Grab layer names from the adata object that correspond to the binary encodings of the read sequences.
+    layers = [layer for layer in record_subset.layers if '_binary_' in layer]
+    layer_map, layer_counts = {}, []
+    for i, layer in enumerate(layers):
+        # Gives an integer mapping to access which sequence base the binary layer is encoding
+        layer_map[i] = layer.split('_')[0]
+        # Get the positional counts from all reads for the given base identity.
+        layer_counts.append(np.sum(record_subset.layers[layer], axis=0))
+    # Combine the positional counts array derived from each binary base layer into an ndarray
+    count_array = np.array(layer_counts)
+    # Determine the row index that contains the largest count for each position and store this in an array.
+    nucleotide_indexes = np.argmax(count_array, axis=0)
+    # Map the base sequence derived from the row index array to attain the consensus sequence in a list.
+    consensus_sequence_list = [layer_map[i] for i in nucleotide_indexes]
+
+    if sample:
+        adata.var[f'{reference}_consensus_from_{sample}'] = consensus_sequence_list
+    else:
+        adata.var[f'{reference}_consensus_across_samples'] = consensus_sequence_list
+
+    adata.uns[f'{reference}_consensus_sequence'] = consensus_sequence_list

smftools/preprocessing/calculate_converted_read_methylation_stats.py

@@ -0,0 +1,94 @@
+## calculate_converted_read_methylation_stats
+
+## Conversion SMF Specific 
+# Read methylation QC
+
+def calculate_converted_read_methylation_stats(adata, reference_column, sample_names_col):
+    """
+    Adds methylation statistics for each read. Indicates whether the read GpC methylation exceeded other_C methylation (background false positives).
+
+    Parameters:
+        adata (AnnData): An adata object
+        reference_column (str): String representing the name of the Reference column to use
+        sample_names_col (str): String representing the name of the sample name column to use
+
+    Returns:
+        None 
+    """
+    import numpy as np
+    import anndata as ad
+    import pandas as pd
+
+    print('Calculating read level methylation statistics')
+
+    references = set(adata.obs[reference_column])
+    sample_names = set(adata.obs[sample_names_col])
+
+    site_types = ['GpC_site', 'CpG_site', 'ambiguous_GpC_CpG_site', 'other_C']
+
+    for site_type in site_types:
+        adata.obs[f'{site_type}_row_methylation_sums'] = pd.Series(0, index=adata.obs_names, dtype=int)
+        adata.obs[f'{site_type}_row_methylation_means'] = pd.Series(np.nan, index=adata.obs_names, dtype=float)
+        adata.obs[f'number_valid_{site_type}_in_read'] = pd.Series(0, index=adata.obs_names, dtype=int)
+        adata.obs[f'fraction_valid_{site_type}_in_range'] = pd.Series(np.nan, index=adata.obs_names, dtype=float)
+    for cat in references:
+        cat_subset = adata[adata.obs[reference_column] == cat].copy()
+        for site_type in site_types:
+            print(f'Iterating over {cat}_{site_type}')
+            observation_matrix = cat_subset.obsm[f'{cat}_{site_type}']
+            number_valid_positions_in_read = np.nansum(~np.isnan(observation_matrix), axis=1)
+            row_methylation_sums = np.nansum(observation_matrix, axis=1)
+            number_valid_positions_in_read[number_valid_positions_in_read == 0] = 1
+            fraction_valid_positions_in_range = number_valid_positions_in_read / np.max(number_valid_positions_in_read)
+            row_methylation_means = np.divide(row_methylation_sums, number_valid_positions_in_read)
+            temp_obs_data = pd.DataFrame({f'number_valid_{site_type}_in_read': number_valid_positions_in_read,
+                                        f'fraction_valid_{site_type}_in_range': fraction_valid_positions_in_range,
+                                        f'{site_type}_row_methylation_sums': row_methylation_sums,
+                                        f'{site_type}_row_methylation_means': row_methylation_means}, index=cat_subset.obs.index)
+            adata.obs.update(temp_obs_data)
+    # Indicate whether the read-level GpC methylation rate exceeds the false methylation rate of the read
+    pass_array = np.array(adata.obs[f'GpC_site_row_methylation_means'] > adata.obs[f'other_C_row_methylation_means'])
+    adata.obs['GpC_above_other_C'] = pd.Series(pass_array, index=adata.obs.index, dtype=bool)
+
+# Below should be a plotting function
+    # adata.uns['methylation_dict'] = {}
+    # n_bins = 50
+    # site_types_to_analyze = ['GpC_site', 'CpG_site', 'ambiguous_GpC_CpG_site', 'other_C']
+
+    # for reference in references:
+    #     reference_adata = adata[adata.obs[reference_column] == reference].copy()
+    #     split_reference = reference.split('_')[0][1:]
+    #     for sample in sample_names:
+    #         sample_adata = reference_adata[reference_adata.obs[sample_names_col] == sample].copy()
+    #         for site_type in site_types_to_analyze:
+    #             methylation_data = sample_adata.obs[f'{site_type}_row_methylation_means']
+    #             max_meth = np.max(sample_adata.obs[f'{site_type}_row_methylation_sums'])
+    #             if not np.isnan(max_meth):
+    #                 n_bins = int(max_meth // 2)
+    #             else:
+    #                 n_bins = 1
+    #             mean = np.mean(methylation_data)
+    #             median = np.median(methylation_data)
+    #             stdev = np.std(methylation_data)
+    #             adata.uns['methylation_dict'][f'{reference}_{sample}_{site_type}'] = [mean, median, stdev]
+    #             if show_methylation_histogram or save_methylation_histogram:
+    #                 fig, ax = plt.subplots(figsize=(6, 4))
+    #                 count, bins, patches =  plt.hist(methylation_data, bins=n_bins, weights=np.ones(len(methylation_data)) / len(methylation_data), alpha=0.7, color='blue', edgecolor='black')
+    #                 plt.axvline(median, color='red', linestyle='dashed', linewidth=1)
+    #                 plt.text(median + stdev, max(count)*0.8, f'Median: {median:.2f}', color='red')
+    #                 plt.axvline(median - stdev, color='green', linestyle='dashed', linewidth=1, label=f'Stdev: {stdev:.2f}')
+    #                 plt.axvline(median + stdev, color='green', linestyle='dashed', linewidth=1)
+    #                 plt.text(median + stdev + 0.05, max(count) / 3, f'+1 Stdev: {stdev:.2f}', color='green')
+    #                 plt.xlabel('Fraction methylated')
+    #                 plt.ylabel('Proportion')
+    #                 title = f'Distribution of {methylation_data.shape[0]} read {site_type} methylation means \nfor {sample} sample on {split_reference} after filtering'
+    #                 plt.title(title, pad=20)
+    #                 plt.xlim(-0.05, 1.05)  # Set x-axis range from 0 to 1
+    #                 ax.spines['right'].set_visible(False)
+    #                 ax.spines['top'].set_visible(False)
+    #                 save_name = output_directory + f'/{readwrite.date_string()} {title}'
+    #                 if save_methylation_histogram:
+    #                     plt.savefig(save_name, bbox_inches='tight', pad_inches=0.1)
+    #                     plt.close()
+    #                 else:
+    #                     plt.show()

smftools/preprocessing/calculate_coverage.py

@@ -0,0 +1,42 @@
+def calculate_coverage(adata, obs_column='Reference_strand', position_nan_threshold=0.05):
+    """
+    Append position-level metadata regarding whether the position is informative within the given observation category.
+
+    Parameters:
+        adata (AnnData): An AnnData object
+        obs_column (str): Observation column value to subset on prior to calculating position statistics for that category.
+        position_nan_threshold (float): A minimal fractional threshold of coverage within the obs_column category to call the position as valid.
+
+    Modifies:
+        - Adds new columns to `adata.var` containing coverage statistics.
+    """
+    import numpy as np
+    import pandas as pd
+    import anndata as ad
+    
+    categories = adata.obs[obs_column].cat.categories
+    n_categories_with_position = np.zeros(adata.shape[1])
+
+    # Loop over categories
+    for cat in categories:
+        print(f'Assessing positional coverage across samples for {cat} reference')
+
+        # Subset to current category
+        cat_mask = adata.obs[obs_column] == cat
+        temp_cat_adata = adata[cat_mask]
+
+        # Compute fraction of valid coverage
+        cat_valid_coverage = np.sum(~np.isnan(temp_cat_adata.X), axis=0)
+        cat_valid_fraction = cat_valid_coverage / temp_cat_adata.shape[0]  # Avoid extra computation
+
+        # Store coverage stats
+        adata.var[f'{cat}_valid_fraction'] = pd.Series(cat_valid_fraction, index=adata.var.index)
+
+        # Assign whether the position is covered based on threshold
+        adata.var[f'position_in_{cat}'] = cat_valid_fraction >= position_nan_threshold
+
+        # Sum the number of categories covering each position
+        n_categories_with_position += adata.var[f'position_in_{cat}'].values
+
+    # Store final category count
+    adata.var[f'N_{obs_column}_with_position'] = n_categories_with_position.astype(int)

smftools/preprocessing/calculate_pairwise_differences.py

@@ -0,0 +1,49 @@
+# calculate_pairwise_differences
+
+def calculate_pairwise_differences(arrays):
+    """
+    Calculate the pairwise differences for a list of h-stacked ndarrays. Ignore N-positions
+
+    Parameters:
+        arrays (str): A list of ndarrays.
+
+    Returns:
+        distance_matrix (ndarray): a 2D array containing the pairwise differences between all arrays.
+    """
+    import numpy as np
+    from tqdm import tqdm
+
+    num_arrays = len(arrays)
+
+    n_rows = 5
+    reshaped_arrays = [array.reshape(n_rows, -1) for array in arrays]
+    N_masks = [array[-1].astype(bool) for array in reshaped_arrays]
+    reshaped_arrays_minus_N = [array[:-1].flatten() for array in reshaped_arrays]
+
+    # Precompute the repeated N masks to avoid repeated computations
+    repeated_N_masks = [np.tile(N_mask, (n_rows - 1)) for N_mask in N_masks]
+
+    # Initialize the distance matrix
+    distance_matrix = np.zeros((num_arrays, num_arrays), dtype=np.float32)
+
+    # Calculate pairwise distances with progress bar
+    for i in tqdm(range(num_arrays), desc="Calculating Pairwise Differences"):
+        array_i = reshaped_arrays_minus_N[i]
+        N_mask_i = repeated_N_masks[i]
+
+        for j in range(i + 1, num_arrays):
+            array_j = reshaped_arrays_minus_N[j]
+            N_mask_j = repeated_N_masks[j]
+
+            # Combined mask to ignore N positions
+            combined_mask = N_mask_i | N_mask_j
+
+            # Calculate the hamming distance directly with boolean operations
+            differences = (array_i != array_j) & ~combined_mask
+            distance = np.sum(differences) / np.sum(~combined_mask)
+            
+            # Store the symmetric distances
+            distance_matrix[i, j] = distance
+            distance_matrix[j, i] = distance
+
+    return distance_matrix

smftools/preprocessing/calculate_pairwise_hamming_distances.py

@@ -0,0 +1,27 @@
+## calculate_pairwise_hamming_distances
+
+## Conversion SMF Specific 
+def calculate_pairwise_hamming_distances(arrays):
+    """
+    Calculate the pairwise Hamming distances for a list of h-stacked ndarrays.
+
+    Parameters:
+        arrays (str): A list of ndarrays.
+
+    Returns:
+        distance_matrix (ndarray): a 2D array containing the pairwise Hamming distances between all arrays.
+
+    """
+    import numpy as np
+    from tqdm import tqdm
+    from scipy.spatial.distance import hamming
+    num_arrays = len(arrays)
+    # Initialize an empty distance matrix
+    distance_matrix = np.zeros((num_arrays, num_arrays))
+    # Calculate pairwise distances with progress bar
+    for i in tqdm(range(num_arrays), desc="Calculating Hamming Distances"):
+        for j in range(i + 1, num_arrays):
+            distance = hamming(arrays[i], arrays[j])
+            distance_matrix[i, j] = distance
+            distance_matrix[j, i] = distance
+    return distance_matrix

smftools/preprocessing/calculate_position_Youden.py

@@ -0,0 +1,115 @@
+## calculate_position_Youden
+
+## Calculating and applying position level thresholds for methylation calls to binarize the SMF data
+def calculate_position_Youden(adata, positive_control_sample='positive', negative_control_sample='negative', J_threshold=0.5, obs_column='Reference', infer_on_percentile=False, inference_variable='', save=False, output_directory=''):
+    """
+    Adds new variable metadata to each position indicating whether the position provides reliable SMF methylation calls. Also outputs plots of the positional ROC curves.
+
+    Parameters:
+        adata (AnnData): An AnnData object.
+        positive_control_sample (str): string representing the sample name corresponding to the Plus MTase control sample.
+        negative_control_sample (str): string representing the sample name corresponding to the Minus MTase control sample.
+        J_threshold (float): A float indicating the J-statistic used to indicate whether a position passes QC for methylation calls.
+        obs_column (str): The category to iterate over.
+        infer_on_perdentile (bool | int): If False, use defined postive and negative control samples. If an int (0 < int < 100) is passed, this uses the top and bottom int percentile of methylated reads based on metric in inference_variable column.
+        inference_variable (str): If infer_on_percentile has an integer value passed, use the AnnData observation column name passed by this string as the metric.
+        save (bool): Whether to save the ROC plots.
+        output_directory (str): String representing the path to the output directory to output the ROC curves.
+    
+    Returns: 
+        None
+    """
+    import numpy as np
+    import pandas as pd
+    import anndata as ad
+    import matplotlib.pyplot as plt
+    from sklearn.metrics import roc_curve, roc_auc_score
+
+    control_samples = [positive_control_sample, negative_control_sample]
+    categories = adata.obs[obs_column].cat.categories 
+    # Iterate over each category in the specified obs_column
+    for cat in categories:
+        print(f"Calculating position Youden statistics for {cat}")
+        # Subset to keep only reads associated with the category
+        cat_subset = adata[adata.obs[obs_column] == cat]
+        # Iterate over positive and negative control samples
+        for control in control_samples:
+            # Initialize a dictionary for the given control sample. This will be keyed by dataset and position to point to a tuple of coordinate position and an array of methylation probabilities
+            adata.uns[f'{cat}_position_methylation_dict_{control}'] = {}
+            if infer_on_percentile:
+                sorted_column = cat_subset.obs[inference_variable].sort_values(ascending=False)
+                if control == "positive":
+                    threshold = np.percentile(sorted_column, 100 - infer_on_percentile)
+                    control_subset = cat_subset[cat_subset.obs[inference_variable] >= threshold, :]
+                else:
+                    threshold = np.percentile(sorted_column, infer_on_percentile)
+                    control_subset = cat_subset[cat_subset.obs[inference_variable] <= threshold, :]   
+            else:
+                # get the current control subset on the given category
+                filtered_obs = cat_subset.obs[cat_subset.obs['Sample_names'].str.contains(control, na=False, regex=True)]
+                control_subset = cat_subset[filtered_obs.index]
+            # Iterate through every position in the control subset
+            for position in range(control_subset.shape[1]):
+                # Get the coordinate name associated with that position
+                coordinate = control_subset.var_names[position]
+                # Get the array of methlyation probabilities for each read in the subset at that position
+                position_data = control_subset.X[:, position]
+                # Get the indexes of everywhere that is not a nan value
+                nan_mask = ~np.isnan(position_data)
+                # Keep only the methlyation data that has real values
+                position_data = position_data[nan_mask]
+                # Get the position data coverage
+                position_coverage = len(position_data)
+                # Get fraction coverage
+                fraction_coverage = position_coverage / control_subset.shape[0]
+                # Save the position and the position methylation data for the control subset
+                adata.uns[f'{cat}_position_methylation_dict_{control}'][f'{position}'] = (position, position_data, fraction_coverage)
+
+    for cat in categories:
+        fig, ax = plt.subplots(figsize=(6, 4))
+        plt.plot([0, 1], [0, 1], linestyle='--', color='gray')
+        plt.xlabel('False Positive Rate')
+        plt.ylabel('True Positive Rate')
+        ax.spines['right'].set_visible(False)
+        ax.spines['top'].set_visible(False)
+        n_passed_positions = 0
+        n_total_positions = 0
+        # Initialize a list that will hold the positional thresholds for the category
+        probability_thresholding_list = [(np.nan, np.nan)] * adata.shape[1]
+        for i, key in enumerate(adata.uns[f'{cat}_position_methylation_dict_{positive_control_sample}'].keys()):
+            position = int(adata.uns[f'{cat}_position_methylation_dict_{positive_control_sample}'][key][0])
+            positive_position_array = adata.uns[f'{cat}_position_methylation_dict_{positive_control_sample}'][key][1]
+            fraction_coverage = adata.uns[f'{cat}_position_methylation_dict_{positive_control_sample}'][key][2]
+            if fraction_coverage > 0.2:
+                try:
+                    negative_position_array = adata.uns[f'{cat}_position_methylation_dict_{negative_control_sample}'][key][1] 
+                    # Combine the negative and positive control data
+                    data = np.concatenate([negative_position_array, positive_position_array])
+                    labels = np.array([0] * len(negative_position_array) + [1] * len(positive_position_array))
+                    # Calculate the ROC curve
+                    fpr, tpr, thresholds = roc_curve(labels, data)
+                    # Calculate Youden's J statistic
+                    J = tpr - fpr
+                    optimal_idx = np.argmax(J)
+                    optimal_threshold = thresholds[optimal_idx]
+                    max_J = np.max(J)
+                    data_tuple = (optimal_threshold, max_J)
+                    probability_thresholding_list[position] = data_tuple
+                    n_total_positions += 1
+                    if max_J > J_threshold:
+                        n_passed_positions += 1
+                        plt.plot(fpr, tpr, label='ROC curve')
+                except:
+                    probability_thresholding_list[position] = (0.8, np.nan)
+        title = f'ROC Curve for {n_passed_positions} positions with J-stat greater than {J_threshold}\n out of {n_total_positions} total positions on {cat}'
+        plt.title(title)
+        save_name = output_directory + f'/{title}'
+        if save:
+            plt.savefig(save_name)
+            plt.close()
+        else:
+            plt.show()   
+
+        adata.var[f'{cat}_position_methylation_thresholding_Youden_stats'] = probability_thresholding_list
+        J_max_list = [probability_thresholding_list[i][1] for i in range(adata.shape[1])]
+        adata.var[f'{cat}_position_passed_QC'] = [True if i > J_threshold else False for i in J_max_list]

smftools/preprocessing/calculate_read_length_stats.py

@@ -0,0 +1,79 @@
+## calculate_read_length_stats
+
+# Read length QC
+def calculate_read_length_stats(adata, reference_column='', sample_names_col=''):
+    """
+    Append first valid position in a read and last valid position in the read. From this determine and append the read length. 
+
+    Parameters:
+        adata (AnnData): An adata object
+        reference_column (str): String representing the name of the Reference column to use
+        sample_names_col (str): String representing the name of the sample name column to use
+    
+    Returns:
+        upper_bound (int): last valid position in the dataset
+        lower_bound (int): first valid position in the dataset
+    """
+    import numpy as np
+    import anndata as ad
+    import pandas as pd
+
+    print('Calculating read length statistics')
+
+    references = set(adata.obs[reference_column])
+    sample_names = set(adata.obs[sample_names_col])
+
+    ## Add basic observation-level (read-level) metadata to the object: first valid position in a read and last valid position in the read. From this determine the read length. Save two new variable which hold the first and last valid positions in the entire dataset
+    print('calculating read length stats')
+    # Add some basic observation-level (read-level) metadata to the anndata object
+    read_first_valid_position = np.array([int(adata.var_names[i]) for i in np.argmax(~np.isnan(adata.X), axis=1)])
+    read_last_valid_position = np.array([int(adata.var_names[i]) for i in (adata.X.shape[1] - 1 - np.argmax(~np.isnan(adata.X[:, ::-1]), axis=1))])
+    read_length = read_last_valid_position - read_first_valid_position + np.ones(len(read_first_valid_position))
+
+    adata.obs['first_valid_position'] = pd.Series(read_first_valid_position, index=adata.obs.index, dtype=int)
+    adata.obs['last_valid_position'] = pd.Series(read_last_valid_position, index=adata.obs.index, dtype=int)
+    adata.obs['read_length'] = pd.Series(read_length, index=adata.obs.index, dtype=int)
+
+    # Define variables to hold the first and last valid position in the dataset
+    upper_bound = int(np.nanmax(adata.obs['last_valid_position']))
+    lower_bound = int(np.nanmin(adata.obs['first_valid_position']))
+
+    return upper_bound, lower_bound
+
+# # Add an unstructured element to the anndata object which points to a dictionary of read lengths keyed by reference and sample name. Points to a tuple containing (mean, median, stdev) of the read lengths of the sample for the given reference strand
+#     ## Plot histogram of read length data and save the median and stdev of the read lengths for each sample.
+#     adata.uns['read_length_dict'] = {}
+
+#     for reference in references:
+#         temp_reference_adata = adata[adata.obs[reference_column] == reference].copy()
+#         split_reference = reference.split('_')[0][1:]
+#         for sample in sample_names:
+#             temp_sample_adata = temp_reference_adata[temp_reference_adata.obs[sample_names_col] == sample].copy()
+#             temp_data = temp_sample_adata.obs['read_length']
+#             max_length = np.max(temp_data)
+#             mean = np.mean(temp_data)
+#             median = np.median(temp_data)
+#             stdev = np.std(temp_data)
+#             adata.uns['read_length_dict'][f'{reference}_{sample}'] = [mean, median, stdev]
+#             if not np.isnan(max_length):
+#                 n_bins = int(max_length // 100)
+#             else:
+#                 n_bins = 1
+#             if show_read_length_histogram or save_read_length_histogram:
+#                 plt.figure(figsize=(10, 6))
+#                 plt.text(median + 0.5, max(plt.hist(temp_data, bins=n_bins)[0]) / 2, f'Median: {median:.2f}', color='red')
+#                 plt.hist(temp_data, bins=n_bins, alpha=0.7, color='blue', edgecolor='black')
+#                 plt.xlabel('Read Length')
+#                 plt.ylabel('Count')
+#                 title = f'Read length distribution of {temp_sample_adata.shape[0]} total reads from {sample} sample on {split_reference} allele'
+#                 plt.title(title)
+#                 # Add a vertical line at the median
+#                 plt.axvline(median, color='red', linestyle='dashed', linewidth=1)
+#                 # Annotate the median
+#                 plt.xlim(lower_bound - 100, upper_bound + 100) 
+#                 if save_read_length_histogram:
+#                     save_name = output_directory + f'/{readwrite.date_string()} {title}'
+#                     plt.savefig(save_name, bbox_inches='tight', pad_inches=0.1)
+#                     plt.close()
+#                 else:
+#                     plt.show()