smftools 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl

smftools/preprocessing/calculate_coverage.py

@@ -1,54 +1,76 @@
-def calculate_coverage(adata, 
-                       ref_column='Reference_strand', 
-                       position_nan_threshold=0.01, 
-                       uns_flag='calculate_coverage_performed'):
-    """
-    Append position-level metadata regarding whether the position is informative within the given observation category.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from smftools.logging_utils import get_logger
+
+if TYPE_CHECKING:
+    import anndata as ad
 
-    Parameters:
-        adata (AnnData): An AnnData object
-        obs_column (str): Observation column value to subset on prior to calculating position statistics for that category.
-        position_nan_threshold (float): A minimal fractional threshold of coverage within the obs_column category to call the position as valid.
+logger = get_logger(__name__)
 
-    Modifies:
-        - Adds new columns to `adata.var` containing coverage statistics.
+
+def calculate_coverage(
+    adata: "ad.AnnData",
+    ref_column: str = "Reference_strand",
+    position_nan_threshold: float = 0.01,
+    smf_modality: str = "deaminase",
+    target_layer: str = "binarized_methylation",
+    uns_flag: str = "calculate_coverage_performed",
+    force_redo: bool = False,
+) -> None:
+    """Append position-level coverage metadata per reference category.
+
+    Args:
+        adata: AnnData object.
+        ref_column: Obs column used to define reference/strand categories.
+        position_nan_threshold: Minimum fraction of coverage to mark a position as valid.
+        smf_modality: SMF modality. Use ``adata.X`` for conversion/deaminase or ``target_layer`` for direct.
+        target_layer: Layer used for direct SMF coverage calculations.
+        uns_flag: Flag in ``adata.uns`` indicating prior completion.
+        force_redo: Whether to rerun even if ``uns_flag`` is set.
     """
     import numpy as np
     import pandas as pd
-    import anndata as ad
 
     # Only run if not already performed
     already = bool(adata.uns.get(uns_flag, False))
-    if already:
+    if already and not force_redo:
         # QC already performed; nothing to do
         return
-    
+
     references = adata.obs[ref_column].cat.categories
     n_categories_with_position = np.zeros(adata.shape[1])
 
     # Loop over references
     for ref in references:
-        print(f'Assessing positional coverage across samples for {ref} reference')
+        logger.info("Assessing positional coverage across samples for %s reference", ref)
 
         # Subset to current category
         ref_mask = adata.obs[ref_column] == ref
         temp_ref_adata = adata[ref_mask]
 
+        if smf_modality == "direct":
+            matrix = temp_ref_adata.layers[target_layer]
+        else:
+            matrix = temp_ref_adata.X
+
         # Compute fraction of valid coverage
-        ref_valid_coverage = np.sum(~np.isnan(temp_ref_adata.X), axis=0)
+        ref_valid_coverage = np.sum(~np.isnan(matrix), axis=0)
         ref_valid_fraction = ref_valid_coverage / temp_ref_adata.shape[0]  # Avoid extra computation
 
         # Store coverage stats
-        adata.var[f'{ref}_valid_fraction'] = pd.Series(ref_valid_fraction, index=adata.var.index)
+        adata.var[f"{ref}_valid_count"] = pd.Series(ref_valid_coverage, index=adata.var.index)
+        adata.var[f"{ref}_valid_fraction"] = pd.Series(ref_valid_fraction, index=adata.var.index)
 
         # Assign whether the position is covered based on threshold
-        adata.var[f'position_in_{ref}'] = ref_valid_fraction >= position_nan_threshold
+        adata.var[f"position_in_{ref}"] = ref_valid_fraction >= position_nan_threshold
 
         # Sum the number of categories covering each position
-        n_categories_with_position += adata.var[f'position_in_{ref}'].values
+        n_categories_with_position += adata.var[f"position_in_{ref}"].values
 
     # Store final category count
-    adata.var[f'N_{ref_column}_with_position'] = n_categories_with_position.astype(int)
+    adata.var[f"N_{ref_column}_with_position"] = n_categories_with_position.astype(int)
 
     # mark as done
-    adata.uns[uns_flag] = True
+    adata.uns[uns_flag] = True

smftools/preprocessing/calculate_pairwise_differences.py

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 # calculate_pairwise_differences
 
+
 def calculate_pairwise_differences(arrays):
     """
     Calculate the pairwise differences for a list of h-stacked ndarrays. Ignore N-positions
@@ -41,7 +44,7 @@ def calculate_pairwise_differences(arrays):
             # Calculate the hamming distance directly with boolean operations
             differences = (array_i != array_j) & ~combined_mask
             distance = np.sum(differences) / np.sum(~combined_mask)
-            
+
             # Store the symmetric distances
             distance_matrix[i, j] = distance
             distance_matrix[j, i] = distance

smftools/preprocessing/calculate_pairwise_hamming_distances.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+
 ## calculate_pairwise_hamming_distances
 
-## Conversion SMF Specific 
+
+## Conversion SMF Specific
 def calculate_pairwise_hamming_distances(arrays):
     """
     Calculate the pairwise Hamming distances for a list of h-stacked ndarrays.
@@ -13,8 +16,9 @@ def calculate_pairwise_hamming_distances(arrays):
 
     """
     import numpy as np
-    from tqdm import tqdm
     from scipy.spatial.distance import hamming
+    from tqdm import tqdm
+
     num_arrays = len(arrays)
     # Initialize an empty distance matrix
     distance_matrix = np.zeros((num_arrays, num_arrays))
@@ -24,4 +28,4 @@ def calculate_pairwise_hamming_distances(arrays):
             distance = hamming(arrays[i], arrays[j])
             distance_matrix[i, j] = distance
             distance_matrix[j, i] = distance
-    return distance_matrix
+    return distance_matrix

smftools/preprocessing/calculate_position_Youden.py

@@ -1,69 +1,102 @@
 ## calculate_position_Youden
 ## Calculating and applying position level thresholds for methylation calls to binarize the SMF data
-def calculate_position_Youden(adata, 
-                              positive_control_sample=None, 
-                              negative_control_sample=None, 
-                              J_threshold=0.5, 
-                              ref_column='Reference_strand', 
-                              sample_column='Sample_names',
-                              infer_on_percentile=True, 
-                              inference_variable='Raw_modification_signal', 
-                              save=False, 
-                              output_directory=''):
-    """
-    Adds new variable metadata to each position indicating whether the position provides reliable SMF methylation calls. Also outputs plots of the positional ROC curves.
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from smftools.logging_utils import get_logger
+from smftools.optional_imports import require
+
+if TYPE_CHECKING:
+    import anndata as ad
+
+logger = get_logger(__name__)
+
+
+def calculate_position_Youden(
+    adata: "ad.AnnData",
+    positive_control_sample: str | None = None,
+    negative_control_sample: str | None = None,
+    J_threshold: float = 0.5,
+    ref_column: str = "Reference_strand",
+    sample_column: str = "Sample_names",
+    infer_on_percentile: bool | int = True,
+    inference_variable: str = "Raw_modification_signal",
+    save: bool = False,
+    output_directory: str | Path = "",
+) -> None:
+    """Add position-level Youden thresholds and optional ROC plots.
 
-    Parameters:
-        adata (AnnData): An AnnData object.
-        positive_control_sample (str): string representing the sample name corresponding to the Plus MTase control sample.
-        negative_control_sample (str): string representing the sample name corresponding to the Minus MTase control sample.
-        J_threshold (float): A float indicating the J-statistic used to indicate whether a position passes QC for methylation calls.
-        obs_column (str): The category to iterate over.
-        infer_on_perdentile (bool | int): If False, use defined postive and negative control samples. If an int (0 < int < 100) is passed, this uses the top and bottom int percentile of methylated reads based on metric in inference_variable column.
-        inference_variable (str): If infer_on_percentile has an integer value passed, use the AnnData observation column name passed by this string as the metric.
-        save (bool): Whether to save the ROC plots.
-        output_directory (str): String representing the path to the output directory to output the ROC curves.
-    
-    Returns: 
-        None
+    Args:
+        adata: AnnData object.
+        positive_control_sample: Sample name for the plus MTase control.
+        negative_control_sample: Sample name for the minus MTase control.
+        J_threshold: J-statistic threshold for QC.
+        ref_column: Obs column for reference/strand categories.
+        sample_column: Obs column for sample identifiers.
+        infer_on_percentile: If ``False``, use provided controls. If an int in ``(0, 100)``,
+            use percentile-based inference from ``inference_variable``.
+        inference_variable: Obs column used for percentile inference.
+        save: Whether to save ROC plots to disk.
+        output_directory: Output directory for ROC plots.
     """
     import numpy as np
-    import pandas as pd
-    import anndata as ad
-    import matplotlib.pyplot as plt
-    from sklearn.metrics import roc_curve, roc_auc_score
+
+    plt = require("matplotlib.pyplot", extra="plotting", purpose="Youden ROC plots")
+    sklearn_metrics = require(
+        "sklearn.metrics",
+        extra="ml-base",
+        purpose="Youden ROC curve calculation",
+    )
+    roc_curve = sklearn_metrics.roc_curve
 
     control_samples = [positive_control_sample, negative_control_sample]
-    references = adata.obs[ref_column].cat.categories 
+    references = adata.obs[ref_column].cat.categories
     # Iterate over each category in the specified obs_column
     for ref in references:
-        print(f"Calculating position Youden statistics for {ref}")
+        logger.info("Calculating position Youden statistics for %s", ref)
         # Subset to keep only reads associated with the category
         ref_subset = adata[adata.obs[ref_column] == ref]
         # Iterate over positive and negative control samples
         for i, control in enumerate(control_samples):
-            # Initialize a dictionary for the given control sample. This will be keyed by dataset and position to point to a tuple of coordinate position and an array of methylation probabilities
-            adata.uns[f'{ref}_position_methylation_dict_{control}'] = {}
             # If controls are not passed and infer on percentile is True, infer thresholds based on top and bottom percentile windows for a given obs column metric.
             if infer_on_percentile and not control:
+                logger.info(
+                    "Inferring methylation control reads for %s based on %s percentiles of %s",
+                    ref,
+                    infer_on_percentile,
+                    inference_variable,
+                )
                 sorted_column = ref_subset.obs[inference_variable].sort_values(ascending=False)
                 if i == 0:
-                    control == 'positive'
+                    logger.info("Using top %s percentile for positive control", infer_on_percentile)
+                    control = "positive"
                     positive_control_sample = control
                     threshold = np.percentile(sorted_column, 100 - infer_on_percentile)
                     control_subset = ref_subset[ref_subset.obs[inference_variable] >= threshold, :]
                 else:
-                    control == 'negative'
+                    logger.info(
+                        "Using bottom %s percentile for negative control", infer_on_percentile
+                    )
+                    control = "negative"
                     negative_control_sample = control
                     threshold = np.percentile(sorted_column, infer_on_percentile)
-                    control_subset = ref_subset[ref_subset.obs[inference_variable] <= threshold, :]   
+                    control_subset = ref_subset[ref_subset.obs[inference_variable] <= threshold, :]
             elif not infer_on_percentile and not control:
-                print("Can not threshold Anndata on Youden threshold. Need to either provide control samples or set infer_on_percentile to True")
+                logger.error(
+                    "Can not threshold Anndata on Youden threshold. Need to either provide control samples or set infer_on_percentile to True"
+                )
                 return
             else:
+                logger.info("Using provided control sample: %s", control)
                 # get the current control subset on the given category
                 filtered_obs = ref_subset.obs[ref_subset.obs[sample_column] == control]
                 control_subset = ref_subset[filtered_obs.index]
+
+            # Initialize a dictionary for the given control sample. This will be keyed by dataset and position to point to a tuple of coordinate position and an array of methylation probabilities
+            adata.uns[f"{ref}_position_methylation_dict_{control}"] = {}
+
             # Iterate through every position in the control subset
             for position in range(control_subset.shape[1]):
                 # Get the coordinate name associated with that position
@@ -79,29 +112,45 @@ def calculate_position_Youden(adata,
                 # Get fraction coverage
                 fraction_coverage = position_coverage / control_subset.shape[0]
                 # Save the position and the position methylation data for the control subset
-                adata.uns[f'{ref}_position_methylation_dict_{control}'][f'{position}'] = (position, position_data, fraction_coverage)
+                adata.uns[f"{ref}_position_methylation_dict_{control}"][f"{position}"] = (
+                    position,
+                    position_data,
+                    fraction_coverage,
+                )
 
     for ref in references:
         fig, ax = plt.subplots(figsize=(6, 4))
-        plt.plot([0, 1], [0, 1], linestyle='--', color='gray')
-        plt.xlabel('False Positive Rate')
-        plt.ylabel('True Positive Rate')
-        ax.spines['right'].set_visible(False)
-        ax.spines['top'].set_visible(False)
+        plt.plot([0, 1], [0, 1], linestyle="--", color="gray")
+        plt.xlabel("False Positive Rate")
+        plt.ylabel("True Positive Rate")
+        ax.spines["right"].set_visible(False)
+        ax.spines["top"].set_visible(False)
         n_passed_positions = 0
         n_total_positions = 0
         # Initialize a list that will hold the positional thresholds for the category
         probability_thresholding_list = [(np.nan, np.nan)] * adata.shape[1]
-        for i, key in enumerate(adata.uns[f'{ref}_position_methylation_dict_{positive_control_sample}'].keys()):
-            position = int(adata.uns[f'{ref}_position_methylation_dict_{positive_control_sample}'][key][0])
-            positive_position_array = adata.uns[f'{ref}_position_methylation_dict_{positive_control_sample}'][key][1]
-            fraction_coverage = adata.uns[f'{ref}_position_methylation_dict_{positive_control_sample}'][key][2]
+        for i, key in enumerate(
+            adata.uns[f"{ref}_position_methylation_dict_{positive_control_sample}"].keys()
+        ):
+            position = int(
+                adata.uns[f"{ref}_position_methylation_dict_{positive_control_sample}"][key][0]
+            )
+            positive_position_array = adata.uns[
+                f"{ref}_position_methylation_dict_{positive_control_sample}"
+            ][key][1]
+            fraction_coverage = adata.uns[
+                f"{ref}_position_methylation_dict_{positive_control_sample}"
+            ][key][2]
             if fraction_coverage > 0.2:
                 try:
-                    negative_position_array = adata.uns[f'{ref}_position_methylation_dict_{negative_control_sample}'][key][1] 
+                    negative_position_array = adata.uns[
+                        f"{ref}_position_methylation_dict_{negative_control_sample}"
+                    ][key][1]
                     # Combine the negative and positive control data
                     data = np.concatenate([negative_position_array, positive_position_array])
-                    labels = np.array([0] * len(negative_position_array) + [1] * len(positive_position_array))
+                    labels = np.array(
+                        [0] * len(negative_position_array) + [1] * len(positive_position_array)
+                    )
                     # Calculate the ROC curve
                     fpr, tpr, thresholds = roc_curve(labels, data)
                     # Calculate Youden's J statistic
@@ -114,18 +163,24 @@ def calculate_position_Youden(adata,
                     n_total_positions += 1
                     if max_J > J_threshold:
                         n_passed_positions += 1
-                        plt.plot(fpr, tpr, label='ROC curve')
-                except:
+                        plt.plot(fpr, tpr, label="ROC curve")
+                except Exception:
                     probability_thresholding_list[position] = (0.8, np.nan)
-        title = f'ROC Curve for {n_passed_positions} positions with J-stat greater than {J_threshold}\n out of {n_total_positions} total positions on {ref}'
+        title = f"ROC Curve for {n_passed_positions} positions with J-stat greater than {J_threshold}\n out of {n_total_positions} total positions on {ref}"
         plt.title(title)
         save_name = output_directory / f"{title}.png"
         if save:
             plt.savefig(save_name)
             plt.close()
         else:
-            plt.show()   
+            plt.show()
 
-        adata.var[f'{ref}_position_methylation_thresholding_Youden_stats'] = probability_thresholding_list
+        adata.var[f"{ref}_position_methylation_thresholding_Youden_stats"] = (
+            probability_thresholding_list
+        )
         J_max_list = [probability_thresholding_list[i][1] for i in range(adata.shape[1])]
-        adata.var[f'{ref}_position_passed_QC'] = [True if i > J_threshold else False for i in J_max_list]
+        adata.var[f"{ref}_position_passed_Youden_thresholding_QC"] = [
+            True if i > J_threshold else False for i in J_max_list
+        ]
+
+    logger.info("Finished calculating position Youden statistics")

smftools/preprocessing/calculate_read_length_stats.py

@@ -1,45 +1,74 @@
 ## calculate_read_length_stats
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from smftools.logging_utils import get_logger
+
+if TYPE_CHECKING:
+    import anndata as ad
+
+logger = get_logger(__name__)
+
+
 # Read length QC
-def calculate_read_length_stats(adata, reference_column='', sample_names_col=''):
-    """
-    Append first valid position in a read and last valid position in the read. From this determine and append the read length. 
+def calculate_read_length_stats(
+    adata: "ad.AnnData",
+    reference_column: str = "",
+    sample_names_col: str = "",
+) -> tuple[int, int]:
+    """Calculate per-read length statistics and store them in ``adata.obs``.
+
+    Args:
+        adata: AnnData object.
+        reference_column: Obs column containing reference identifiers.
+        sample_names_col: Obs column containing sample identifiers.
 
-    Parameters:
-        adata (AnnData): An adata object
-        reference_column (str): String representing the name of the Reference column to use
-        sample_names_col (str): String representing the name of the sample name column to use
-    
     Returns:
-        upper_bound (int): last valid position in the dataset
-        lower_bound (int): first valid position in the dataset
+        tuple[int, int]: ``(upper_bound, lower_bound)`` for valid positions in the dataset.
     """
     import numpy as np
-    import anndata as ad
     import pandas as pd
 
-    print('Calculating read length statistics')
+    logger.info("Calculating read length statistics")
 
     references = set(adata.obs[reference_column])
     sample_names = set(adata.obs[sample_names_col])
 
     ## Add basic observation-level (read-level) metadata to the object: first valid position in a read and last valid position in the read. From this determine the read length. Save two new variable which hold the first and last valid positions in the entire dataset
-    print('calculating read length stats')
+    logger.info("Calculating read length stats")
     # Add some basic observation-level (read-level) metadata to the anndata object
-    read_first_valid_position = np.array([int(adata.var_names[i]) for i in np.argmax(~np.isnan(adata.X), axis=1)])
-    read_last_valid_position = np.array([int(adata.var_names[i]) for i in (adata.X.shape[1] - 1 - np.argmax(~np.isnan(adata.X[:, ::-1]), axis=1))])
-    read_length = read_last_valid_position - read_first_valid_position + np.ones(len(read_first_valid_position))
+    read_first_valid_position = np.array(
+        [int(adata.var_names[i]) for i in np.argmax(~np.isnan(adata.X), axis=1)]
+    )
+    read_last_valid_position = np.array(
+        [
+            int(adata.var_names[i])
+            for i in (adata.X.shape[1] - 1 - np.argmax(~np.isnan(adata.X[:, ::-1]), axis=1))
+        ]
+    )
+    read_length = (
+        read_last_valid_position
+        - read_first_valid_position
+        + np.ones(len(read_first_valid_position))
+    )
 
-    adata.obs['first_valid_position'] = pd.Series(read_first_valid_position, index=adata.obs.index, dtype=int)
-    adata.obs['last_valid_position'] = pd.Series(read_last_valid_position, index=adata.obs.index, dtype=int)
-    adata.obs['read_length'] = pd.Series(read_length, index=adata.obs.index, dtype=int)
+    adata.obs["first_valid_position"] = pd.Series(
+        read_first_valid_position, index=adata.obs.index, dtype=int
+    )
+    adata.obs["last_valid_position"] = pd.Series(
+        read_last_valid_position, index=adata.obs.index, dtype=int
+    )
+    adata.obs["read_length"] = pd.Series(read_length, index=adata.obs.index, dtype=int)
 
     # Define variables to hold the first and last valid position in the dataset
-    upper_bound = int(np.nanmax(adata.obs['last_valid_position']))
-    lower_bound = int(np.nanmin(adata.obs['first_valid_position']))
+    upper_bound = int(np.nanmax(adata.obs["last_valid_position"]))
+    lower_bound = int(np.nanmin(adata.obs["first_valid_position"]))
 
     return upper_bound, lower_bound
 
+
 # # Add an unstructured element to the anndata object which points to a dictionary of read lengths keyed by reference and sample name. Points to a tuple containing (mean, median, stdev) of the read lengths of the sample for the given reference strand
 #     ## Plot histogram of read length data and save the median and stdev of the read lengths for each sample.
 #     adata.uns['read_length_dict'] = {}
@@ -70,10 +99,10 @@ def calculate_read_length_stats(adata, reference_column='', sample_names_col='')
 #                 # Add a vertical line at the median
 #                 plt.axvline(median, color='red', linestyle='dashed', linewidth=1)
 #                 # Annotate the median
-#                 plt.xlim(lower_bound - 100, upper_bound + 100) 
+#                 plt.xlim(lower_bound - 100, upper_bound + 100)
 #                 if save_read_length_histogram:
 #                     save_name = output_directory + f'/{readwrite.date_string()} {title}'
 #                     plt.savefig(save_name, bbox_inches='tight', pad_inches=0.1)
 #                     plt.close()
 #                 else:
-#                     plt.show()
+#                     plt.show()