smftools 0.1.6__py3-none-any.whl → 0.2.1__py3-none-any.whl

smftools/preprocessing/binarize_on_Youden.py

@@ -0,0 +1,45 @@
+def binarize_on_Youden(adata, obs_column='Reference'):
+    """
+    Binarize SMF values based on position thresholds determined by calculate_position_Youden.
+
+    Parameters:
+        adata (AnnData): The anndata object to binarize. `calculate_position_Youden` must have been run first.
+        obs_column (str): The obs column to stratify on. Needs to match what was passed in `calculate_position_Youden`.
+
+    Modifies:
+        Adds a new layer to `adata.layers['binarized_methylation']` containing the binarized methylation matrix.
+    """
+    import numpy as np
+    import anndata as ad    
+
+    # Initialize an empty matrix to store the binarized methylation values
+    binarized_methylation = np.full_like(adata.X, np.nan, dtype=float)  # Keeps same shape as adata.X
+
+    # Get unique categories
+    categories = adata.obs[obs_column].cat.categories
+
+    for cat in categories:
+        # Select subset for this category
+        cat_mask = adata.obs[obs_column] == cat
+        cat_subset = adata[cat_mask]
+
+        # Extract the probability matrix
+        original_matrix = cat_subset.X.copy()
+
+        # Extract the thresholds for each position efficiently
+        thresholds = np.array(cat_subset.var[f'{cat}_position_methylation_thresholding_Youden_stats'].apply(lambda x: x[0]))
+
+        # Identify NaN values
+        nan_mask = np.isnan(original_matrix)
+
+        # Binarize based on threshold
+        binarized_matrix = (original_matrix > thresholds).astype(float)
+
+        # Restore NaN values
+        binarized_matrix[nan_mask] = np.nan
+
+        # Assign the binarized values back into the preallocated storage
+        binarized_methylation[cat_mask, :] = binarized_matrix
+
+    # Store the binarized matrix in a new layer
+    adata.layers['binarized_methylation'] = binarized_methylation

smftools/preprocessing/binary_layers_to_ohe.py

@@ -0,0 +1,40 @@
+## binary_layers_to_ohe
+
+## Conversion SMF Specific 
+def binary_layers_to_ohe(adata, binary_layers, stack='hstack'):
+    """
+    Parameters:
+        adata (AnnData): Anndata object.
+        binary_layers (list): a list of strings. Each string represents a layer in the adata object. The layer should encode a binary matrix. 
+        stack (str): Dimension to stack the one-hot-encoding. Options include 'hstack' and 'vstack'. Default is 'hstack', since this is more efficient.
+    
+    Returns:
+        ohe_dict (dict): A dictionary keyed by obs_name that points to a stacked (hstack or vstack) one-hot encoding of the binary layers
+    Input: An adata object and a list of layers containing a binary encoding.
+    """
+    import numpy as np
+    import anndata as ad
+
+    # Ensure that the N layer is last!
+    # Grab all binary layers that are not encoding N
+    ACGT_binary_layers = [layer for layer in binary_layers if 'binary' in layer and layer != 'N_binary_encoding']
+    # If there is a binary layer encoding N, hold it in N_binary_layer
+    N_binary_layer = [layer for layer in binary_layers if layer == 'N_binary_encoding']
+    # Add the N_binary_encoding layer to the end of the list of binary layers
+    all_binary_layers = ACGT_binary_layers + N_binary_layer
+    print(f'Found {all_binary_layers} layers in adata')
+
+    # Extract the layers
+    layers = [adata.layers[layer_name] for layer_name in all_binary_layers]
+    n_reads = layers[0].shape[0]
+    ohe_dict = {}
+    for i in range(n_reads):
+        read_ohe = []
+        for layer in layers:
+            read_ohe.append(layer[i])
+        read_name = adata.obs_names[i]
+        if stack == 'hstack':
+            ohe_dict[read_name] = np.hstack(read_ohe)
+        elif stack == 'vstack':
+            ohe_dict[read_name] = np.vstack(read_ohe)
+    return ohe_dict

smftools/preprocessing/calculate_complexity.py

@@ -0,0 +1,72 @@
+## calculate_complexity
+
+def calculate_complexity(adata, output_directory='', obs_column='Reference', sample_col='Sample_names', plot=True, save_plot=False):
+    """
+    A complexity analysis of the library.
+
+    Parameters:
+        adata (AnnData): An adata object with mark_duplicates already run.
+        output_directory (str): String representing the path to the output directory.
+        obs_column (str): String of the obs column to iterate over.
+        sample_col (str): String of the sample column to iterate over.
+        plot (bool): Whether to plot the complexity model.
+        save_plot (bool): Whether to save the complexity model.
+
+    Returns:
+        None
+
+    """
+    import numpy as np
+    import pandas as pd
+    from scipy.optimize import curve_fit
+
+    def lander_waterman(x, C0):
+        return C0 * (1 - np.exp(-x / C0))
+
+    def count_unique_reads(reads, depth):
+        subsample = np.random.choice(reads, depth, replace=False)
+        return len(np.unique(subsample))
+
+    categories = adata.obs[obs_column].cat.categories 
+    sample_names = adata.obs[sample_col].cat.categories 
+
+    for cat in categories:
+        for sample in sample_names:
+            unique_reads = adata.uns[f'Hamming_distance_cluster_count_within_{cat}_{sample}']
+            total_reads = adata.uns[f'total_reads_within_{cat}_{sample}']
+            reads = np.concatenate((np.arange(unique_reads), np.random.choice(unique_reads, total_reads - unique_reads, replace=True)))
+            # Subsampling depths
+            subsampling_depths = [total_reads // (i+1) for i in range(10)]
+            # Arrays to store results
+            subsampled_total_reads = []
+            subsampled_unique_reads = []
+            # Perform subsampling
+            for depth in subsampling_depths:
+                unique_count = count_unique_reads(reads, depth)
+                subsampled_total_reads.append(depth)
+                subsampled_unique_reads.append(unique_count)
+            # Fit the Lander-Waterman model to the data
+            popt, _ = curve_fit(lander_waterman, subsampled_total_reads, subsampled_unique_reads)
+            # Generate data for the complexity curve
+            x_data = np.linspace(0, 5000, 100)
+            y_data = lander_waterman(x_data, *popt)
+            adata.uns[f'Library_complexity_of_{sample}_on_{cat}'] = popt[0]
+            if plot:
+                import matplotlib.pyplot as plt
+                # Plot the complexity curve
+                plt.figure(figsize=(6, 4))
+                plt.plot(total_reads, unique_reads, 'o', label='Observed unique reads')
+                plt.plot(x_data, y_data, '-', label=f'Lander-Waterman fit\nEstimated C0 = {popt[0]:.2f}')
+                plt.xlabel('Total number of reads')
+                plt.ylabel('Number of unique reads')
+                title = f'Library Complexity Analysis for {sample} on {cat}'
+                plt.title(title)
+                plt.legend()
+                plt.grid(True)
+                if save_plot:
+                    date_string = date_string()
+                    save_name = output_directory + f'/{date_string}_{title}'
+                    plt.savefig(save_name, bbox_inches='tight', pad_inches=0.1)
+                    plt.close()
+                else:
+                    plt.show()

smftools/preprocessing/calculate_complexity_II.py

@@ -0,0 +1,248 @@
+from typing import Optional
+def calculate_complexity_II(
+    adata,
+    output_directory='',
+    sample_col='Sample_names',
+    ref_col: Optional[str] = 'Reference_strand',
+    cluster_col='sequence__merged_cluster_id',
+    plot=True,
+    save_plot=False,
+    n_boot=30,
+    n_depths=12,
+    random_state=0,
+    csv_summary=True,
+    uns_flag='complexity_analysis_complete',
+    force_redo=False,
+    bypass=False
+):
+    """
+    Estimate and plot library complexity.
+
+    If ref_col is None (default), behaves as before: one calculation per sample.
+    If ref_col is provided, computes complexity for each (sample, ref) pair.
+
+    Results:
+      - adata.uns['Library_complexity_results'] : dict keyed by (sample,) or (sample, ref) -> dict with fields
+          C0, n_reads, n_unique, depths, mean_unique, ci_low, ci_high
+      - Also stores per-entity record in adata.uns[f'Library_complexity_{sanitized_name}'] (backwards compatible)
+      - Optionally saves PNGs and CSVs (curve points + fit summary)
+    """
+    import os
+    import numpy as np
+    import pandas as pd
+    import matplotlib.pyplot as plt
+    from scipy.optimize import curve_fit
+    from datetime import datetime
+
+    # early exits
+    already = bool(adata.uns.get(uns_flag, False))
+    if (already and not force_redo):
+        return None
+    if bypass:
+        return None
+
+    rng = np.random.default_rng(random_state)
+
+    def lw(x, C0):
+        return C0 * (1.0 - np.exp(-x / C0))
+
+    def sanitize(name: str) -> str:
+        return "".join(c if c.isalnum() or c in "-._" else "_" for c in str(name))
+
+    # checks
+    for col in (sample_col, cluster_col):
+        if col not in adata.obs.columns:
+            raise KeyError(f"Required column '{col}' not found in adata.obs")
+    if ref_col is not None and ref_col not in adata.obs.columns:
+        raise KeyError(f"ref_col '{ref_col}' not found in adata.obs")
+
+    if save_plot or csv_summary:
+        os.makedirs(output_directory or ".", exist_ok=True)
+
+    # containers to collect CSV rows across all groups
+    fit_records = []
+    curve_records = []
+
+    # output dict stored centrally
+    results = {}
+
+    # build list of groups: either samples only, or (sample, ref) pairs
+    sseries = adata.obs[sample_col].astype("category")
+    samples = list(sseries.cat.categories)
+    if ref_col is None:
+        group_keys = [(s,) for s in samples]
+    else:
+        rseries = adata.obs[ref_col].astype("category")
+        references = list(rseries.cat.categories)
+        group_keys = []
+        # iterate only pairs that exist in data to avoid empty processing
+        for s in samples:
+            mask_s = (adata.obs[sample_col] == s)
+            # find references present for this sample
+            ref_present = pd.Categorical(adata.obs.loc[mask_s, ref_col]).categories
+            # Use intersection of known reference categories and those present for sample
+            for r in ref_present:
+                group_keys.append((s, r))
+
+    # iterate groups
+    for g in group_keys:
+        if ref_col is None:
+            sample = g[0]
+            # filter mask
+            mask = (adata.obs[sample_col] == sample).values
+            group_label = f"{sample}"
+        else:
+            sample, ref = g
+            mask = (adata.obs[sample_col] == sample) & (adata.obs[ref_col] == ref)
+            group_label = f"{sample}__{ref}"
+
+        n_reads = int(mask.sum())
+        if n_reads < 2:
+            # store empty placeholders and continue
+            results[g] = {
+                "C0": np.nan,
+                "n_reads": int(n_reads),
+                "n_unique": 0,
+                "depths": np.array([], dtype=int),
+                "mean_unique": np.array([], dtype=float),
+                "ci_low": np.array([], dtype=float),
+                "ci_high": np.array([], dtype=float),
+            }
+            # also store back-compat key
+            adata.uns[f'Library_complexity_{sanitize(group_label)}'] = results[g]
+            continue
+
+        # cluster ids array for this group
+        clusters = adata.obs.loc[mask, cluster_col].to_numpy()
+        # observed unique molecules at full depth
+        observed_unique = int(pd.unique(clusters).size)
+
+        # choose subsampling depths
+        if n_depths < 2:
+            depths = np.array([n_reads], dtype=int)
+        else:
+            lo = max(10, int(0.05 * n_reads))
+            depths = np.unique(np.linspace(lo, n_reads, n_depths, dtype=int))
+            depths = depths[depths > 0]
+        depths = depths.astype(int)
+        if depths.size == 0:
+            depths = np.array([n_reads], dtype=int)
+
+        # bootstrap sampling: for each depth, sample without replacement (if possible)
+        idx_all = np.arange(n_reads)
+        boot_unique = np.zeros((len(depths), n_boot), dtype=float)
+        for di, d in enumerate(depths):
+            d_use = int(min(d, n_reads))
+            # if d_use == n_reads we can short-circuit and set boot results to full observed uniques
+            if d_use == n_reads:
+                # bootstraps are deterministic in this special case
+                uniq_val = float(observed_unique)
+                boot_unique[di, :] = uniq_val
+                continue
+            # otherwise run bootstraps
+            for b in range(n_boot):
+                take = rng.choice(idx_all, size=d_use, replace=False)
+                boot_unique[di, b] = np.unique(clusters[take]).size
+
+        mean_unique = boot_unique.mean(axis=1)
+        lo_ci = np.percentile(boot_unique, 2.5, axis=1)
+        hi_ci = np.percentile(boot_unique, 97.5, axis=1)
+
+        # fit Lander-Waterman to the mean curve (safe bounds)
+        C0_init = max(observed_unique, mean_unique[-1] if mean_unique.size else observed_unique)
+        try:
+            popt, _ = curve_fit(
+                lw,
+                xdata=depths.astype(float),
+                ydata=mean_unique.astype(float),
+                p0=[C0_init],
+                bounds=(1.0, 1e12),
+                maxfev=10000,
+            )
+            C0 = float(popt[0])
+        except Exception:
+            C0 = float(observed_unique)
+
+        # store results
+        results[g] = {
+            "C0": C0,
+            "n_reads": int(n_reads),
+            "n_unique": int(observed_unique),
+            "depths": depths,
+            "mean_unique": mean_unique,
+            "ci_low": lo_ci,
+            "ci_high": hi_ci,
+        }
+
+        # save per-group in adata.uns for backward compatibility
+        adata.uns[f'Library_complexity_{sanitize(group_label)}'] = results[g]
+
+        # prepare curve and fit records for CSV
+        fit_records.append({
+            "sample": sample,
+            "reference": ref if ref_col is not None else "",
+            "C0": float(C0),
+            "n_reads": int(n_reads),
+            "n_unique_observed": int(observed_unique),
+        })
+
+        x_fit = np.linspace(0, max(n_reads, int(depths[-1]) if depths.size else n_reads), 200)
+        y_fit = lw(x_fit, C0)
+        for d, mu, lo, hi in zip(depths, mean_unique, lo_ci, hi_ci):
+            curve_records.append({
+                "sample": sample,
+                "reference": ref if ref_col is not None else "",
+                "type": "bootstrap",
+                "depth": int(d),
+                "mean_unique": float(mu),
+                "ci_low": float(lo),
+                "ci_high": float(hi),
+            })
+        for xf, yf in zip(x_fit, y_fit):
+            curve_records.append({
+                "sample": sample,
+                "reference": ref if ref_col is not None else "",
+                "type": "fit",
+                "depth": float(xf),
+                "mean_unique": float(yf),
+                "ci_low": np.nan,
+                "ci_high": np.nan,
+            })
+
+        # plotting for this group
+        if plot:
+            plt.figure(figsize=(6.5, 4.5))
+            plt.fill_between(depths, lo_ci, hi_ci, alpha=0.25, label="Bootstrap 95% CI")
+            plt.plot(depths, mean_unique, "o", label="Bootstrap mean")
+            plt.plot([n_reads], [observed_unique], "s", label="Observed (full)")
+            plt.plot(x_fit, y_fit, "-", label=f"LW fit  C0≈{C0:,.0f}")
+            plt.xlabel("Total reads (subsampled depth)")
+            plt.ylabel("Unique molecules (clusters)")
+            title = f"Library Complexity — {sample}" + (f" / {ref}" if ref_col is not None else "")
+            plt.title(title)
+            plt.grid(True, alpha=0.3)
+            plt.legend()
+            plt.tight_layout()
+
+            if save_plot:
+                fname = f"complexity_{sanitize(group_label)}.png"
+                plt.savefig(os.path.join(output_directory or ".", fname), dpi=160, bbox_inches="tight")
+                plt.close()
+            else:
+                plt.show()
+
+    # store central results dict
+    adata.uns["Library_complexity_results"] = results
+
+    # mark complexity analysis as complete
+    adata.uns[uns_flag] = True
+
+    # CSV outputs
+    if csv_summary and (fit_records or curve_records):
+        fit_df = pd.DataFrame(fit_records)
+        curve_df = pd.DataFrame(curve_records)
+        base = output_directory or "."
+        fit_df.to_csv(os.path.join(base, f"complexity_fit_summary.csv"), index=False)
+        curve_df.to_csv(os.path.join(base, f"complexity_curves.csv"), index=False)
+
+    return results

smftools/preprocessing/calculate_consensus.py

@@ -0,0 +1,47 @@
+# calculate_consensus
+
+def calculate_consensus(adata, reference, sample=False, reference_column='Reference', sample_column='Sample'):
+    """
+    Takes an input AnnData object, the reference to subset on, and the sample name to subset on to calculate the consensus sequence of the read set.
+
+    Parameters:
+        adata (AnnData): The input adata to append consensus metadata to.
+        reference (str): The name of the reference to subset the adata on.
+        sample (bool | str): If False, uses all samples. If a string is passed, the adata is further subsetted to only analyze that sample.
+        reference_column (str): The name of the reference column (Default is 'Reference')
+        sample_column (str): The name of the sample column (Default is 'Sample)
+
+    Returns:
+        None
+    
+    """
+    import numpy as np
+
+    # Subset the adata on the refernce of interest. Optionally, subset additionally on a sample of interest.
+    record_subset = adata[adata.obs[reference_column] == reference].copy()
+    if sample:
+        record_subset = record_subset[record_subset.obs[sample_column] == sample].copy()
+    else:
+        pass
+
+    # Grab layer names from the adata object that correspond to the binary encodings of the read sequences.
+    layers = [layer for layer in record_subset.layers if '_binary_' in layer]
+    layer_map, layer_counts = {}, []
+    for i, layer in enumerate(layers):
+        # Gives an integer mapping to access which sequence base the binary layer is encoding
+        layer_map[i] = layer.split('_')[0]
+        # Get the positional counts from all reads for the given base identity.
+        layer_counts.append(np.sum(record_subset.layers[layer], axis=0))
+    # Combine the positional counts array derived from each binary base layer into an ndarray
+    count_array = np.array(layer_counts)
+    # Determine the row index that contains the largest count for each position and store this in an array.
+    nucleotide_indexes = np.argmax(count_array, axis=0)
+    # Map the base sequence derived from the row index array to attain the consensus sequence in a list.
+    consensus_sequence_list = [layer_map[i] for i in nucleotide_indexes]
+
+    if sample:
+        adata.var[f'{reference}_consensus_from_{sample}'] = consensus_sequence_list
+    else:
+        adata.var[f'{reference}_consensus_across_samples'] = consensus_sequence_list
+
+    adata.uns[f'{reference}_consensus_sequence'] = consensus_sequence_list

smftools/preprocessing/calculate_coverage.py

@@ -0,0 +1,51 @@
+def calculate_coverage(adata, obs_column='Reference_strand', position_nan_threshold=0.00001, uns_flag='positional_coverage_calculated'):
+    """
+    Append position-level metadata regarding whether the position is informative within the given observation category.
+
+    Parameters:
+        adata (AnnData): An AnnData object
+        obs_column (str): Observation column value to subset on prior to calculating position statistics for that category.
+        position_nan_threshold (float): A minimal fractional threshold of coverage within the obs_column category to call the position as valid.
+
+    Modifies:
+        - Adds new columns to `adata.var` containing coverage statistics.
+    """
+    import numpy as np
+    import pandas as pd
+    import anndata as ad
+
+    # Only run if not already performed
+    already = bool(adata.uns.get(uns_flag, False))
+    if already:
+        # QC already performed; nothing to do
+        return
+    
+    categories = adata.obs[obs_column].cat.categories
+    n_categories_with_position = np.zeros(adata.shape[1])
+
+    # Loop over categories
+    for cat in categories:
+        print(f'Assessing positional coverage across samples for {cat} reference')
+
+        # Subset to current category
+        cat_mask = adata.obs[obs_column] == cat
+        temp_cat_adata = adata[cat_mask]
+
+        # Compute fraction of valid coverage
+        cat_valid_coverage = np.sum(~np.isnan(temp_cat_adata.X), axis=0)
+        cat_valid_fraction = cat_valid_coverage / temp_cat_adata.shape[0]  # Avoid extra computation
+
+        # Store coverage stats
+        adata.var[f'{cat}_valid_fraction'] = pd.Series(cat_valid_fraction, index=adata.var.index)
+
+        # Assign whether the position is covered based on threshold
+        adata.var[f'position_in_{cat}'] = cat_valid_fraction >= position_nan_threshold
+
+        # Sum the number of categories covering each position
+        n_categories_with_position += adata.var[f'position_in_{cat}'].values
+
+    # Store final category count
+    adata.var[f'N_{obs_column}_with_position'] = n_categories_with_position.astype(int)
+
+    # mark as done
+    adata.uns[uns_flag] = True

smftools/preprocessing/calculate_pairwise_differences.py

@@ -0,0 +1,49 @@
+# calculate_pairwise_differences
+
+def calculate_pairwise_differences(arrays):
+    """
+    Calculate the pairwise differences for a list of h-stacked ndarrays. Ignore N-positions
+
+    Parameters:
+        arrays (str): A list of ndarrays.
+
+    Returns:
+        distance_matrix (ndarray): a 2D array containing the pairwise differences between all arrays.
+    """
+    import numpy as np
+    from tqdm import tqdm
+
+    num_arrays = len(arrays)
+
+    n_rows = 5
+    reshaped_arrays = [array.reshape(n_rows, -1) for array in arrays]
+    N_masks = [array[-1].astype(bool) for array in reshaped_arrays]
+    reshaped_arrays_minus_N = [array[:-1].flatten() for array in reshaped_arrays]
+
+    # Precompute the repeated N masks to avoid repeated computations
+    repeated_N_masks = [np.tile(N_mask, (n_rows - 1)) for N_mask in N_masks]
+
+    # Initialize the distance matrix
+    distance_matrix = np.zeros((num_arrays, num_arrays), dtype=np.float32)
+
+    # Calculate pairwise distances with progress bar
+    for i in tqdm(range(num_arrays), desc="Calculating Pairwise Differences"):
+        array_i = reshaped_arrays_minus_N[i]
+        N_mask_i = repeated_N_masks[i]
+
+        for j in range(i + 1, num_arrays):
+            array_j = reshaped_arrays_minus_N[j]
+            N_mask_j = repeated_N_masks[j]
+
+            # Combined mask to ignore N positions
+            combined_mask = N_mask_i | N_mask_j
+
+            # Calculate the hamming distance directly with boolean operations
+            differences = (array_i != array_j) & ~combined_mask
+            distance = np.sum(differences) / np.sum(~combined_mask)
+            
+            # Store the symmetric distances
+            distance_matrix[i, j] = distance
+            distance_matrix[j, i] = distance
+
+    return distance_matrix

smftools/preprocessing/calculate_pairwise_hamming_distances.py

@@ -0,0 +1,27 @@
+## calculate_pairwise_hamming_distances
+
+## Conversion SMF Specific 
+def calculate_pairwise_hamming_distances(arrays):
+    """
+    Calculate the pairwise Hamming distances for a list of h-stacked ndarrays.
+
+    Parameters:
+        arrays (str): A list of ndarrays.
+
+    Returns:
+        distance_matrix (ndarray): a 2D array containing the pairwise Hamming distances between all arrays.
+
+    """
+    import numpy as np
+    from tqdm import tqdm
+    from scipy.spatial.distance import hamming
+    num_arrays = len(arrays)
+    # Initialize an empty distance matrix
+    distance_matrix = np.zeros((num_arrays, num_arrays))
+    # Calculate pairwise distances with progress bar
+    for i in tqdm(range(num_arrays), desc="Calculating Hamming Distances"):
+        for j in range(i + 1, num_arrays):
+            distance = hamming(arrays[i], arrays[j])
+            distance_matrix[i, j] = distance
+            distance_matrix[j, i] = distance
+    return distance_matrix