smftools 0.1.6__py3-none-any.whl → 0.2.1__py3-none-any.whl

smftools/preprocessing/invert_adata.py

@@ -0,0 +1,37 @@
+## invert_adata
+
+def invert_adata(adata, uns_flag='adata_positions_inverted', force_redo=False):
+    """
+    Inverts the AnnData object along the column (variable) axis.
+
+    Parameters:
+        adata (AnnData): An AnnData object.
+
+    Returns:
+        AnnData: A new AnnData object with inverted column ordering.
+    """
+    import numpy as np
+    import anndata as ad
+
+    # Only run if not already performed
+    already = bool(adata.uns.get(uns_flag, False))
+    if (already and not force_redo):
+        # QC already performed; nothing to do
+        return adata
+
+    print("Inverting AnnData along the column axis...")
+
+    # Reverse the order of columns (variables)
+    inverted_adata = adata[:, ::-1].copy()
+
+    # Reassign var_names with new order
+    inverted_adata.var_names = adata.var_names
+
+    # Optional: Store original coordinates for reference
+    inverted_adata.var["Original_var_names"] = adata.var_names[::-1]
+
+    # mark as done
+    inverted_adata.uns[uns_flag] = True
+
+    print("Inversion complete!")
+    return inverted_adata

smftools/preprocessing/load_sample_sheet.py

@@ -0,0 +1,53 @@
+def load_sample_sheet(adata, 
+                      sample_sheet_path, 
+                      mapping_key_column='obs_names',
+                      as_category=True,
+                      uns_flag='sample_sheet_loaded',
+                      force_reload=True
+                        ):
+    """
+    Loads a sample sheet CSV and maps metadata into the AnnData object as categorical columns.
+
+    Parameters:
+        adata (AnnData): The AnnData object to append sample information to.
+        sample_sheet_path (str): Path to the CSV file.
+        mapping_key_column (str): Column name in the CSV to map against adata.obs_names or an existing obs column.
+        as_category (bool): If True, added columns will be cast as pandas Categorical.
+
+    Returns:
+        AnnData: Updated AnnData object.
+    """
+    import pandas as pd
+
+    # Only run if not already performed
+    already = bool(adata.uns.get(uns_flag, False))
+    if already and not force_reload:
+        # QC already performed; nothing to do
+        return
+
+    print('Loading sample sheet...')
+    df = pd.read_csv(sample_sheet_path)
+    df[mapping_key_column] = df[mapping_key_column].astype(str)
+    
+    # If matching against obs_names directly
+    if mapping_key_column == 'obs_names':
+        key_series = adata.obs_names.astype(str)
+    else:
+        key_series = adata.obs[mapping_key_column].astype(str)
+
+    value_columns = [col for col in df.columns if col != mapping_key_column]
+    
+    print(f'Appending metadata columns: {value_columns}')
+    df = df.set_index(mapping_key_column)
+
+    for col in value_columns:
+        mapped = key_series.map(df[col])
+        if as_category:
+            mapped = mapped.astype('category')
+        adata.obs[col] = mapped
+
+    # mark as done
+    adata.uns[uns_flag] = True
+
+    print('Sample sheet metadata successfully added as categories.' if as_category else 'Metadata added.')
+    return adata

smftools/preprocessing/make_dirs.py

@@ -0,0 +1,21 @@
+## make_dirs
+
+# General
+def make_dirs(directories):
+    """
+    Takes a list of file paths and makes new directories if the directory does not already exist.
+
+    Parameters:
+        directories (list): A list of directories to make
+    
+    Returns:
+        None
+    """
+    import os
+
+    for directory in directories:
+        if not os.path.isdir(directory):
+            os.mkdir(directory)
+            print(f"Directory '{directory}' created successfully.")
+        else:
+            print(f"Directory '{directory}' already exists.")

smftools/preprocessing/min_non_diagonal.py

@@ -0,0 +1,25 @@
+## min_non_diagonal
+
+def min_non_diagonal(matrix):
+    """
+    Takes a matrix and returns the smallest value from each row with the diagonal masked.
+
+    Parameters:
+        matrix (ndarray): A 2D ndarray.
+
+    Returns:
+        min_values (list): A list of minimum values from each row of the matrix
+    """
+    import numpy as np
+
+    n = matrix.shape[0]
+    min_values = []
+    for i in range(n):
+        # Mask to exclude the diagonal element
+        row_mask = np.ones(n, dtype=bool)
+        row_mask[i] = False
+        # Extract the row excluding the diagonal element
+        row = matrix[i, row_mask]
+        # Find the minimum value in the row
+        min_values.append(np.min(row))
+    return min_values

smftools/preprocessing/recipes.py

@@ -0,0 +1,127 @@
+# recipes
+
+def recipe_1_Kissiov_and_McKenna_2025(adata, sample_sheet_path, output_directory, mapping_key_column='Sample', reference_column = 'Reference', sample_names_col='Sample_names', invert=True):
+    """
+    The first part of the preprocessing workflow applied to the smf.inform.pod_to_adata() output derived from Kissiov_and_McKenna_2025.
+
+    Performs the following tasks:
+    1) Loads a sample CSV to append metadata mappings to the adata object.
+    2) Appends a boolean indicating whether each position in var_names is within a given reference.
+    3) Appends the cytosine context to each position from each reference.
+    4) Calculate read level methylation statistics.
+    5) Calculates read length statistics (start position, end position, read length).
+    6) Optionally inverts the adata to flip the position coordinate orientation.
+    7) Adds new layers containing NaN replaced variants of adata.X (fill_closest, nan0_0minus1, nan1_12).
+    8) Returns a dictionary to pass the variable namespace to the parent scope.
+
+    Parameters:
+        adata (AnnData): The AnnData object to use as input.
+        sample_sheet_path (str): String representing the path to the sample sheet csv containing the sample metadata.
+        output_directory (str): String representing the path to the output directory for plots.
+        mapping_key_column (str): The column name to use as the mapping keys for applying the sample sheet metadata.
+        reference_column (str): The name of the reference column to use.
+        sample_names_col (str): The name of the sample name column to use.
+        invert (bool): Whether to invert the positional coordinates of the adata object.
+
+    Returns:
+        variables (dict): A dictionary of variables to append to the parent scope.
+    """
+    import anndata as ad
+    import pandas as pd
+    import numpy as np
+    from .load_sample_sheet import load_sample_sheet
+    from .calculate_coverage import calculate_coverage
+    from .append_C_context import append_C_context
+    from .calculate_converted_read_methylation_stats import calculate_converted_read_methylation_stats
+    from .invert_adata import invert_adata
+    from .calculate_read_length_stats import calculate_read_length_stats
+    from .clean_NaN import clean_NaN
+
+    # Clean up some of the Reference metadata and save variable names that point to sets of values in the column.
+    adata.obs[reference_column] = adata.obs[reference_column].astype('category')
+    references = adata.obs[reference_column].cat.categories
+    split_references = [(reference, reference.split('_')[0][1:]) for reference in references]
+    reference_mapping = {k: v for k, v in split_references}
+    adata.obs[f'{reference_column}_short'] = adata.obs[reference_column].map(reference_mapping)
+    short_references = set(adata.obs[f'{reference_column}_short'])
+    binary_layers = list(adata.layers.keys())
+
+    # load sample sheet metadata
+    load_sample_sheet(adata, sample_sheet_path, mapping_key_column)
+
+    # hold sample names set
+    adata.obs[sample_names_col] = adata.obs[sample_names_col].astype('category')
+    sample_names = adata.obs[sample_names_col].cat.categories
+
+    # Add position level metadata
+    calculate_coverage(adata, obs_column=reference_column)
+    adata.var['SNP_position'] = (adata.var[f'N_{reference_column}_with_position'] > 0) & (adata.var[f'N_{reference_column}_with_position'] < len(references)).astype(bool)
+
+    # Append cytosine context to the reference positions based on the conversion strand.
+    append_C_context(adata, obs_column=reference_column, use_consensus=False)
+
+    # Calculate read level methylation statistics. Assess if GpC methylation level is above other_C methylation level as a QC.
+    calculate_converted_read_methylation_stats(adata, reference_column, sample_names_col)
+
+    # Calculate read length statistics
+    upper_bound, lower_bound = calculate_read_length_stats(adata, reference_column, sample_names_col)
+
+    # Invert the adata object (ie flip the strand orientation for visualization)
+    if invert:
+        adata = invert_adata(adata)
+    else:
+        pass
+
+    # NaN replacement strategies stored in additional layers. Having layer=None uses adata.X
+    clean_NaN(adata, layer=None)
+
+    variables = {
+        "short_references": short_references,
+        "binary_layers": binary_layers,
+        "sample_names": sample_names,
+        "upper_bound": upper_bound,
+        "lower_bound": lower_bound,
+        "references": references
+    }
+    return variables
+
+def recipe_2_Kissiov_and_McKenna_2025(adata, output_directory, binary_layers, distance_thresholds={}, reference_column = 'Reference', sample_names_col='Sample_names'):
+    """
+    The second part of the preprocessing workflow applied to the adata that has already been preprocessed by recipe_1_Kissiov_and_McKenna_2025.
+
+    Performs the following tasks:
+    1) Marks putative PCR duplicates using pairwise hamming distance metrics.
+    2) Performs a complexity analysis of the library based on the PCR duplicate detection rate.
+    3) Removes PCR duplicates from the adata.
+    4) Returns two adata object: one for the filtered adata and one for the duplicate adata.
+
+    Parameters:
+        adata (AnnData): The AnnData object to use as input.
+        output_directory (str): String representing the path to the output directory for plots.
+        binary_layers (list): A list of layers to used for the binary encoding of read sequences. Used for duplicate detection.
+        distance_thresholds (dict): A dictionary keyed by obs_column categories that points to a float corresponding to the distance threshold to apply. Default is an empty dict.
+        reference_column (str): The name of the reference column to use.
+        sample_names_col (str): The name of the sample name column to use.
+
+    Returns:
+        filtered_adata (AnnData): An AnnData object containing the filtered reads
+        duplicates (AnnData): An AnnData object containing the duplicate reads
+    """
+    import anndata as ad
+    import pandas as pd
+    import numpy as np
+    from .mark_duplicates import mark_duplicates
+    from .calculate_complexity import calculate_complexity
+    from .remove_duplicates import remove_duplicates
+
+    # Add here a way to remove reads below a given read quality (based on nan content). Need to also add a way to pull from BAM files the read quality from each read
+
+    # Duplicate detection using pairwise hamming distance across reads
+    mark_duplicates(adata, binary_layers, obs_column=reference_column, sample_col=sample_names_col, distance_thresholds=distance_thresholds, method='N_masked_distances')
+
+    # Complexity analysis using the marked duplicates and the lander-watermann algorithm
+    calculate_complexity(adata, output_directory, obs_column=reference_column, sample_col=sample_names_col, plot=True, save_plot=False)
+
+    # Remove duplicate reads and store the duplicate reads in a new AnnData object named duplicates.
+    filtered_adata, duplicates = remove_duplicates(adata)
+    return filtered_adata, duplicates

smftools/preprocessing/subsample_adata.py

@@ -0,0 +1,58 @@
+def subsample_adata(adata, obs_columns=None, max_samples=2000, random_seed=42):
+    """
+    Subsamples an AnnData object so that each unique combination of categories 
+    in the given `obs_columns` has at most `max_samples` observations.
+    If `obs_columns` is None or empty, the function randomly subsamples the entire dataset.
+    
+    Parameters:
+        adata (AnnData): The AnnData object to subsample.
+        obs_columns (list of str, optional): List of observation column names to group by.
+        max_samples (int): The maximum number of observations per category combination.
+        random_seed (int): Random seed for reproducibility.
+
+    Returns:
+        AnnData: A new AnnData object with subsampled observations.
+    """
+    import anndata as ad
+    import numpy as np
+
+    np.random.seed(random_seed)  # Ensure reproducibility
+
+    if not obs_columns:  # If no obs columns are given, sample globally
+        if adata.shape[0] > max_samples:
+            sampled_indices = np.random.choice(adata.obs.index, max_samples, replace=False)
+        else:
+            sampled_indices = adata.obs.index  # Keep all if fewer than max_samples
+        
+        return adata[sampled_indices].copy()
+
+    sampled_indices = []
+
+    # Get unique category combinations from all specified obs columns
+    unique_combinations = adata.obs[obs_columns].drop_duplicates()
+
+    for _, row in unique_combinations.iterrows():
+        # Build filter condition dynamically for multiple columns
+        condition = (adata.obs[obs_columns] == row.values).all(axis=1)
+        
+        # Get indices for the current category combination
+        subset_indices = adata.obs[condition].index.to_numpy()
+
+        # Subsample or take all
+        if len(subset_indices) > max_samples:
+            sampled = np.random.choice(subset_indices, max_samples, replace=False)
+        else:
+            sampled = subset_indices  # Keep all if fewer than max_samples
+
+        sampled_indices.extend(sampled)
+
+    # ⚠ Handle backed mode detection
+    if adata.isbacked:
+        print("⚠ Detected backed mode. Subset will be loaded fully into memory.")
+        subset = adata[sampled_indices]
+        subset = subset.to_memory()
+    else:
+        subset = adata[sampled_indices]
+
+    # Create a new AnnData object with only the selected indices
+    return subset[sampled_indices].copy()