risk-network 0.0.11__py3-none-any.whl → 0.0.12b0__py3-none-any.whl

risk/stats/permutation/permutation.py

@@ -1,237 +0,0 @@
-"""
-risk/stats/permutation/permutation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-"""
-
-from multiprocessing import get_context, Manager
-from multiprocessing.managers import ValueProxy
-from typing import Any, Callable, Dict, List, Tuple, Union
-
-import numpy as np
-from scipy.sparse import csr_matrix
-from threadpoolctl import threadpool_limits
-from tqdm import tqdm
-
-from risk.stats.permutation.test_functions import DISPATCH_TEST_FUNCTIONS
-
-
-def compute_permutation_test(
-    neighborhoods: csr_matrix,
-    annotations: csr_matrix,
-    score_metric: str = "sum",
-    null_distribution: str = "network",
-    num_permutations: int = 1000,
-    random_seed: int = 888,
-    max_workers: int = 1,
-) -> Dict[str, Any]:
-    """Compute permutation test for enrichment and depletion in neighborhoods.
-
-    Args:
-        neighborhoods (csr_matrix): Sparse binary matrix representing neighborhoods.
-        annotations (csr_matrix): Sparse binary matrix representing annotations.
-        score_metric (str, optional): Metric to use for scoring ('sum' or 'stdev'). Defaults to "sum".
-        null_distribution (str, optional): Type of null distribution ('network' or 'annotations'). Defaults to "network".
-        num_permutations (int, optional): Number of permutations to run. Defaults to 1000.
-        random_seed (int, optional): Seed for random number generation. Defaults to 888.
-        max_workers (int, optional): Number of workers for multiprocessing. Defaults to 1.
-
-    Returns:
-        Dict[str, Any]: Dictionary containing depletion and enrichment p-values.
-    """
-    # Ensure that the matrices are in the correct format and free of NaN values
-    # NOTE: Keep the data type as float32 to avoid locking issues with dot product operations
-    neighborhoods = neighborhoods.astype(np.float32)
-    annotations = annotations.astype(np.float32)
-    # Retrieve the appropriate neighborhood score function based on the metric
-    neighborhood_score_func = DISPATCH_TEST_FUNCTIONS[score_metric]
-
-    # Run the permutation test to calculate depletion and enrichment counts
-    counts_depletion, counts_enrichment = _run_permutation_test(
-        neighborhoods=neighborhoods,
-        annotations=annotations,
-        neighborhood_score_func=neighborhood_score_func,
-        null_distribution=null_distribution,
-        num_permutations=num_permutations,
-        random_seed=random_seed,
-        max_workers=max_workers,
-    )
-    # Compute p-values for depletion and enrichment
-    # If counts are 0, set p-value to 1/num_permutations to avoid zero p-values
-    depletion_pvals = np.maximum(counts_depletion, 1) / num_permutations
-    enrichment_pvals = np.maximum(counts_enrichment, 1) / num_permutations
-
-    return {
-        "depletion_pvals": depletion_pvals,
-        "enrichment_pvals": enrichment_pvals,
-    }
-
-
-def _run_permutation_test(
-    neighborhoods: csr_matrix,
-    annotations: csr_matrix,
-    neighborhood_score_func: Callable,
-    null_distribution: str = "network",
-    num_permutations: int = 1000,
-    random_seed: int = 888,
-    max_workers: int = 4,
-) -> tuple:
-    """Run the permutation test to calculate depletion and enrichment counts.
-
-    Args:
-        neighborhoods (csr_matrix): Sparse binary matrix representing neighborhoods.
-        annotations (csr_matrix): Sparse binary matrix representing annotations.
-        neighborhood_score_func (Callable): Function to calculate neighborhood scores.
-        null_distribution (str, optional): Type of null distribution ('network' or 'annotations'). Defaults to "network".
-        num_permutations (int, optional): Number of permutations. Defaults to 1000.
-        random_seed (int, optional): Seed for random number generation. Defaults to 888.
-        max_workers (int, optional): Number of workers for multiprocessing. Defaults to 4.
-
-    Returns:
-        tuple: Depletion and enrichment counts.
-    """
-    # Initialize the RNG for reproducibility
-    rng = np.random.default_rng(seed=random_seed)
-    # Determine the indices to use based on the null distribution type
-    if null_distribution == "network":
-        idxs = range(annotations.shape[0])
-    elif null_distribution == "annotations":
-        idxs = np.nonzero(annotations.getnnz(axis=1) > 0)[0]
-    else:
-        raise ValueError(
-            "Invalid null_distribution value. Choose either 'network' or 'annotations'."
-        )
-
-    # Replace NaNs with zeros in the sparse annotations matrix
-    annotations.data[np.isnan(annotations.data)] = 0
-    annotation_matrix_obsv = annotations[idxs]
-    neighborhoods_matrix_obsv = neighborhoods.T[idxs].T
-    # Calculate observed neighborhood scores
-    with np.errstate(invalid="ignore", divide="ignore"):
-        observed_neighborhood_scores = neighborhood_score_func(
-            neighborhoods_matrix_obsv, annotation_matrix_obsv
-        )
-
-    # Initialize count matrices for depletion and enrichment
-    counts_depletion = np.zeros(observed_neighborhood_scores.shape)
-    counts_enrichment = np.zeros(observed_neighborhood_scores.shape)
-    # Determine the number of permutations to run in each worker process
-    subset_size = num_permutations // max_workers
-    remainder = num_permutations % max_workers
-
-    # Use the spawn context for creating a new multiprocessing pool
-    ctx = get_context("spawn")
-    manager = Manager()
-    progress_counter = manager.Value("i", 0)
-    total_progress = num_permutations
-
-    # Generate precomputed permutations
-    permutations = [rng.permutation(idxs) for _ in range(num_permutations)]
-    # Divide permutations into batches for workers
-    batch_size = subset_size + (1 if remainder > 0 else 0)
-    permutation_batches = [
-        permutations[i * batch_size : (i + 1) * batch_size] for i in range(max_workers)
-    ]
-
-    # Execute the permutation test using multiprocessing
-    with ctx.Pool(max_workers) as pool:
-        with tqdm(total=total_progress, desc="Total progress", position=0) as progress:
-            # Prepare parameters for multiprocessing
-            params_list = [
-                (
-                    permutation_batches[i],  # Pass the batch of precomputed permutations
-                    annotations,
-                    neighborhoods_matrix_obsv,
-                    observed_neighborhood_scores,
-                    neighborhood_score_func,
-                    num_permutations,
-                    progress_counter,
-                    max_workers,
-                )
-                for i in range(max_workers)
-            ]
-
-            # Start the permutation process in parallel
-            results = pool.starmap_async(_permutation_process_batch, params_list, chunksize=1)
-
-            # Update progress bar based on progress_counter
-            while not results.ready():
-                progress.update(progress_counter.value - progress.n)
-                results.wait(0.1)  # Wait for 100ms
-            # Ensure progress bar reaches 100%
-            progress.update(total_progress - progress.n)
-
-    # Accumulate results from each worker
-    for local_counts_depletion, local_counts_enrichment in results.get():
-        counts_depletion = np.add(counts_depletion, local_counts_depletion)
-        counts_enrichment = np.add(counts_enrichment, local_counts_enrichment)
-
-    return counts_depletion, counts_enrichment
-
-
-def _permutation_process_batch(
-    permutations: Union[List, Tuple, np.ndarray],
-    annotation_matrix: csr_matrix,
-    neighborhoods_matrix_obsv: csr_matrix,
-    observed_neighborhood_scores: np.ndarray,
-    neighborhood_score_func: Callable,
-    num_permutations: int,
-    progress_counter: ValueProxy,
-    max_workers: int,
-) -> tuple:
-    """Process a batch of permutations in a worker process.
-
-    Args:
-        permutations (Union[List, Tuple, np.ndarray]): Permutation batch to process.
-        annotation_matrix (csr_matrix): Sparse binary matrix representing annotations.
-        neighborhoods_matrix_obsv (csr_matrix): Sparse binary matrix representing observed neighborhoods.
-        observed_neighborhood_scores (np.ndarray): Observed neighborhood scores.
-        neighborhood_score_func (Callable): Function to calculate neighborhood scores.
-        num_permutations (int): Number of total permutations across all subsets.
-        progress_counter (multiprocessing.managers.ValueProxy): Shared counter for tracking progress.
-        max_workers (int): Number of workers for multiprocessing.
-
-    Returns:
-        tuple: Local counts of depletion and enrichment.
-    """
-    # Initialize local count matrices for this worker
-    local_counts_depletion = np.zeros(observed_neighborhood_scores.shape)
-    local_counts_enrichment = np.zeros(observed_neighborhood_scores.shape)
-
-    # Limit the number of threads used by NumPy's BLAS implementation to 1 when more than one worker is used
-    # NOTE: This does not work for Mac M chips due to a bug in the threadpoolctl package
-    # This is currently a known issue and is being addressed by the maintainers [https://github.com/joblib/threadpoolctl/issues/135]
-    limits = None if max_workers == 1 else 1
-    with threadpool_limits(limits=limits, user_api="blas"):
-        # Initialize a local counter for batched progress updates
-        local_progress = 0
-        # Calculate the modulo value based on total permutations for 1/100th frequency updates
-        modulo_value = max(1, num_permutations // 100)
-
-        for permuted_idxs in permutations:
-            # Apply precomputed permutation
-            annotation_matrix_permut = annotation_matrix[permuted_idxs]
-            # Calculate permuted neighborhood scores
-            with np.errstate(invalid="ignore", divide="ignore"):
-                permuted_neighborhood_scores = neighborhood_score_func(
-                    neighborhoods_matrix_obsv, annotation_matrix_permut
-                )
-
-            # Update local depletion and enrichment counts
-            local_counts_depletion = np.add(
-                local_counts_depletion, permuted_neighborhood_scores <= observed_neighborhood_scores
-            )
-            local_counts_enrichment = np.add(
-                local_counts_enrichment,
-                permuted_neighborhood_scores >= observed_neighborhood_scores,
-            )
-
-            # Update progress
-            local_progress += 1
-            if local_progress % modulo_value == 0:
-                progress_counter.value += modulo_value
-
-        # Final progress update for any remaining iterations
-        if local_progress % modulo_value != 0:
-            progress_counter.value += modulo_value
-
-    return local_counts_depletion, local_counts_enrichment

risk/stats/permutation/test_functions.py

@@ -1,70 +0,0 @@
-"""
-risk/stats/permutation/test_functions
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-"""
-
-import numpy as np
-from scipy.sparse import csr_matrix
-
-# NOTE: Cython optimizations provided minimal performance benefits.
-# The final version with Cython is archived in the `cython_permutation` branch.
-
-# DISPATCH_TEST_FUNCTIONS can be found at the end of the file.
-
-
-def compute_neighborhood_score_by_sum(
-    neighborhoods_matrix: csr_matrix, annotation_matrix: csr_matrix
-) -> np.ndarray:
-    """Compute the sum of attribute values for each neighborhood using sparse matrices.
-
-    Args:
-        neighborhoods_matrix (csr_matrix): Sparse binary matrix representing neighborhoods.
-        annotation_matrix (csr_matrix): Sparse matrix representing annotation values.
-
-    Returns:
-        np.ndarray: Dense array of summed attribute values for each neighborhood.
-    """
-    # Calculate the neighborhood score as the dot product of neighborhoods and annotations
-    neighborhood_score = neighborhoods_matrix @ annotation_matrix  # Sparse matrix multiplication
-    # Convert the result to a dense array for downstream calculations
-    neighborhood_score_dense = neighborhood_score.toarray()
-    return neighborhood_score_dense
-
-
-def compute_neighborhood_score_by_stdev(
-    neighborhoods_matrix: csr_matrix, annotation_matrix: csr_matrix
-) -> np.ndarray:
-    """Compute the standard deviation of neighborhood scores for sparse matrices.
-
-    Args:
-        neighborhoods_matrix (csr_matrix): Sparse binary matrix representing neighborhoods.
-        annotation_matrix (csr_matrix): Sparse matrix representing annotation values.
-
-    Returns:
-        np.ndarray: Standard deviation of the neighborhood scores.
-    """
-    # Calculate the neighborhood score as the dot product of neighborhoods and annotations
-    neighborhood_score = neighborhoods_matrix @ annotation_matrix  # Sparse matrix multiplication
-    # Calculate the number of elements in each neighborhood (sum of rows)
-    N = neighborhoods_matrix.sum(axis=1).A.flatten()  # Convert to 1D array
-    # Avoid division by zero by replacing zeros in N with np.nan temporarily
-    N[N == 0] = np.nan
-    # Compute the mean of the neighborhood scores
-    M = neighborhood_score.multiply(1 / N[:, None]).toarray()  # Sparse element-wise division
-    # Compute the mean of squares (EXX) directly using squared annotation matrix
-    annotation_squared = annotation_matrix.multiply(annotation_matrix)  # Element-wise squaring
-    EXX = (neighborhoods_matrix @ annotation_squared).multiply(1 / N[:, None]).toarray()
-    # Calculate variance as EXX - M^2
-    variance = EXX - np.power(M, 2)
-    # Compute the standard deviation as the square root of the variance
-    neighborhood_stdev = np.sqrt(variance)
-    # Replace np.nan back with zeros in case N was 0 (no elements in the neighborhood)
-    neighborhood_stdev[np.isnan(neighborhood_stdev)] = 0
-    return neighborhood_stdev
-
-
-# Dictionary to dispatch statistical test functions based on the score metric
-DISPATCH_TEST_FUNCTIONS = {
-    "sum": compute_neighborhood_score_by_sum,
-    "stdev": compute_neighborhood_score_by_stdev,
-}

risk/stats/significance.py

@@ -1,166 +0,0 @@
-"""
-risk/stats/significance
-~~~~~~~~~~~~~~~~~~~~~~~
-"""
-
-from typing import Any, Dict, Union
-
-import numpy as np
-from statsmodels.stats.multitest import fdrcorrection
-
-
-def calculate_significance_matrices(
-    depletion_pvals: np.ndarray,
-    enrichment_pvals: np.ndarray,
-    tail: str = "right",
-    pval_cutoff: float = 0.05,
-    fdr_cutoff: float = 0.05,
-) -> Dict[str, Any]:
-    """Calculate significance matrices based on p-values and specified tail.
-
-    Args:
-        depletion_pvals (np.ndarray): Matrix of depletion p-values.
-        enrichment_pvals (np.ndarray): Matrix of enrichment p-values.
-        tail (str, optional): The tail type for significance selection ('left', 'right', 'both'). Defaults to 'right'.
-        pval_cutoff (float, optional): Cutoff for p-value significance. Defaults to 0.05.
-        fdr_cutoff (float, optional): Cutoff for FDR significance if applied. Defaults to 0.05.
-
-    Returns:
-        Dict[str, Any]: Dictionary containing the enrichment matrix, binary significance matrix,
-            and the matrix of significant enrichment values.
-    """
-    if fdr_cutoff < 1.0:
-        # Apply FDR correction to depletion p-values
-        depletion_qvals = np.apply_along_axis(fdrcorrection, 1, depletion_pvals)[:, 1, :]
-        depletion_alpha_threshold_matrix = _compute_threshold_matrix(
-            depletion_pvals, depletion_qvals, pval_cutoff=pval_cutoff, fdr_cutoff=fdr_cutoff
-        )
-        # Compute the depletion matrix using both q-values and p-values
-        depletion_matrix = (depletion_qvals**2) * (depletion_pvals**0.5)
-
-        # Apply FDR correction to enrichment p-values
-        enrichment_qvals = np.apply_along_axis(fdrcorrection, 1, enrichment_pvals)[:, 1, :]
-        enrichment_alpha_threshold_matrix = _compute_threshold_matrix(
-            enrichment_pvals, enrichment_qvals, pval_cutoff=pval_cutoff, fdr_cutoff=fdr_cutoff
-        )
-        # Compute the enrichment matrix using both q-values and p-values
-        enrichment_matrix = (enrichment_pvals**0.5) * (enrichment_qvals**2)
-    else:
-        # Compute threshold matrices based on p-value cutoffs only
-        depletion_alpha_threshold_matrix = _compute_threshold_matrix(
-            depletion_pvals, pval_cutoff=pval_cutoff
-        )
-        depletion_matrix = depletion_pvals
-
-        enrichment_alpha_threshold_matrix = _compute_threshold_matrix(
-            enrichment_pvals, pval_cutoff=pval_cutoff
-        )
-        enrichment_matrix = enrichment_pvals
-
-    # Apply a negative log10 transformation for visualization purposes
-    log_depletion_matrix = -np.log10(depletion_matrix)
-    log_enrichment_matrix = -np.log10(enrichment_matrix)
-
-    # Select the appropriate significance matrices based on the specified tail
-    significance_matrix, significant_binary_significance_matrix = _select_significance_matrices(
-        tail,
-        log_depletion_matrix,
-        depletion_alpha_threshold_matrix,
-        log_enrichment_matrix,
-        enrichment_alpha_threshold_matrix,
-    )
-
-    # Filter the enrichment matrix using the binary significance matrix
-    significant_significance_matrix = np.where(
-        significant_binary_significance_matrix == 1, significance_matrix, 0
-    )
-
-    return {
-        "significance_matrix": significance_matrix,
-        "significant_significance_matrix": significant_significance_matrix,
-        "significant_binary_significance_matrix": significant_binary_significance_matrix,
-    }
-
-
-def _select_significance_matrices(
-    tail: str,
-    log_depletion_matrix: np.ndarray,
-    depletion_alpha_threshold_matrix: np.ndarray,
-    log_enrichment_matrix: np.ndarray,
-    enrichment_alpha_threshold_matrix: np.ndarray,
-) -> tuple:
-    """Select significance matrices based on the specified tail type.
-
-    Args:
-        tail (str): The tail type for significance selection. Options are 'left', 'right', or 'both'.
-        log_depletion_matrix (np.ndarray): Matrix of log-transformed depletion values.
-        depletion_alpha_threshold_matrix (np.ndarray): Alpha threshold matrix for depletion significance.
-        log_enrichment_matrix (np.ndarray): Matrix of log-transformed enrichment values.
-        enrichment_alpha_threshold_matrix (np.ndarray): Alpha threshold matrix for enrichment significance.
-
-    Returns:
-        tuple: A tuple containing the selected enrichment matrix and binary significance matrix.
-
-    Raises:
-        ValueError: If the provided tail type is not 'left', 'right', or 'both'.
-    """
-    if tail not in {"left", "right", "both"}:
-        raise ValueError("Invalid value for 'tail'. Must be 'left', 'right', or 'both'.")
-
-    if tail == "left":
-        # Select depletion matrix and corresponding alpha threshold for left-tail analysis
-        significance_matrix = -log_depletion_matrix
-        alpha_threshold_matrix = depletion_alpha_threshold_matrix
-    elif tail == "right":
-        # Select enrichment matrix and corresponding alpha threshold for right-tail analysis
-        significance_matrix = log_enrichment_matrix
-        alpha_threshold_matrix = enrichment_alpha_threshold_matrix
-    elif tail == "both":
-        # Select the matrix with the highest absolute values while preserving the sign
-        significance_matrix = np.where(
-            np.abs(log_depletion_matrix) >= np.abs(log_enrichment_matrix),
-            -log_depletion_matrix,
-            log_enrichment_matrix,
-        )
-        # Combine alpha thresholds using a logical OR operation
-        alpha_threshold_matrix = np.logical_or(
-            depletion_alpha_threshold_matrix, enrichment_alpha_threshold_matrix
-        )
-    else:
-        raise ValueError("Invalid value for 'tail'. Must be 'left', 'right', or 'both'.")
-
-    # Create a binary significance matrix where valid indices meet the alpha threshold
-    valid_idxs = ~np.isnan(alpha_threshold_matrix)
-    significant_binary_significance_matrix = np.zeros(alpha_threshold_matrix.shape)
-    significant_binary_significance_matrix[valid_idxs] = alpha_threshold_matrix[valid_idxs]
-
-    return significance_matrix, significant_binary_significance_matrix
-
-
-def _compute_threshold_matrix(
-    pvals: np.ndarray,
-    fdr_pvals: Union[np.ndarray, None] = None,
-    pval_cutoff: float = 0.05,
-    fdr_cutoff: float = 0.05,
-) -> np.ndarray:
-    """Compute a threshold matrix indicating significance based on p-value and FDR cutoffs.
-
-    Args:
-        pvals (np.ndarray): Array of p-values for statistical tests.
-        fdr_pvals (np.ndarray, optional): Array of FDR-corrected p-values corresponding to the p-values. Defaults to None.
-        pval_cutoff (float, optional): Cutoff for p-value significance. Defaults to 0.05.
-        fdr_cutoff (float, optional): Cutoff for FDR significance. Defaults to 0.05.
-
-    Returns:
-        np.ndarray: A threshold matrix where 1 indicates significance based on the provided cutoffs, 0 otherwise.
-    """
-    if fdr_pvals is not None:
-        # Compute the threshold matrix based on both p-value and FDR cutoffs
-        pval_below_cutoff = pvals <= pval_cutoff
-        fdr_below_cutoff = fdr_pvals <= fdr_cutoff
-        threshold_matrix = np.logical_and(pval_below_cutoff, fdr_below_cutoff).astype(int)
-    else:
-        # Compute the threshold matrix based only on p-value cutoff
-        threshold_matrix = (pvals <= pval_cutoff).astype(int)
-
-    return threshold_matrix