risk-network 0.0.6b9__py3-none-any.whl → 0.0.7__py3-none-any.whl

risk/risk.py

@@ -10,7 +10,7 @@ import numpy as np
 import pandas as pd
 
 from risk.annotations import AnnotationsIO, define_top_annotations
-from risk.log import params, print_header
+from risk.log import params, logger, log_header, set_global_verbosity
 from risk.neighborhoods import (
     define_domains,
     get_network_neighborhoods,
@@ -20,9 +20,9 @@ from risk.neighborhoods import (
 from risk.network import NetworkIO, NetworkGraph, NetworkPlotter
 from risk.stats import (
     calculate_significance_matrices,
-    compute_fisher_exact_test,
     compute_hypergeom_test,
     compute_permutation_test,
+    compute_poisson_test,
 )
 
 
@@ -33,60 +33,60 @@ class RISK(NetworkIO, AnnotationsIO):
     and performing network-based statistical analysis, such as neighborhood significance testing.
     """
 
-    def __init__(self, *args, **kwargs):
-        """Initialize the RISK class with configuration settings."""
+    def __init__(self, verbose: bool = True):
+        """Initialize the RISK class with configuration settings.
+
+        Args:
+            verbose (bool): If False, suppresses all log messages to the console. Defaults to True.
+        """
+        # Set global verbosity for logging
+        set_global_verbosity(verbose)
         # Initialize and log network parameters
         params.initialize()
-        # Initialize the parent classes
-        super().__init__(*args, **kwargs)
+        super().__init__()
 
     @property
-    def params(self):
-        """Access the logged parameters."""
+    def params(self) -> params:
+        """Access the logged network parameters.
+
+        Returns:
+            Params: An instance of the Params class with logged parameters and methods to access or update them.
+        """
         return params
 
-    def load_neighborhoods_by_permutation(
+    def load_neighborhoods_by_hypergeom(
         self,
         network: nx.Graph,
         annotations: Dict[str, Any],
-        distance_metric: str = "dijkstra",
+        distance_metric: str = "louvain",
         louvain_resolution: float = 0.1,
         edge_length_threshold: float = 0.5,
-        score_metric: str = "sum",
         null_distribution: str = "network",
-        num_permutations: int = 1000,
         random_seed: int = 888,
-        max_workers: int = 1,
     ) -> Dict[str, Any]:
-        """Load significant neighborhoods for the network using the permutation test.
+        """Load significant neighborhoods for the network using the hypergeometric test.
 
         Args:
             network (nx.Graph): The network graph.
-            annotations (pd.DataFrame): The matrix of annotations associated with the network.
-            distance_metric (str, optional): Distance metric for neighborhood analysis. Defaults to "dijkstra".
+            annotations (dict): The annotations associated with the network.
+            distance_metric (str, optional): Distance metric for neighborhood analysis. Defaults to "louvain".
             louvain_resolution (float, optional): Resolution parameter for Louvain clustering. Defaults to 0.1.
             edge_length_threshold (float, optional): Edge length threshold for neighborhood analysis. Defaults to 0.5.
-            score_metric (str, optional): Scoring metric for neighborhood significance. Defaults to "sum".
-            null_distribution (str, optional): Distribution used for permutation tests. Defaults to "network".
-            num_permutations (int, optional): Number of permutations for significance testing. Defaults to 1000.
+            null_distribution (str, optional): Type of null distribution ('network' or 'annotations'). Defaults to "network".
             random_seed (int, optional): Seed for random number generation. Defaults to 888.
-            max_workers (int, optional): Maximum number of workers for parallel computation. Defaults to 1.
 
         Returns:
             dict: Computed significance of neighborhoods.
         """
-        print_header("Running permutation test")
+        log_header("Running hypergeometric test")
         # Log neighborhood analysis parameters
         params.log_neighborhoods(
             distance_metric=distance_metric,
             louvain_resolution=louvain_resolution,
             edge_length_threshold=edge_length_threshold,
-            statistical_test_function="permutation",
-            score_metric=score_metric,
+            statistical_test_function="hypergeom",
             null_distribution=null_distribution,
-            num_permutations=num_permutations,
             random_seed=random_seed,
-            max_workers=max_workers,
         )
 
         # Load neighborhoods based on the network and distance metric
@@ -97,59 +97,49 @@ class RISK(NetworkIO, AnnotationsIO):
             edge_length_threshold=edge_length_threshold,
             random_seed=random_seed,
         )
-
-        # Log and display permutation test settings
-        print(f"Neighborhood scoring metric: '{score_metric}'")
-        print(f"Null distribution: '{null_distribution}'")
-        print(f"Number of permutations: {num_permutations}")
-        print(f"Maximum workers: {max_workers}")
-        # Run permutation test to compute neighborhood significance
-        neighborhood_significance = compute_permutation_test(
+        # Run hypergeometric test to compute neighborhood significance
+        neighborhood_significance = compute_hypergeom_test(
             neighborhoods=neighborhoods,
             annotations=annotations["matrix"],
-            score_metric=score_metric,
             null_distribution=null_distribution,
-            num_permutations=num_permutations,
-            random_seed=random_seed,
-            max_workers=max_workers,
         )
 
         # Return the computed neighborhood significance
         return neighborhood_significance
 
-    def load_neighborhoods_by_fisher_exact(
+    def load_neighborhoods_by_poisson(
         self,
         network: nx.Graph,
         annotations: Dict[str, Any],
-        distance_metric: str = "dijkstra",
+        distance_metric: str = "louvain",
         louvain_resolution: float = 0.1,
         edge_length_threshold: float = 0.5,
+        null_distribution: str = "network",
         random_seed: int = 888,
-        max_workers: int = 1,
     ) -> Dict[str, Any]:
-        """Load significant neighborhoods for the network using the Fisher's exact test.
+        """Load significant neighborhoods for the network using the Poisson test.
 
         Args:
             network (nx.Graph): The network graph.
-            annotations (pd.DataFrame): The matrix of annotations associated with the network.
-            distance_metric (str, optional): Distance metric for neighborhood analysis. Defaults to "dijkstra".
+            annotations (dict): The annotations associated with the network.
+            distance_metric (str, optional): Distance metric for neighborhood analysis. Defaults to "louvain".
             louvain_resolution (float, optional): Resolution parameter for Louvain clustering. Defaults to 0.1.
             edge_length_threshold (float, optional): Edge length threshold for neighborhood analysis. Defaults to 0.5.
+            null_distribution (str, optional): Type of null distribution ('network' or 'annotations'). Defaults to "network".
             random_seed (int, optional): Seed for random number generation. Defaults to 888.
-            max_workers (int, optional): Maximum number of workers for parallel computation. Defaults to 1.
 
         Returns:
             dict: Computed significance of neighborhoods.
         """
-        print_header("Running Fisher's exact test")
+        log_header("Running Poisson test")
         # Log neighborhood analysis parameters
         params.log_neighborhoods(
             distance_metric=distance_metric,
             louvain_resolution=louvain_resolution,
             edge_length_threshold=edge_length_threshold,
-            statistical_test_function="fisher_exact",
+            statistical_test_function="poisson",
+            null_distribution=null_distribution,
             random_seed=random_seed,
-            max_workers=max_workers,
         )
 
         # Load neighborhoods based on the network and distance metric
@@ -160,50 +150,56 @@ class RISK(NetworkIO, AnnotationsIO):
             edge_length_threshold=edge_length_threshold,
             random_seed=random_seed,
         )
-
-        # Log and display Fisher's exact test settings
-        print(f"Maximum workers: {max_workers}")
-        # Run Fisher's exact test to compute neighborhood significance
-        neighborhood_significance = compute_fisher_exact_test(
+        # Run Poisson test to compute neighborhood significance
+        neighborhood_significance = compute_poisson_test(
             neighborhoods=neighborhoods,
             annotations=annotations["matrix"],
-            max_workers=max_workers,
+            null_distribution=null_distribution,
         )
 
         # Return the computed neighborhood significance
         return neighborhood_significance
 
-    def load_neighborhoods_by_hypergeom(
+    def load_neighborhoods_by_permutation(
         self,
         network: nx.Graph,
         annotations: Dict[str, Any],
-        distance_metric: str = "dijkstra",
+        distance_metric: str = "louvain",
         louvain_resolution: float = 0.1,
         edge_length_threshold: float = 0.5,
+        score_metric: str = "sum",
+        null_distribution: str = "network",
+        num_permutations: int = 1000,
         random_seed: int = 888,
         max_workers: int = 1,
     ) -> Dict[str, Any]:
-        """Load significant neighborhoods for the network using the hypergeometric test.
+        """Load significant neighborhoods for the network using the permutation test.
 
         Args:
             network (nx.Graph): The network graph.
-            annotations (pd.DataFrame): The matrix of annotations associated with the network.
-            distance_metric (str, optional): Distance metric for neighborhood analysis. Defaults to "dijkstra".
+            annotations (dict): The annotations associated with the network.
+            distance_metric (str, optional): Distance metric for neighborhood analysis. Defaults to "louvain".
             louvain_resolution (float, optional): Resolution parameter for Louvain clustering. Defaults to 0.1.
             edge_length_threshold (float, optional): Edge length threshold for neighborhood analysis. Defaults to 0.5.
+            score_metric (str, optional): Scoring metric for neighborhood significance. Defaults to "sum".
+            null_distribution (str, optional): Type of null distribution ('network' or 'annotations'). Defaults to "network".
+            num_permutations (int, optional): Number of permutations for significance testing. Defaults to 1000.
             random_seed (int, optional): Seed for random number generation. Defaults to 888.
             max_workers (int, optional): Maximum number of workers for parallel computation. Defaults to 1.
 
         Returns:
             dict: Computed significance of neighborhoods.
         """
-        print_header("Running hypergeometric test")
+        log_header("Running permutation test")
         # Log neighborhood analysis parameters
         params.log_neighborhoods(
             distance_metric=distance_metric,
             louvain_resolution=louvain_resolution,
             edge_length_threshold=edge_length_threshold,
-            statistical_test_function="hypergeom",
+            statistical_test_function="permutation",
+            score_metric=score_metric,
+            null_distribution=null_distribution,
+            num_permutations=num_permutations,
             random_seed=random_seed,
             max_workers=max_workers,
         )
@@ -217,12 +213,19 @@ class RISK(NetworkIO, AnnotationsIO):
             random_seed=random_seed,
         )
 
-        # Log and display hypergeometric test settings
-        print(f"Maximum workers: {max_workers}")
-        # Run hypergeometric test to compute neighborhood significance
-        neighborhood_significance = compute_hypergeom_test(
+        # Log and display permutation test settings
+        logger.debug(f"Neighborhood scoring metric: '{score_metric}'")
+        logger.debug(f"Null distribution: '{null_distribution}'")
+        logger.debug(f"Number of permutations: {num_permutations}")
+        logger.debug(f"Maximum workers: {max_workers}")
+        # Run permutation test to compute neighborhood significance
+        neighborhood_significance = compute_permutation_test(
             neighborhoods=neighborhoods,
             annotations=annotations["matrix"],
+            score_metric=score_metric,
+            null_distribution=null_distribution,
+            num_permutations=num_permutations,
+            random_seed=random_seed,
             max_workers=max_workers,
         )
 
@@ -237,7 +240,7 @@ class RISK(NetworkIO, AnnotationsIO):
         tail: str = "right",  # OPTIONS: "right" (enrichment), "left" (depletion), "both"
         pval_cutoff: float = 0.01,  # OPTIONS: Any value between 0 to 1
         fdr_cutoff: float = 0.9999,  # OPTIONS: Any value between 0 to 1
-        impute_depth: int = 1,
+        impute_depth: int = 0,
         prune_threshold: float = 0.0,
         linkage_criterion: str = "distance",
         linkage_method: str = "average",
@@ -254,7 +257,7 @@ class RISK(NetworkIO, AnnotationsIO):
             tail (str, optional): Type of significance tail ("right", "left", "both"). Defaults to "right".
             pval_cutoff (float, optional): p-value cutoff for significance. Defaults to 0.01.
             fdr_cutoff (float, optional): FDR cutoff for significance. Defaults to 0.9999.
-            impute_depth (int, optional): Depth for imputing neighbors. Defaults to 1.
+            impute_depth (int, optional): Depth for imputing neighbors. Defaults to 0.
             prune_threshold (float, optional): Distance threshold for pruning neighbors. Defaults to 0.0.
             linkage_criterion (str, optional): Clustering criterion for defining domains. Defaults to "distance".
             linkage_method (str, optional): Clustering method to use. Defaults to "average".
@@ -266,7 +269,7 @@ class RISK(NetworkIO, AnnotationsIO):
             NetworkGraph: A fully initialized and processed NetworkGraph object.
         """
         # Log the parameters and display headers
-        print_header("Finding significant neighborhoods")
+        log_header("Finding significant neighborhoods")
         params.log_graph(
             tail=tail,
             pval_cutoff=pval_cutoff,
@@ -280,9 +283,9 @@ class RISK(NetworkIO, AnnotationsIO):
             max_cluster_size=max_cluster_size,
         )
 
-        print(f"p-value cutoff: {pval_cutoff}")
-        print(f"FDR BH cutoff: {fdr_cutoff}")
-        print(
+        logger.debug(f"p-value cutoff: {pval_cutoff}")
+        logger.debug(f"FDR BH cutoff: {fdr_cutoff}")
+        logger.debug(
             f"Significance tail: '{tail}' ({'enrichment' if tail == 'right' else 'depletion' if tail == 'left' else 'both'})"
         )
         # Calculate significant neighborhoods based on the provided parameters
@@ -294,7 +297,7 @@ class RISK(NetworkIO, AnnotationsIO):
             fdr_cutoff=fdr_cutoff,
         )
 
-        print_header("Processing neighborhoods")
+        log_header("Processing neighborhoods")
         # Process neighborhoods by imputing and pruning based on the given settings
         processed_neighborhoods = process_neighborhoods(
             network=network,
@@ -303,9 +306,9 @@ class RISK(NetworkIO, AnnotationsIO):
             prune_threshold=prune_threshold,
         )
 
-        print_header("Finding top annotations")
-        print(f"Min cluster size: {min_cluster_size}")
-        print(f"Max cluster size: {max_cluster_size}")
+        log_header("Finding top annotations")
+        logger.debug(f"Min cluster size: {min_cluster_size}")
+        logger.debug(f"Max cluster size: {max_cluster_size}")
         # Define top annotations based on processed neighborhoods
         top_annotations = self._define_top_annotations(
             network=network,
@@ -315,7 +318,7 @@ class RISK(NetworkIO, AnnotationsIO):
             max_cluster_size=max_cluster_size,
         )
 
-        print_header(f"Optimizing distance threshold for domains")
+        log_header("Optimizing distance threshold for domains")
         # Define domains in the network using the specified clustering settings
         domains = self._define_domains(
             neighborhoods=processed_neighborhoods,
@@ -363,7 +366,7 @@ class RISK(NetworkIO, AnnotationsIO):
         Returns:
             NetworkPlotter: A NetworkPlotter object configured with the given parameters.
         """
-        print_header("Loading plotter")
+        log_header("Loading plotter")
         # Log the plotter settings
         params.log_plotter(
             figsize=figsize,
@@ -380,7 +383,7 @@ class RISK(NetworkIO, AnnotationsIO):
     def _load_neighborhoods(
         self,
         network: nx.Graph,
-        distance_metric: str = "dijkstra",
+        distance_metric: str = "louvain",
         louvain_resolution: float = 0.1,
         edge_length_threshold: float = 0.5,
         random_seed: int = 888,
@@ -390,7 +393,7 @@ class RISK(NetworkIO, AnnotationsIO):
         Args:
             network (nx.Graph): The network graph.
             annotations (pd.DataFrame): The matrix of annotations associated with the network.
-            distance_metric (str, optional): Distance metric for neighborhood analysis. Defaults to "dijkstra".
+            distance_metric (str, optional): Distance metric for neighborhood analysis. Defaults to "louvain".
             louvain_resolution (float, optional): Resolution parameter for Louvain clustering. Defaults to 0.1.
             edge_length_threshold (float, optional): Edge length threshold for neighborhood analysis. Defaults to 0.5.
             random_seed (int, optional): Seed for random number generation. Defaults to 888.
@@ -404,9 +407,9 @@ class RISK(NetworkIO, AnnotationsIO):
         else:
             for_print_distance_metric = distance_metric
         # Log and display neighborhood settings
-        print(f"Distance metric: '{for_print_distance_metric}'")
-        print(f"Edge length threshold: {edge_length_threshold}")
-        print(f"Random seed: {random_seed}")
+        logger.debug(f"Distance metric: '{for_print_distance_metric}'")
+        logger.debug(f"Edge length threshold: {edge_length_threshold}")
+        logger.debug(f"Random seed: {random_seed}")
 
         # Compute neighborhoods based on the network and distance metric
         neighborhoods = get_network_neighborhoods(

risk/stats/__init__.py

@@ -3,7 +3,7 @@ risk/stats
 ~~~~~~~~~~
 """
 
-from .stats import calculate_significance_matrices
-from .fisher_exact import compute_fisher_exact_test
 from .hypergeom import compute_hypergeom_test
 from .permutation import compute_permutation_test
+from .poisson import compute_poisson_test
+from .stats import calculate_significance_matrices

risk/stats/hypergeom.py

@@ -3,8 +3,6 @@ risk/stats/hypergeom
 ~~~~~~~~~~~~~~~~~~~~
 """
 
-from multiprocessing import get_context, Manager
-from tqdm import tqdm
 from typing import Any, Dict
 
 import numpy as np
@@ -12,120 +10,45 @@ from scipy.stats import hypergeom
 
 
 def compute_hypergeom_test(
-    neighborhoods: np.ndarray,
-    annotations: np.ndarray,
-    max_workers: int = 4,
+    neighborhoods: np.ndarray, annotations: np.ndarray, null_distribution: str = "network"
 ) -> Dict[str, Any]:
-    """Compute hypergeometric test for enrichment and depletion in neighborhoods.
+    """Compute hypergeometric test for enrichment and depletion in neighborhoods with selectable null distribution.
 
     Args:
         neighborhoods (np.ndarray): Binary matrix representing neighborhoods.
         annotations (np.ndarray): Binary matrix representing annotations.
-        max_workers (int, optional): Number of workers for multiprocessing. Defaults to 4.
+        null_distribution (str, optional): Type of null distribution ('network' or 'annotations'). Defaults to "network".
 
     Returns:
         dict: Dictionary containing depletion and enrichment p-values.
     """
-    # Ensure that the matrices are binary (boolean) and free of NaN values
-    neighborhoods = neighborhoods.astype(bool)  # Convert to boolean
-    annotations = annotations.astype(bool)  # Convert to boolean
-
-    # Initialize the process of calculating p-values using multiprocessing
-    ctx = get_context("spawn")
-    manager = Manager()
-    progress_counter = manager.Value("i", 0)
-    total_tasks = neighborhoods.shape[1] * annotations.shape[1]
-
-    # Calculate the workload per worker
-    chunk_size = total_tasks // max_workers
-    remainder = total_tasks % max_workers
-
-    # Execute the hypergeometric test using multiprocessing
-    with ctx.Pool(max_workers) as pool:
-        with tqdm(total=total_tasks, desc="Total progress", position=0) as progress:
-            params_list = []
-            start_idx = 0
-            for i in range(max_workers):
-                end_idx = start_idx + chunk_size + (1 if i < remainder else 0)
-                params_list.append(
-                    (neighborhoods, annotations, start_idx, end_idx, progress_counter)
-                )
-                start_idx = end_idx
-
-            # Start the hypergeometric test process in parallel
-            results = pool.starmap_async(_hypergeom_process_subset, params_list, chunksize=1)
-
-            # Update progress bar based on progress_counter
-            while not results.ready():
-                progress.update(progress_counter.value - progress.n)
-                results.wait(0.05)  # Wait for 50ms
-            # Ensure progress bar reaches 100%
-            progress.update(total_tasks - progress.n)
-
-            # Accumulate results from each worker
-            depletion_pvals, enrichment_pvals = [], []
-            for dp, ep in results.get():
-                depletion_pvals.extend(dp)
-                enrichment_pvals.extend(ep)
-
-    # Reshape the results back into arrays with the appropriate dimensions
-    depletion_pvals = np.array(depletion_pvals).reshape(
-        neighborhoods.shape[1], annotations.shape[1]
+    # Get the total number of nodes in the network
+    total_node_count = neighborhoods.shape[0]
+
+    if null_distribution == "network":
+        # Case 1: Use all nodes as the background
+        background_population = total_node_count
+        neighborhood_sums = np.sum(neighborhoods, axis=0, keepdims=True).T
+        annotation_sums = np.sum(annotations, axis=0, keepdims=True)
+    elif null_distribution == "annotations":
+        # Case 2: Only consider nodes with at least one annotation
+        annotated_nodes = np.sum(annotations, axis=1) > 0
+        background_population = np.sum(annotated_nodes)
+        neighborhood_sums = np.sum(neighborhoods[annotated_nodes], axis=0, keepdims=True).T
+        annotation_sums = np.sum(annotations[annotated_nodes], axis=0, keepdims=True)
+    else:
+        raise ValueError(
+            "Invalid null_distribution value. Choose either 'network' or 'annotations'."
+        )
+
+    # Matrix multiplication for annotated nodes in each neighborhood
+    annotated_in_neighborhood = neighborhoods.T @ annotations
+    # Calculate depletion and enrichment p-values using the hypergeometric distribution
+    depletion_pvals = hypergeom.cdf(
+        annotated_in_neighborhood, background_population, annotation_sums, neighborhood_sums
     )
-    enrichment_pvals = np.array(enrichment_pvals).reshape(
-        neighborhoods.shape[1], annotations.shape[1]
+    enrichment_pvals = hypergeom.sf(
+        annotated_in_neighborhood - 1, background_population, annotation_sums, neighborhood_sums
     )
 
-    return {
-        "depletion_pvals": depletion_pvals,
-        "enrichment_pvals": enrichment_pvals,
-    }
-
-
-def _hypergeom_process_subset(
-    neighborhoods: np.ndarray,
-    annotations: np.ndarray,
-    start_idx: int,
-    end_idx: int,
-    progress_counter,
-) -> tuple:
-    """Process a subset of neighborhoods using the hypergeometric test.
-
-    Args:
-        neighborhoods (np.ndarray): The full neighborhood matrix.
-        annotations (np.ndarray): The annotation matrix.
-        start_idx (int): Starting index of the neighborhood-annotation pairs to process.
-        end_idx (int): Ending index of the neighborhood-annotation pairs to process.
-        progress_counter: Shared counter for tracking progress.
-
-    Returns:
-        tuple: Local p-values for depletion and enrichment.
-    """
-    # Initialize lists to store p-values for depletion and enrichment
-    depletion_pvals = []
-    enrichment_pvals = []
-    # Process the subset of tasks assigned to this worker
-    for idx in range(start_idx, end_idx):
-        i = idx // annotations.shape[1]  # Neighborhood index
-        j = idx % annotations.shape[1]  # Annotation index
-
-        neighborhood = neighborhoods[:, i]
-        annotation = annotations[:, j]
-
-        # Calculate the required values for the hypergeometric test
-        M = annotations.shape[0]  # Total number of items (population size)
-        n = np.sum(annotation)  # Total number of successes in population
-        N = np.sum(neighborhood)  # Total number of draws (sample size)
-        k = np.sum(neighborhood & annotation)  # Number of successes in sample
-
-        # Perform hypergeometric test for depletion
-        p_value_depletion = hypergeom.cdf(k, M, n, N)
-        depletion_pvals.append(p_value_depletion)
-        # Perform hypergeometric test for enrichment
-        p_value_enrichment = hypergeom.sf(k - 1, M, n, N)
-        enrichment_pvals.append(p_value_enrichment)
-
-        # Update the shared progress counter
-        progress_counter.value += 1
-
-    return depletion_pvals, enrichment_pvals
+    return {"depletion_pvals": depletion_pvals, "enrichment_pvals": enrichment_pvals}