risk-network 0.0.3b1__py3-none-any.whl

risk/log/params.py

@@ -0,0 +1,198 @@
+"""
+risk/log/params
+~~~~~~~~~~~~~~~
+"""
+
+import csv
+import json
+import warnings
+from datetime import datetime
+from typing import Any, Dict
+
+import numpy as np
+
+from .console import print_header
+
+# Suppress all warnings - this is to resolve warnings from multiprocessing
+warnings.filterwarnings("ignore")
+
+
+def _safe_param_export(func):
+    """A decorator to wrap parameter export functions in a try-except block for safe execution.
+
+    Args:
+        func (function): The function to be wrapped.
+
+    Returns:
+        function: The wrapped function with error handling.
+    """
+
+    def wrapper(*args, **kwargs):
+        try:
+            result = func(*args, **kwargs)
+            filepath = (
+                kwargs.get("filepath") or args[1]
+            )  # Assuming filepath is always the second argument
+            print(f"Parameters successfully exported to filepath: {filepath}")
+            return result
+        except Exception as e:
+            filepath = kwargs.get("filepath") or args[1]
+            print(f"An error occurred while exporting parameters to {filepath}: {e}")
+            return None
+
+    return wrapper
+
+
+class Params:
+    """Handles the storage and logging of various parameters for network analysis.
+
+    The Params class provides methods to log parameters related to different components of the analysis,
+    such as the network, annotations, neighborhoods, graph, and plotter settings. It also stores
+    the current datetime when the parameters were initialized.
+    """
+
+    def __init__(self):
+        """Initialize the Params object with default settings and current datetime."""
+        self.initialize()
+        self.datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    def initialize(self) -> None:
+        """Initialize the parameter dictionaries for different components."""
+        self.network = {}
+        self.annotations = {}
+        self.neighborhoods = {}
+        self.graph = {}
+        self.plotter = {}
+
+    def log_network(self, **kwargs) -> None:
+        """Log network-related parameters.
+
+        Args:
+            **kwargs: Network parameters to log.
+        """
+        self.network = {**self.network, **kwargs}
+
+    def log_annotations(self, **kwargs) -> None:
+        """Log annotation-related parameters.
+
+        Args:
+            **kwargs: Annotation parameters to log.
+        """
+        self.annotations = {**self.annotations, **kwargs}
+
+    def log_neighborhoods(self, **kwargs) -> None:
+        """Log neighborhood-related parameters.
+
+        Args:
+            **kwargs: Neighborhood parameters to log.
+        """
+        self.neighborhoods = {**self.neighborhoods, **kwargs}
+
+    def log_graph(self, **kwargs) -> None:
+        """Log graph-related parameters.
+
+        Args:
+            **kwargs: Graph parameters to log.
+        """
+        self.graph = {**self.graph, **kwargs}
+
+    def log_plotter(self, **kwargs) -> None:
+        """Log plotter-related parameters.
+
+        Args:
+            **kwargs: Plotter parameters to log.
+        """
+        self.plotter = {**self.plotter, **kwargs}
+
+    @_safe_param_export
+    def to_csv(self, filepath: str) -> None:
+        """Export the parameters to a CSV file.
+
+        Args:
+            filepath (str): The path where the CSV file will be saved.
+        """
+        # Load the parameter dictionary
+        params = self.load()
+        # Open the file in write mode
+        with open(filepath, "w", newline="") as csv_file:
+            writer = csv.writer(csv_file)
+            # Write the header
+            writer.writerow(["parent_key", "child_key", "value"])
+            # Write the rows
+            for parent_key, parent_value in params.items():
+                if isinstance(parent_value, dict):
+                    for child_key, child_value in parent_value.items():
+                        writer.writerow([parent_key, child_key, child_value])
+                else:
+                    writer.writerow([parent_key, "", parent_value])
+
+    @_safe_param_export
+    def to_json(self, filepath: str) -> None:
+        """Export the parameters to a JSON file.
+
+        Args:
+            filepath (str): The path where the JSON file will be saved.
+        """
+        with open(filepath, "w") as json_file:
+            json.dump(self.load(), json_file, indent=4)
+
+    @_safe_param_export
+    def to_txt(self, filepath: str) -> None:
+        """Export the parameters to a text file.
+
+        Args:
+            filepath (str): The path where the text file will be saved.
+        """
+        # Load the parameter dictionary
+        params = self.load()
+        # Open the file in write mode
+        with open(filepath, "w") as txt_file:
+            for key, nested_dict in params.items():
+                # Write the key
+                txt_file.write(f"{key}:\n")
+                # Write the nested dictionary values, one per line
+                for nested_key, nested_value in nested_dict.items():
+                    txt_file.write(f"  {nested_key}: {nested_value}\n")
+                # Add a blank line between different keys
+                txt_file.write("\n")
+
+    def load(self) -> Dict[str, Any]:
+        """Load and process various parameters, converting any np.ndarray values to lists.
+
+        Returns:
+            dict: A dictionary containing the processed parameters.
+        """
+        print_header("Loading parameters")
+        return _convert_ndarray_to_list(
+            {
+                "annotations": self.annotations,
+                "datetime": self.datetime,
+                "graph": self.graph,
+                "neighborhoods": self.neighborhoods,
+                "network": self.network,
+                "plotter": self.plotter,
+            }
+        )
+
+
+def _convert_ndarray_to_list(d: Any) -> Any:
+    """Recursively convert all np.ndarray values in the dictionary to lists.
+
+    Args:
+        d (dict): The dictionary to process.
+
+    Returns:
+        dict: The processed dictionary with np.ndarray values converted to lists.
+    """
+    if isinstance(d, dict):
+        # Recursively process each value in the dictionary
+        return {k: _convert_ndarray_to_list(v) for k, v in d.items()}
+    elif isinstance(d, list):
+        # Recursively process each item in the list
+        return [_convert_ndarray_to_list(v) for v in d]
+    elif isinstance(d, np.ndarray):
+        # Convert numpy arrays to lists
+        return d.tolist()
+    else:
+        # Return the value unchanged if it's not a dict, list, or ndarray
+        return d

risk/neighborhoods/__init__.py

@@ -0,0 +1,10 @@
+"""
+risk/neighborhoods
+~~~~~~~~~~~~~~~~~~
+"""
+
+from .domains import define_domains, trim_domains_and_top_annotations
+from .neighborhoods import (
+    get_network_neighborhoods,
+    process_neighborhoods,
+)

risk/neighborhoods/community.py

@@ -0,0 +1,189 @@
+"""
+risk/neighborhoods/community
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+"""
+
+import community as community_louvain
+import networkx as nx
+import numpy as np
+import markov_clustering as mc
+from networkx.algorithms.community import asyn_lpa_communities
+
+
+def calculate_dijkstra_neighborhoods(network: nx.Graph) -> np.ndarray:
+    """Calculate neighborhoods using Dijkstra's shortest path distances.
+
+    Args:
+        network (nx.Graph): The network graph.
+
+    Returns:
+        np.ndarray: Neighborhood matrix based on Dijkstra's distances.
+    """
+    # Compute Dijkstra's distance for all pairs of nodes in the network
+    all_dijkstra_paths = dict(nx.all_pairs_dijkstra_path_length(network, weight="length"))
+    neighborhoods = np.zeros((network.number_of_nodes(), network.number_of_nodes()), dtype=int)
+
+    # Populate the neighborhoods matrix based on Dijkstra's distances
+    for source, targets in all_dijkstra_paths.items():
+        for target, length in targets.items():
+            neighborhoods[source, target] = (
+                1 if np.isnan(length) or length == 0 else np.sqrt(1 / length)
+            )
+
+    return neighborhoods
+
+
+def calculate_label_propagation_neighborhoods(network: nx.Graph) -> np.ndarray:
+    """Apply Label Propagation to the network to detect communities.
+
+    Args:
+        network (nx.Graph): The network graph.
+
+    Returns:
+        np.ndarray: Neighborhood matrix based on Label Propagation.
+    """
+    # Apply Label Propagation
+    communities = nx.algorithms.community.label_propagation.label_propagation_communities(network)
+
+    # Create a mapping from node to community
+    community_dict = {}
+    for community_id, community in enumerate(communities):
+        for node in community:
+            community_dict[node] = community_id
+
+    # Create a neighborhood matrix
+    num_nodes = network.number_of_nodes()
+    neighborhoods = np.zeros((num_nodes, num_nodes), dtype=int)
+
+    # Assign neighborhoods based on community labels
+    for node_i, community_i in community_dict.items():
+        for node_j, community_j in community_dict.items():
+            if community_i == community_j:
+                neighborhoods[node_i, node_j] = 1
+
+    return neighborhoods
+
+
+def calculate_louvain_neighborhoods(
+    network: nx.Graph, resolution: float, random_seed: int = 888
+) -> np.ndarray:
+    """Calculate neighborhoods using the Louvain method.
+
+    Args:
+        network (nx.Graph): The network graph.
+        resolution (float): Resolution parameter for the Louvain method.
+        random_seed (int, optional): Random seed for reproducibility. Defaults to 888.
+
+    Returns:
+        np.ndarray: Neighborhood matrix based on the Louvain method.
+    """
+    # Apply Louvain method to partition the network
+    partition = community_louvain.best_partition(
+        network, resolution=resolution, random_state=random_seed
+    )
+    neighborhoods = np.zeros((network.number_of_nodes(), network.number_of_nodes()), dtype=int)
+
+    # Assign neighborhoods based on community partitions
+    for node_i, community_i in partition.items():
+        for node_j, community_j in partition.items():
+            if community_i == community_j:
+                neighborhoods[node_i, node_j] = 1
+
+    return neighborhoods
+
+
+def calculate_markov_clustering_neighborhoods(network: nx.Graph) -> np.ndarray:
+    """Apply Markov Clustering (MCL) to the network.
+
+    Args:
+        network (nx.Graph): The network graph.
+
+    Returns:
+        np.ndarray: Neighborhood matrix based on Markov Clustering.
+    """
+    # Convert the graph to an adjacency matrix
+    adjacency_matrix = nx.to_numpy_array(network)
+    # Run Markov Clustering
+    result = mc.run_mcl(adjacency_matrix)  # Run MCL with default parameters
+    # Get clusters
+    clusters = mc.get_clusters(result)
+
+    # Create a community label for each node
+    community_dict = {}
+    for community_id, community in enumerate(clusters):
+        for node in community:
+            community_dict[node] = community_id
+
+    # Create a neighborhood matrix
+    num_nodes = network.number_of_nodes()
+    neighborhoods = np.zeros((num_nodes, num_nodes), dtype=int)
+
+    # Assign neighborhoods based on community labels
+    for node_i, community_i in community_dict.items():
+        for node_j, community_j in community_dict.items():
+            if community_i == community_j:
+                neighborhoods[node_i, node_j] = 1
+
+    return neighborhoods
+
+
+def calculate_spinglass_neighborhoods(network: nx.Graph) -> np.ndarray:
+    """Apply Spin Glass Community Detection to the network.
+
+    Args:
+        network (nx.Graph): The network graph.
+
+    Returns:
+        np.ndarray: Neighborhood matrix based on Spin Glass communities.
+    """
+    # Use the asynchronous label propagation algorithm as a proxy for Spin Glass
+    communities = asyn_lpa_communities(network)
+
+    # Create a community label for each node
+    community_dict = {}
+    for community_id, community in enumerate(communities):
+        for node in community:
+            community_dict[node] = community_id
+
+    # Create a neighborhood matrix
+    num_nodes = network.number_of_nodes()
+    neighborhoods = np.zeros((num_nodes, num_nodes), dtype=int)
+
+    # Assign neighborhoods based on community labels
+    for node_i, community_i in community_dict.items():
+        for node_j, community_j in community_dict.items():
+            if community_i == community_j:
+                neighborhoods[node_i, node_j] = 1
+
+    return neighborhoods
+
+
+def calculate_walktrap_neighborhoods(network: nx.Graph) -> np.ndarray:
+    """Apply Walktrap Community Detection to the network.
+
+    Args:
+        network (nx.Graph): The network graph.
+
+    Returns:
+        np.ndarray: Neighborhood matrix based on Walktrap communities.
+    """
+    # Use the asynchronous label propagation algorithm as a proxy for Walktrap
+    communities = asyn_lpa_communities(network)
+
+    # Create a community label for each node
+    community_dict = {}
+    for community_id, community in enumerate(communities):
+        for node in community:
+            community_dict[node] = community_id
+
+    # Create a neighborhood matrix
+    num_nodes = network.number_of_nodes()
+    neighborhoods = np.zeros((num_nodes, num_nodes), dtype=int)
+
+    # Assign neighborhoods based on community labels
+    for node_i, community_i in community_dict.items():
+        for node_j, community_j in community_dict.items():
+            if community_i == community_j:
+                neighborhoods[node_i, node_j] = 1
+
+    return neighborhoods

risk/neighborhoods/domains.py

@@ -0,0 +1,257 @@
+"""
+risk/neighborhoods/domains
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+"""
+
+from contextlib import suppress
+from tqdm import tqdm
+from typing import Tuple
+
+import numpy as np
+import pandas as pd
+from scipy.cluster.hierarchy import linkage, fcluster
+from sklearn.metrics import silhouette_score
+
+from risk.annotations import get_description
+from risk.constants import GROUP_LINKAGE_METHODS, GROUP_DISTANCE_METRICS
+
+
+def define_domains(
+    top_annotations: pd.DataFrame,
+    significant_neighborhoods_enrichment: np.ndarray,
+    linkage_criterion: str,
+    linkage_method: str,
+    linkage_metric: str,
+) -> pd.DataFrame:
+    """Define domains and assign nodes to these domains based on their enrichment scores and clustering.
+
+    Args:
+        top_annotations (pd.DataFrame): DataFrame of top annotations data for the network nodes.
+        significant_neighborhoods_enrichment (np.ndarray): The binary enrichment matrix below alpha.
+        linkage_criterion (str): The clustering criterion for defining groups.
+        linkage_method (str): The linkage method for clustering.
+        linkage_metric (str): The linkage metric for clustering.
+
+    Returns:
+        pd.DataFrame: DataFrame with the primary domain for each node.
+    """
+    # Perform hierarchical clustering on the binary enrichment matrix
+    m = significant_neighborhoods_enrichment[:, top_annotations["top attributes"]].T
+    best_linkage, best_metric, best_threshold = _optimize_silhouette_across_linkage_and_metrics(
+        m, linkage_criterion, linkage_method, linkage_metric
+    )
+    try:
+        Z = linkage(m, method=best_linkage, metric=best_metric)
+    except ValueError as e:
+        raise ValueError("No significant annotations found.") from e
+
+    print(
+        f"Linkage criterion: '{linkage_criterion}'\nLinkage method: '{best_linkage}'\nLinkage metric: '{best_metric}'"
+    )
+    print(f"Optimal linkage threshold: {round(best_threshold, 3)}")
+
+    max_d_optimal = np.max(Z[:, 2]) * best_threshold
+    domains = fcluster(Z, max_d_optimal, criterion=linkage_criterion)
+    # Assign domains to the annotations matrix
+    top_annotations["domain"] = 0
+    top_annotations.loc[top_annotations["top attributes"], "domain"] = domains
+
+    # Create DataFrames to store domain information
+    node_to_enrichment = pd.DataFrame(
+        data=significant_neighborhoods_enrichment,
+        columns=[top_annotations.index.values, top_annotations["domain"]],
+    )
+    node_to_domain = node_to_enrichment.groupby(level="domain", axis=1).sum()
+
+    t_max = node_to_domain.loc[:, 1:].max(axis=1)
+    t_idxmax = node_to_domain.loc[:, 1:].idxmax(axis=1)
+    t_idxmax[t_max == 0] = 0
+
+    # Assign primary domain
+    node_to_domain["primary domain"] = t_idxmax
+
+    return node_to_domain
+
+
+def trim_domains_and_top_annotations(
+    domains: pd.DataFrame,
+    top_annotations: pd.DataFrame,
+    min_cluster_size: int = 5,
+    max_cluster_size: int = 1000,
+) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+    """Trim domains and top annotations that do not meet size criteria and find outliers.
+
+    Args:
+        domains (pd.DataFrame): DataFrame of domain data for the network nodes.
+        top_annotations (pd.DataFrame): DataFrame of top annotations data for the network nodes.
+        min_cluster_size (int, optional): Minimum size of a cluster to be retained. Defaults to 5.
+        max_cluster_size (int, optional): Maximum size of a cluster to be retained. Defaults to 1000.
+
+    Returns:
+        tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: A tuple containing:
+            - Trimmed annotations (pd.DataFrame)
+            - Trimmed domains (pd.DataFrame)
+            - A DataFrame with domain labels (pd.DataFrame)
+    """
+    # Identify domains to remove based on size criteria
+    domain_counts = domains["primary domain"].value_counts()
+    to_remove = set(
+        domain_counts[(domain_counts < min_cluster_size) | (domain_counts > max_cluster_size)].index
+    )
+
+    # Add invalid domain IDs
+    invalid_domain_id = 888888
+    invalid_domain_ids = {0, invalid_domain_id}
+    # Mark domains to be removed
+    top_annotations["domain"].replace(to_remove, invalid_domain_id, inplace=True)
+    domains.loc[domains["primary domain"].isin(to_remove), ["primary domain"]] = invalid_domain_id
+
+    # Normalize "num enriched neighborhoods" by percentile for each domain and scale to 0-10
+    top_annotations["normalized_value"] = top_annotations.groupby("domain")[
+        "neighborhood enrichment sums"
+    ].transform(lambda x: (x.rank(pct=True) * 10).apply(np.ceil).astype(int))
+    # Multiply 'words' column by normalized values
+    top_annotations["words"] = top_annotations.apply(
+        lambda row: " ".join([row["words"]] * row["normalized_value"]), axis=1
+    )
+
+    # Generate domain labels
+    domain_labels = top_annotations.groupby("domain")["words"].apply(get_description).reset_index()
+    trimmed_domains_matrix = domain_labels.rename(
+        columns={"domain": "id", "words": "label"}
+    ).set_index("id")
+
+    # Remove invalid domains
+    valid_annotations = top_annotations[~top_annotations["domain"].isin(invalid_domain_ids)].drop(
+        columns=["normalized_value"]
+    )
+    valid_domains = domains[~domains["primary domain"].isin(invalid_domain_ids)]
+    valid_trimmed_domains_matrix = trimmed_domains_matrix[
+        ~trimmed_domains_matrix.index.isin(invalid_domain_ids)
+    ]
+
+    return valid_annotations, valid_domains, valid_trimmed_domains_matrix
+
+
+def _optimize_silhouette_across_linkage_and_metrics(
+    m: np.ndarray, linkage_criterion: str, linkage_method: str, linkage_metric: str
+) -> Tuple[str, str, float]:
+    """Optimize silhouette score across different linkage methods and distance metrics.
+
+    Args:
+        m (np.ndarray): Data matrix.
+        linkage_criterion (str): Clustering criterion.
+        linkage_method (str): Linkage method for clustering.
+        linkage_metric (str): Linkage metric for clustering.
+
+    Returns:
+        tuple[str, str, float]: A tuple containing:
+            - Best linkage method (str)
+            - Best linkage metric (str)
+            - Best threshold (float)
+    """
+    best_overall_method = linkage_method
+    best_overall_metric = linkage_metric
+    best_overall_score = -np.inf
+    best_overall_threshold = 1
+
+    linkage_methods = GROUP_LINKAGE_METHODS if linkage_method == "auto" else [linkage_method]
+    linkage_metrics = GROUP_DISTANCE_METRICS if linkage_metric == "auto" else [linkage_metric]
+    total_combinations = len(linkage_methods) * len(linkage_metrics)
+
+    # Evaluating optimal linkage method and metric
+    for method in tqdm(
+        linkage_methods,
+        desc="Evaluating optimal linkage method and metric",
+        total=total_combinations,
+        bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]",
+    ):
+        for metric in linkage_metrics:
+            with suppress(Exception):
+                Z = linkage(m, method=method, metric=metric)
+                threshold, score = _find_best_silhouette_score(Z, m, metric, linkage_criterion)
+                if score > best_overall_score:
+                    best_overall_score = score
+                    best_overall_threshold = threshold
+                    best_overall_method = method
+                    best_overall_metric = metric
+
+    return best_overall_method, best_overall_metric, best_overall_threshold
+
+
+def _find_best_silhouette_score(
+    Z: np.ndarray,
+    m: np.ndarray,
+    linkage_metric: str,
+    linkage_criterion: str,
+    lower_bound: float = 0.001,
+    upper_bound: float = 1.0,
+    resolution: float = 0.001,
+) -> Tuple[float, float]:
+    """Find the best silhouette score using binary search.
+
+    Args:
+        Z (np.ndarray): Linkage matrix.
+        m (np.ndarray): Data matrix.
+        linkage_metric (str): Linkage metric for silhouette score calculation.
+        linkage_criterion (str): Clustering criterion.
+        lower_bound (float, optional): Lower bound for search. Defaults to 0.001.
+        upper_bound (float, optional): Upper bound for search. Defaults to 1.0.
+        resolution (float, optional): Desired resolution for the best threshold. Defaults to 0.001.
+
+    Returns:
+        tuple[float, float]: A tuple containing:
+            - Best threshold (float): The threshold that yields the best silhouette score.
+            - Best silhouette score (float): The highest silhouette score achieved.
+    """
+    best_score = -np.inf
+    best_threshold = None
+
+    # Test lower bound
+    max_d_lower = np.max(Z[:, 2]) * lower_bound
+    clusters_lower = fcluster(Z, max_d_lower, criterion=linkage_criterion)
+    try:
+        score_lower = silhouette_score(m, clusters_lower, metric=linkage_metric)
+    except ValueError:
+        score_lower = -np.inf
+
+    # Test upper bound
+    max_d_upper = np.max(Z[:, 2]) * upper_bound
+    clusters_upper = fcluster(Z, max_d_upper, criterion=linkage_criterion)
+    try:
+        score_upper = silhouette_score(m, clusters_upper, metric=linkage_metric)
+    except ValueError:
+        score_upper = -np.inf
+
+    # Determine initial bounds for binary search
+    if score_lower > score_upper:
+        best_score = score_lower
+        best_threshold = lower_bound
+        upper_bound = (lower_bound + upper_bound) / 2
+    else:
+        best_score = score_upper
+        best_threshold = upper_bound
+        lower_bound = (lower_bound + upper_bound) / 2
+
+    # Binary search loop
+    while upper_bound - lower_bound > resolution:
+        mid_threshold = (upper_bound + lower_bound) / 2
+        max_d_mid = np.max(Z[:, 2]) * mid_threshold
+        clusters_mid = fcluster(Z, max_d_mid, criterion=linkage_criterion)
+        try:
+            score_mid = silhouette_score(m, clusters_mid, metric=linkage_metric)
+        except ValueError:
+            score_mid = -np.inf
+
+        # Update best score and threshold if mid-point is better
+        if score_mid > best_score:
+            best_score = score_mid
+            best_threshold = mid_threshold
+
+        # Adjust bounds based on the scores
+        if score_lower > score_upper:
+            upper_bound = mid_threshold
+        else:
+            lower_bound = mid_threshold
+
+    return best_threshold, float(best_score)