risk-network 0.0.8b18__py3-none-any.whl → 0.0.9b26__py3-none-any.whl

risk/neighborhoods/domains.py

@@ -5,32 +5,32 @@ risk/neighborhoods/domains
 
 from contextlib import suppress
 from itertools import product
-from tqdm import tqdm
 from typing import Tuple
 
 import numpy as np
 import pandas as pd
 from scipy.cluster.hierarchy import linkage, fcluster
 from sklearn.metrics import silhouette_score
+from tqdm import tqdm
 
-from risk.annotations import get_description
+from risk.annotations import get_weighted_description
 from risk.constants import GROUP_LINKAGE_METHODS, GROUP_DISTANCE_METRICS
 from risk.log import logger
 
 
 def define_domains(
     top_annotations: pd.DataFrame,
-    significant_neighborhoods_enrichment: np.ndarray,
+    significant_neighborhoods_significance: np.ndarray,
     linkage_criterion: str,
     linkage_method: str,
     linkage_metric: str,
 ) -> pd.DataFrame:
-    """Define domains and assign nodes to these domains based on their enrichment scores and clustering,
+    """Define domains and assign nodes to these domains based on their significance scores and clustering,
     handling errors by assigning unique domains when clustering fails.
 
     Args:
         top_annotations (pd.DataFrame): DataFrame of top annotations data for the network nodes.
-        significant_neighborhoods_enrichment (np.ndarray): The binary enrichment matrix below alpha.
+        significant_neighborhoods_significance (np.ndarray): The binary significance matrix below alpha.
         linkage_criterion (str): The clustering criterion for defining groups.
         linkage_method (str): The linkage method for clustering.
         linkage_metric (str): The linkage metric for clustering.
@@ -39,8 +39,14 @@ def define_domains(
         pd.DataFrame: DataFrame with the primary domain for each node.
     """
     try:
+        if linkage_criterion == "off":
+            raise ValueError("Clustering is turned off.")
+
         # Transpose the matrix to cluster annotations
-        m = significant_neighborhoods_enrichment[:, top_annotations["top attributes"]].T
+        m = significant_neighborhoods_significance[:, top_annotations["significant_annotations"]].T
+        # Safeguard the matrix by replacing NaN, Inf, and -Inf values
+        m = _safeguard_matrix(m)
+        # Optimize silhouette score across different linkage methods and distance metrics
         best_linkage, best_metric, best_threshold = _optimize_silhouette_across_linkage_and_metrics(
             m, linkage_criterion, linkage_method, linkage_metric
         )
@@ -55,40 +61,49 @@ def define_domains(
         # Assign domains to the annotations matrix
         domains = fcluster(Z, max_d_optimal, criterion=linkage_criterion)
         top_annotations["domain"] = 0
-        top_annotations.loc[top_annotations["top attributes"], "domain"] = domains
+        top_annotations.loc[top_annotations["significant_annotations"], "domain"] = domains
     except ValueError:
         # If a ValueError is encountered, handle it by assigning unique domains
         n_rows = len(top_annotations)
-        logger.error(
-            f"Error encountered. Skipping clustering and assigning {n_rows} unique domains."
-        )
+        if linkage_criterion == "off":
+            logger.warning(
+                f"Clustering is turned off. Skipping clustering and assigning {n_rows} unique domains."
+            )
+        else:
+            logger.error(
+                f"Error encountered. Skipping clustering and assigning {n_rows} unique domains."
+            )
         top_annotations["domain"] = range(1, n_rows + 1)  # Assign unique domains
 
     # Create DataFrames to store domain information
-    node_to_enrichment = pd.DataFrame(
-        data=significant_neighborhoods_enrichment,
+    node_to_significance = pd.DataFrame(
+        data=significant_neighborhoods_significance,
         columns=[top_annotations.index.values, top_annotations["domain"]],
     )
-    node_to_domain = node_to_enrichment.groupby(level="domain", axis=1).sum()
+    node_to_domain = node_to_significance.T.groupby(level="domain").sum().T
 
-    # Find the maximum enrichment score for each node
+    # Find the maximum significance score for each node
     t_max = node_to_domain.loc[:, 1:].max(axis=1)
     t_idxmax = node_to_domain.loc[:, 1:].idxmax(axis=1)
     t_idxmax[t_max == 0] = 0
 
+    # Assign all domains where the score is greater than 0
+    node_to_domain["all_domains"] = node_to_domain.loc[:, 1:].apply(
+        lambda row: list(row[row > 0].index), axis=1
+    )
     # Assign primary domain
-    node_to_domain["primary domain"] = t_idxmax
+    node_to_domain["primary_domain"] = t_idxmax
 
     return node_to_domain
 
 
-def trim_domains_and_top_annotations(
+def trim_domains(
     domains: pd.DataFrame,
     top_annotations: pd.DataFrame,
     min_cluster_size: int = 5,
     max_cluster_size: int = 1000,
 ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
-    """Trim domains and top annotations that do not meet size criteria and find outliers.
+    """Trim domains that do not meet size criteria and find outliers.
 
     Args:
         domains (pd.DataFrame): DataFrame of domain data for the network nodes.
@@ -97,13 +112,12 @@ def trim_domains_and_top_annotations(
         max_cluster_size (int, optional): Maximum size of a cluster to be retained. Defaults to 1000.
 
     Returns:
-        tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: A tuple containing:
-            - Trimmed annotations (pd.DataFrame)
+        Tuple[pd.DataFrame, pd.DataFrame]:
             - Trimmed domains (pd.DataFrame)
             - A DataFrame with domain labels (pd.DataFrame)
     """
     # Identify domains to remove based on size criteria
-    domain_counts = domains["primary domain"].value_counts()
+    domain_counts = domains["primary_domain"].value_counts()
     to_remove = set(
         domain_counts[(domain_counts < min_cluster_size) | (domain_counts > max_cluster_size)].index
     )
@@ -112,34 +126,73 @@ def trim_domains_and_top_annotations(
     invalid_domain_id = 888888
     invalid_domain_ids = {0, invalid_domain_id}
     # Mark domains to be removed
-    top_annotations["domain"].replace(to_remove, invalid_domain_id, inplace=True)
-    domains.loc[domains["primary domain"].isin(to_remove), ["primary domain"]] = invalid_domain_id
+    top_annotations["domain"] = top_annotations["domain"].replace(to_remove, invalid_domain_id)
+    domains.loc[domains["primary_domain"].isin(to_remove), ["primary_domain"]] = invalid_domain_id
 
-    # Normalize "num enriched neighborhoods" by percentile for each domain and scale to 0-10
+    # Normalize "num significant neighborhoods" by percentile for each domain and scale to 0-10
     top_annotations["normalized_value"] = top_annotations.groupby("domain")[
-        "neighborhood enrichment sums"
+        "significant_neighborhood_significance_sums"
     ].transform(lambda x: (x.rank(pct=True) * 10).apply(np.ceil).astype(int))
-    # Multiply 'words' column by normalized values
-    top_annotations["words"] = top_annotations.apply(
-        lambda row: " ".join([str(row["words"])] * row["normalized_value"]), axis=1
+    # Modify the lambda function to pass both full_terms and significant_significance_score
+    top_annotations["combined_terms"] = top_annotations.apply(
+        lambda row: " ".join([str(row["full_terms"])] * row["normalized_value"]), axis=1
     )
 
-    # Generate domain labels
-    domain_labels = top_annotations.groupby("domain")["words"].apply(get_description).reset_index()
+    # Perform the groupby operation while retaining the other columns and adding the weighting with significance scores
+    domain_labels = (
+        top_annotations.groupby("domain")
+        .agg(
+            full_terms=("full_terms", lambda x: list(x)),
+            significance_scores=("significant_significance_score", lambda x: list(x)),
+        )
+        .reset_index()
+    )
+    domain_labels["combined_terms"] = domain_labels.apply(
+        lambda row: get_weighted_description(
+            pd.Series(row["full_terms"]), pd.Series(row["significance_scores"])
+        ),
+        axis=1,
+    )
+
+    # Rename the columns as necessary
     trimmed_domains_matrix = domain_labels.rename(
-        columns={"domain": "id", "words": "label"}
+        columns={
+            "domain": "id",
+            "combined_terms": "normalized_description",
+            "full_terms": "full_descriptions",
+            "significance_scores": "significance_scores",
+        }
     ).set_index("id")
 
     # Remove invalid domains
-    valid_annotations = top_annotations[~top_annotations["domain"].isin(invalid_domain_ids)].drop(
-        columns=["normalized_value"]
-    )
-    valid_domains = domains[~domains["primary domain"].isin(invalid_domain_ids)]
+    valid_domains = domains[~domains["primary_domain"].isin(invalid_domain_ids)]
     valid_trimmed_domains_matrix = trimmed_domains_matrix[
         ~trimmed_domains_matrix.index.isin(invalid_domain_ids)
     ]
+    return valid_domains, valid_trimmed_domains_matrix
+
+
+def _safeguard_matrix(matrix: np.ndarray) -> np.ndarray:
+    """Safeguard the matrix by replacing NaN, Inf, and -Inf values.
 
-    return valid_annotations, valid_domains, valid_trimmed_domains_matrix
+    Args:
+        matrix (np.ndarray): Data matrix.
+
+    Returns:
+        np.ndarray: Safeguarded data matrix.
+    """
+    # Replace NaN with column mean
+    nan_replacement = np.nanmean(matrix, axis=0)
+    matrix = np.where(np.isnan(matrix), nan_replacement, matrix)
+    # Replace Inf/-Inf with maximum/minimum finite values
+    finite_max = np.nanmax(matrix[np.isfinite(matrix)])
+    finite_min = np.nanmin(matrix[np.isfinite(matrix)])
+    matrix = np.where(np.isposinf(matrix), finite_max, matrix)
+    matrix = np.where(np.isneginf(matrix), finite_min, matrix)
+    # Ensure rows have non-zero variance (optional step)
+    row_variance = np.var(matrix, axis=1)
+    matrix = matrix[row_variance > 0]
+    return matrix
 
 
 def _optimize_silhouette_across_linkage_and_metrics(
@@ -154,7 +207,7 @@ def _optimize_silhouette_across_linkage_and_metrics(
         linkage_metric (str): Linkage metric for clustering.
 
     Returns:
-        tuple[str, str, float]: A tuple containing:
+        Tuple[str, str, float]:
             - Best linkage method (str)
             - Best linkage metric (str)
             - Best threshold (float)
@@ -175,7 +228,8 @@ def _optimize_silhouette_across_linkage_and_metrics(
         total=total_combinations,
         bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]",
     ):
-        with suppress(Exception):
+        # Some linkage methods and metrics may not work with certain data
+        with suppress(ValueError):
             Z = linkage(m, method=method, metric=metric)
             threshold, score = _find_best_silhouette_score(Z, m, metric, linkage_criterion)
             if score > best_overall_score:
@@ -208,7 +262,7 @@ def _find_best_silhouette_score(
         resolution (float, optional): Desired resolution for the best threshold. Defaults to 0.001.
 
     Returns:
-        tuple[float, float]: A tuple containing:
+        Tuple[float, float]:
             - Best threshold (float): The threshold that yields the best silhouette score.
             - Best silhouette score (float): The highest silhouette score achieved.
     """