risk-network 0.0.9b39__py3-none-any.whl → 0.0.9b40__py3-none-any.whl

risk/__init__.py

@@ -7,4 +7,4 @@ RISK: Regional Inference of Significant Kinships
 
 from risk.risk import RISK
 
-__version__ = "0.0.9-beta.39"
+__version__ = "0.0.9-beta.40"

risk/annotations/annotations.py

@@ -12,8 +12,9 @@ import networkx as nx
 import nltk
 import numpy as np
 import pandas as pd
-from nltk.tokenize import word_tokenize
 from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+from nltk.tokenize import word_tokenize
 
 from risk.log import logger
 from scipy.sparse import coo_matrix
@@ -31,11 +32,17 @@ def _setup_nltk():
     except LookupError:
         nltk.download("stopwords")
 
+    try:
+        nltk.data.find("corpora/wordnet")
+    except LookupError:
+        nltk.download("wordnet")
+
 
 # Ensure you have the necessary NLTK data
 _setup_nltk()
-# Initialize English stopwords
-stop_words = set(stopwords.words("english"))
+# Use NLTK's stopwords
+STOP_WORDS = set(stopwords.words("english"))
+LEMMATIZER = WordNetLemmatizer()
 
 
 def load_annotations(
@@ -208,104 +215,121 @@ def define_top_annotations(
 
 def get_weighted_description(words_column: pd.Series, scores_column: pd.Series) -> str:
     """Generate a weighted description from words and their corresponding scores,
-    with support for stopwords filtering and improved weighting logic.
+    using improved weighting logic with normalization, lemmatization, and aggregation.
 
     Args:
-        words_column (pd.Series): A pandas Series containing strings to process.
+        words_column (pd.Series): A pandas Series containing strings (phrases) to process.
         scores_column (pd.Series): A pandas Series containing significance scores to weigh the terms.
 
     Returns:
-        str: A coherent description formed from the most frequent and significant words, weighed by significance scores.
+        str: A coherent description formed from the most frequent and significant words.
     """
-    # Handle case where all scores are the same
+    # Normalize significance scores to [0,1]. If all scores are identical, use 1.
     if scores_column.max() == scores_column.min():
-        normalized_scores = pd.Series([1] * len(scores_column))
+        normalized_scores = pd.Series([1] * len(scores_column), index=scores_column.index)
     else:
-        # Normalize the significance scores to be between 0 and 1
         normalized_scores = (scores_column - scores_column.min()) / (
             scores_column.max() - scores_column.min()
         )
 
-    # Combine words and normalized scores to create weighted words
+    # Accumulate weighted counts for each token (after cleaning and lemmatization)
+    weighted_counts = {}
+    for phrase, score in zip(words_column, normalized_scores):
+        # Tokenize the phrase
+        tokens = word_tokenize(str(phrase))
+        # Determine the weight (scale factor; here multiplying normalized score by 10)
+        weight = max(1, int((0 if pd.isna(score) else score) * 10))
+        for token in tokens:
+            # Clean token: lowercase and remove extraneous punctuation (but preserve intra-word hyphens)
+            token_clean = re.sub(r"[^\w\-]", "", token.lower()).strip()
+            if not token_clean:
+                continue
+            # Skip tokens that are pure numbers
+            if token_clean.isdigit():
+                continue
+            # Skip stopwords
+            if token_clean in STOP_WORDS:
+                continue
+            # Lemmatize the token to merge similar forms
+            token_norm = LEMMATIZER.lemmatize(token_clean)
+            weighted_counts[token_norm] = weighted_counts.get(token_norm, 0) + weight
+
+    # Reconstruct a weighted token list by repeating each token by its aggregated count.
     weighted_words = []
-    for word, score in zip(words_column, normalized_scores):
-        word = str(word)
-        if word not in stop_words:  # Skip stopwords
-            weight = max(1, int((0 if pd.isna(score) else score) * 10))
-            weighted_words.extend([word] * weight)
-
-    # Tokenize the weighted words, but preserve number-word patterns like '4-alpha'
-    tokens = word_tokenize(" ".join(weighted_words))
-    # Ensure we treat "4-alpha" or other "number-word" patterns as single tokens
+    for token, count in weighted_counts.items():
+        weighted_words.extend([token] * count)
+
+    # Combine tokens that match number-word patterns (e.g. "4-alpha") and remove pure numeric tokens.
     combined_tokens = []
-    for token in tokens:
-        # Match patterns like '4-alpha' or '5-hydroxy' and keep them together
+    for token in weighted_words:
         if re.match(r"^\d+-\w+", token):
             combined_tokens.append(token)
-        elif token.replace(".", "", 1).isdigit():  # Handle pure numeric tokens
-            # Ignore pure numbers as descriptions unless necessary
+        elif token.replace(".", "", 1).isdigit():
             continue
         else:
             combined_tokens.append(token)
 
-    # Prevent descriptions like just '4' from being selected
+    # If the only token is numeric, return a default value.
     if len(combined_tokens) == 1 and combined_tokens[0].isdigit():
-        return "N/A"  # Return "N/A" for cases where it's just a number
+        return "N/A"
 
-    # Simplify the word list and generate the description
+    # Simplify the token list to remove near-duplicates based on the Jaccard index.
     simplified_words = _simplify_word_list(combined_tokens)
+    # Generate a coherent description from the simplified words.
     description = _generate_coherent_description(simplified_words)
 
     return description
 
 
 def _simplify_word_list(words: List[str], threshold: float = 0.80) -> List[str]:
-    """Filter out words that are too similar based on the Jaccard index, keeping the word with the higher count.
+    """Filter out words that are too similar based on the Jaccard index,
+    keeping the word with the higher aggregated count.
 
     Args:
-        words (list of str): The list of words to be filtered.
+        words (List[str]): The list of tokens to be filtered.
         threshold (float, optional): The similarity threshold for the Jaccard index. Defaults to 0.80.
 
     Returns:
-        list of str: A list of filtered words, where similar words are reduced to the most frequent one.
+        List[str]: A list of filtered words, where similar words are reduced to the most frequent one.
     """
-    # Count the occurrences of each word
+    # Count the occurrences (which reflect the weighted importance)
     word_counts = Counter(words)
     filtered_words = []
     used_words = set()
-    # Iterate through the words to find similar words
-    for word in word_counts:
+
+    # Iterate through words sorted by descending weighted frequency
+    for word in sorted(word_counts, key=lambda w: word_counts[w], reverse=True):
         if word in used_words:
             continue
 
         word_set = set(word)
-        # Find similar words based on the Jaccard index
+        # Find similar words (including the current word) based on the Jaccard index
         similar_words = [
             other_word
             for other_word in word_counts
             if _calculate_jaccard_index(word_set, set(other_word)) >= threshold
         ]
-        # Sort by frequency and choose the most frequent word
+        # Choose the word with the highest weighted count among the similar group
         similar_words.sort(key=lambda w: word_counts[w], reverse=True)
         best_word = similar_words[0]
         filtered_words.append(best_word)
         used_words.update(similar_words)
 
+    # Preserve the original order (by frequency) from the filtered set
     final_words = [word for word in words if word in filtered_words]
 
     return final_words
 
 
 def _calculate_jaccard_index(set1: Set[Any], set2: Set[Any]) -> float:
-    """Calculate the Jaccard Index of two sets.
+    """Calculate the Jaccard index between two sets.
 
     Args:
-        set1 (set): The first set for comparison.
-        set2 (set): The second set for comparison.
+        set1 (Set[Any]): The first set.
+        set2 (Set[Any]): The second set.
 
     Returns:
-        float: The Jaccard Index, which is the ratio of the intersection to the union of the two sets.
-               Returns 0 if the union of the sets is empty.
+        float: The Jaccard index (intersection over union). Returns 0 if the union is empty.
     """
     intersection = len(set1.intersection(set2))
     union = len(set1.union(set2))
@@ -313,28 +337,28 @@ def _calculate_jaccard_index(set1: Set[Any], set2: Set[Any]) -> float:
 
 
 def _generate_coherent_description(words: List[str]) -> str:
-    """Generate a coherent description from a list of words or numerical string values.
+    """Generate a coherent description from a list of words.
+
     If there is only one unique entry, return it directly.
+    Otherwise, order the words by frequency and join them into a single string.
 
     Args:
-        words (List): A list of words or numerical string values.
+        words (List[str]): A list of tokens.
 
     Returns:
-        str: A coherent description formed by arranging the words in a logical sequence.
+        str: A coherent, space-separated description.
     """
-    # If there are no words, return a keyword indicating no data is available
     if not words:
         return "N/A"
 
-    # If there's only one unique word, return it directly
+    # If there is only one unique word, return it directly
     unique_words = set(words)
     if len(unique_words) == 1:
         return list(unique_words)[0]
 
-    # Count the frequency of each word and sort them by frequency
+    # Count weighted occurrences and sort in descending order.
     word_counts = Counter(words)
     most_common_words = [word for word, _ in word_counts.most_common()]
-    # Join the most common words to form a coherent description based on frequency
     description = " ".join(most_common_words)
 
     return description

risk/neighborhoods/domains.py

@@ -14,17 +14,27 @@ from sklearn.metrics import silhouette_score
 from tqdm import tqdm
 
 from risk.annotations import get_weighted_description
-from risk.constants import GROUP_LINKAGE_METHODS, GROUP_DISTANCE_METRICS
 from risk.log import logger
 
 
+# Define constants for clustering
+# fmt: off
+LINKAGE_METHODS = {"single", "complete", "average", "weighted", "centroid", "median", "ward"}
+LINKAGE_METRICS = {
+    "braycurtis","canberra", "chebyshev", "cityblock", "correlation", "cosine", "dice", "euclidean",
+    "hamming", "jaccard", "jensenshannon", "kulczynski1", "mahalanobis", "matching", "minkowski",
+    "rogerstanimoto", "russellrao", "seuclidean", "sokalmichener", "sokalsneath", "sqeuclidean", "yule",
+}
+# fmt: on
+
+
 def define_domains(
     top_annotations: pd.DataFrame,
     significant_neighborhoods_significance: np.ndarray,
     linkage_criterion: str,
     linkage_method: str,
     linkage_metric: str,
-    linkage_threshold: float,
+    linkage_threshold: Union[float, str],
 ) -> pd.DataFrame:
     """Define domains and assign nodes to these domains based on their significance scores and clustering,
     handling errors by assigning unique domains when clustering fails.
@@ -33,9 +43,9 @@ def define_domains(
         top_annotations (pd.DataFrame): DataFrame of top annotations data for the network nodes.
         significant_neighborhoods_significance (np.ndarray): The binary significance matrix below alpha.
         linkage_criterion (str): The clustering criterion for defining groups.
-        linkage_method (str): The linkage method for clustering.
-        linkage_metric (str): The linkage metric for clustering.
-        linkage_threshold (float): The threshold for clustering.
+        linkage_method (str): The linkage method for clustering. Choose "auto" to optimize.
+        linkage_metric (str): The linkage metric for clustering. Choose "auto" to optimize.
+        linkage_threshold (float, str): The threshold for clustering. Choose "auto" to optimize.
 
     Returns:
         pd.DataFrame: DataFrame with the primary domain for each node.
@@ -55,9 +65,8 @@ def define_domains(
         # Perform hierarchical clustering
         Z = linkage(m, method=best_linkage, metric=best_metric)
         logger.warning(
-            f"Linkage criterion: '{linkage_criterion}'\nLinkage method: '{best_linkage}'\nLinkage metric: '{best_metric}'"
+            f"Linkage criterion: '{linkage_criterion}'\nLinkage method: '{best_linkage}'\nLinkage metric: '{best_metric}'\nLinkage threshold: {round(best_threshold, 3)}"
         )
-        logger.debug(f"Optimal linkage threshold: {round(best_threshold, 3)}")
         # Calculate the optimal threshold for clustering
         max_d_optimal = np.max(Z[:, 2]) * best_threshold
         # Assign domains to the annotations matrix
@@ -209,9 +218,9 @@ def _optimize_silhouette_across_linkage_and_metrics(
     Args:
         m (np.ndarray): Data matrix.
         linkage_criterion (str): Clustering criterion.
-        linkage_method (str): Linkage method for clustering.
-        linkage_metric (str): Linkage metric for clustering.
-        linkage_threshold (Union[str, float]): Threshold for clustering. Set to "auto" to optimize.
+        linkage_method (str): Linkage method for clustering. Choose "auto" to optimize.
+        linkage_metric (str): Linkage metric for clustering. Choose "auto" to optimize.
+        linkage_threshold (Union[str, float]): Threshold for clustering. Choose "auto" to optimize.
 
     Returns:
         Tuple[str, str, float]:
@@ -226,8 +235,8 @@ def _optimize_silhouette_across_linkage_and_metrics(
     best_overall_score = -np.inf
 
     # Set linkage methods and metrics to all combinations if "auto" is selected
-    linkage_methods = GROUP_LINKAGE_METHODS if linkage_method == "auto" else [linkage_method]
-    linkage_metrics = GROUP_DISTANCE_METRICS if linkage_metric == "auto" else [linkage_metric]
+    linkage_methods = LINKAGE_METHODS if linkage_method == "auto" else [linkage_method]
+    linkage_metrics = LINKAGE_METRICS if linkage_metric == "auto" else [linkage_metric]
     total_combinations = len(linkage_methods) * len(linkage_metrics)
 
     # Evaluating optimal linkage method and metric

risk/network/graph/api.py

@@ -4,7 +4,7 @@ risk/network/graph/api
 """
 
 import copy
-from typing import Any, Dict
+from typing import Any, Dict, Union
 
 import networkx as nx
 import pandas as pd
@@ -42,7 +42,7 @@ class GraphAPI:
         linkage_criterion: str = "distance",
         linkage_method: str = "average",
         linkage_metric: str = "yule",
-        linkage_threshold: float = 0.2,
+        linkage_threshold: Union[float, str] = 0.2,
         min_cluster_size: int = 5,
         max_cluster_size: int = 1000,
     ) -> Graph:
@@ -58,9 +58,11 @@ class GraphAPI:
             impute_depth (int, optional): Depth for imputing neighbors. Defaults to 0.
             prune_threshold (float, optional): Distance threshold for pruning neighbors. Defaults to 0.0.
             linkage_criterion (str, optional): Clustering criterion for defining domains. Defaults to "distance".
-            linkage_method (str, optional): Clustering method to use. Defaults to "average".
-            linkage_metric (str, optional): Metric to use for calculating distances. Defaults to "yule".
-            linkage_threshold (float, optional): Threshold for clustering. Defaults to 0.2.
+            linkage_method (str, optional): Clustering method to use. Choose "auto" to optimize. Defaults to "average".
+            linkage_metric (str, optional): Metric to use for calculating distances. Choose "auto" to optimize.
+                Defaults to "yule".
+            linkage_threshold (float, str, optional): Threshold for clustering. Choose "auto" to optimize.
+                Defaults to 0.2.
             min_cluster_size (int, optional): Minimum size for clusters. Defaults to 5.
             max_cluster_size (int, optional): Maximum size for clusters. Defaults to 1000.
 

{risk_network-0.0.9b39.dist-info → risk_network-0.0.9b40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: risk-network
-Version: 0.0.9b39
+Version: 0.0.9b40
 Summary: A Python package for biological network analysis
 Author: Ira Horecka
 Author-email: Ira Horecka <ira89@icloud.com>

{risk_network-0.0.9b39.dist-info → risk_network-0.0.9b40.dist-info}/RECORD

@@ -1,8 +1,7 @@
-risk/__init__.py,sha256=ewYSGDLHigkwFLI9IW6qDbQk4uS6nb3RTd-k2GCD1b0,127
-risk/constants.py,sha256=XInRaH78Slnw_sWgAsBFbUHkyA0h0jL0DKGuQNbOvjM,550
+risk/__init__.py,sha256=2Ucmxw9wGNzUhqe_QGlEi2pnGhkdOrl9wa8w-MUIfm8,127
 risk/risk.py,sha256=s827_lRknFseOP9O4zW8sP-IcCd2EzrpV_tnVY_tz5s,1104
 risk/annotations/__init__.py,sha256=parsbcux1U4urpUqh9AdzbDWuLj9HlMidycMPkpSQFo,179
-risk/annotations/annotations.py,sha256=g8ca9H49dZIqHv6Od3Dem4BIo_euy8alL3PDauT6ZJI,14088
+risk/annotations/annotations.py,sha256=Sq24YBtNPMxXOvWoxqPwOJ4bsFAMIBYpVWjEvsQPtNo,14912
 risk/annotations/io.py,sha256=z1AJySsU-KL_IYuHa7j3nvuczmOHgK3WfaQ4TRunvrA,10499
 risk/log/__init__.py,sha256=7LxDysQu7doi0LAvlY2YbjN6iJH0fNknqy8lSLgeljo,217
 risk/log/console.py,sha256=PgjyEvyhYLUSHXPUKEqOmxsDsfrjPICIgqo_cAHq0N8,4575
@@ -10,13 +9,13 @@ risk/log/parameters.py,sha256=VtwfMzLU1xI4yji3-Ch5vHjH-KdwTfwaEMmi7hFQTs0,5716
 risk/neighborhoods/__init__.py,sha256=Q74HwTH7okI-vaskJPy2bYwb5sNjGASTzJ6m8V8arCU,234
 risk/neighborhoods/api.py,sha256=ywngw2TQVV27gYlWDXcs8-qnmeepnvb-W9ov6J6VEPM,23341
 risk/neighborhoods/community.py,sha256=5Q_-VAJC-5SY5EUsB8gIlemeDoAL85uLjyl16pItHiQ,16699
-risk/neighborhoods/domains.py,sha256=Yu93mKNCuOpBGa87knAH-XIl260kf-rswPfn3aC9GNo,13937
+risk/neighborhoods/domains.py,sha256=4K1tbiia3_TQKUrGdfmKVdYlRD2EEzPnMCKRv6IGxu4,14448
 risk/neighborhoods/neighborhoods.py,sha256=l9FhADB1C-OxM8E9QXOcA4osUDgA1vs4ud-OCGKKybc,21457
 risk/network/__init__.py,sha256=oVi3FA1XXKD84014Cykq-9bpX4_s0F3aAUfNOU-07Qw,73
 risk/network/geometry.py,sha256=eVtGHMgBf9fEqQZUFdHWjw-zFYYpfUONoHFSAxoRkug,6219
 risk/network/io.py,sha256=RCH4nQdgYDXcNwMfpSz7qEmPO0pJ1p9fL0rNQptsQrc,21673
 risk/network/graph/__init__.py,sha256=ziGJew3yhtqvrb9LUuneDu_LwW2Wa9vd4UuhoL5l1CA,91
-risk/network/graph/api.py,sha256=9yoviP7EqFU1okLJZlaLBZzFNmjOHv30B1JgDFNP1bg,8399
+risk/network/graph/api.py,sha256=xS_rNDvZPdwIar2E9x9BKMeR0DcYuwcHiUpc_EcJ4-o,8536
 risk/network/graph/graph.py,sha256=qEWyZvuaGT_vvjhreBdmRPX3gst2wQFaXhFAvikPSqw,12158
 risk/network/graph/summary.py,sha256=Y_0rL2C1UoQeZQIPVe5LbaCO356Mcc8HisnrXwQsRm8,10289
 risk/network/plotter/__init__.py,sha256=4gWtQHGzQVNHmEBXi31Zf0tX0y2sTcE66J_yGnn7268,99
@@ -34,8 +33,8 @@ risk/stats/stat_tests.py,sha256=tj0ri9w89_1fsjGLuafTWpfBEwZXpSLn7Ej2aAQ5lxk,1177
 risk/stats/permutation/__init__.py,sha256=OLmYLm2uj96hPsSaUs0vUqFYw6Thwch_aHtpL7L0ZFw,127
 risk/stats/permutation/permutation.py,sha256=BWjgdBpLVcHvmwHy0bmD4aJFccxifNBSrrCBPppyKf4,10569
 risk/stats/permutation/test_functions.py,sha256=KlECWTz1EZ6EPF_OAgHb0uznaIhopiVYb_AKUKuC4no,3120
-risk_network-0.0.9b39.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-risk_network-0.0.9b39.dist-info/METADATA,sha256=y3xDx1OCYpCS1OgBMUTNIK5y8HFORWHag4PLnyAXc5g,47627
-risk_network-0.0.9b39.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-risk_network-0.0.9b39.dist-info/top_level.txt,sha256=NX7C2PFKTvC1JhVKv14DFlFAIFnKc6Lpsu1ZfxvQwVw,5
-risk_network-0.0.9b39.dist-info/RECORD,,
+risk_network-0.0.9b40.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+risk_network-0.0.9b40.dist-info/METADATA,sha256=0gk-H9_4YiOCT5iykSjB89qALDejboNUa2mZy_XtLNc,47627
+risk_network-0.0.9b40.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+risk_network-0.0.9b40.dist-info/top_level.txt,sha256=NX7C2PFKTvC1JhVKv14DFlFAIFnKc6Lpsu1ZfxvQwVw,5
+risk_network-0.0.9b40.dist-info/RECORD,,

risk/constants.py

@@ -1,31 +0,0 @@
-"""
-risk/constants
-~~~~~~~~~~~~~~
-"""
-
-GROUP_LINKAGE_METHODS = ["single", "complete", "average", "weighted", "centroid", "median", "ward"]
-
-GROUP_DISTANCE_METRICS = [
-    "braycurtis",
-    "canberra",
-    "chebyshev",
-    "cityblock",
-    "correlation",
-    "cosine",
-    "dice",
-    "euclidean",
-    "hamming",
-    "jaccard",
-    "jensenshannon",
-    "kulczynski1",
-    "mahalanobis",
-    "matching",
-    "minkowski",
-    "rogerstanimoto",
-    "russellrao",
-    "seuclidean",
-    "sokalmichener",
-    "sokalsneath",
-    "sqeuclidean",
-    "yule",
-]