risk-network 0.0.9b39__tar.gz → 0.0.9b41__tar.gz

{risk_network-0.0.9b39 → risk_network-0.0.9b41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: risk-network
-Version: 0.0.9b39
+Version: 0.0.9b41
 Summary: A Python package for biological network analysis
 Author: Ira Horecka
 Author-email: Ira Horecka <ira89@icloud.com>

{risk_network-0.0.9b39 → risk_network-0.0.9b41}/risk/__init__.py

@@ -7,4 +7,4 @@ RISK: Regional Inference of Significant Kinships
 
 from risk.risk import RISK
 
-__version__ = "0.0.9-beta.39"
+__version__ = "0.0.9-beta.41"

{risk_network-0.0.9b39 → risk_network-0.0.9b41}/risk/annotations/annotations.py

@@ -12,8 +12,9 @@ import networkx as nx
 import nltk
 import numpy as np
 import pandas as pd
-from nltk.tokenize import word_tokenize
 from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+from nltk.tokenize import word_tokenize
 
 from risk.log import logger
 from scipy.sparse import coo_matrix
@@ -24,18 +25,25 @@ def _setup_nltk():
     try:
         nltk.data.find("tokenizers/punkt")
     except LookupError:
-        nltk.download("punkt")
+        # Force download if not found
+        nltk.download("punkt", force=True)
 
     try:
         nltk.data.find("corpora/stopwords")
     except LookupError:
-        nltk.download("stopwords")
+        nltk.download("stopwords", force=True)
+
+    try:
+        nltk.data.find("corpora/wordnet")
+    except LookupError:
+        nltk.download("wordnet", force=True)
 
 
 # Ensure you have the necessary NLTK data
 _setup_nltk()
-# Initialize English stopwords
-stop_words = set(stopwords.words("english"))
+# Use NLTK's stopwords
+STOP_WORDS = set(stopwords.words("english"))
+LEMMATIZER = WordNetLemmatizer()
 
 
 def load_annotations(
@@ -208,104 +216,121 @@ def define_top_annotations(
 
 def get_weighted_description(words_column: pd.Series, scores_column: pd.Series) -> str:
     """Generate a weighted description from words and their corresponding scores,
-    with support for stopwords filtering and improved weighting logic.
+    using improved weighting logic with normalization, lemmatization, and aggregation.
 
     Args:
-        words_column (pd.Series): A pandas Series containing strings to process.
+        words_column (pd.Series): A pandas Series containing strings (phrases) to process.
         scores_column (pd.Series): A pandas Series containing significance scores to weigh the terms.
 
     Returns:
-        str: A coherent description formed from the most frequent and significant words, weighed by significance scores.
+        str: A coherent description formed from the most frequent and significant words.
     """
-    # Handle case where all scores are the same
+    # Normalize significance scores to [0,1]. If all scores are identical, use 1.
     if scores_column.max() == scores_column.min():
-        normalized_scores = pd.Series([1] * len(scores_column))
+        normalized_scores = pd.Series([1] * len(scores_column), index=scores_column.index)
     else:
-        # Normalize the significance scores to be between 0 and 1
         normalized_scores = (scores_column - scores_column.min()) / (
             scores_column.max() - scores_column.min()
         )
 
-    # Combine words and normalized scores to create weighted words
+    # Accumulate weighted counts for each token (after cleaning and lemmatization)
+    weighted_counts = {}
+    for phrase, score in zip(words_column, normalized_scores):
+        # Tokenize the phrase
+        tokens = word_tokenize(str(phrase))
+        # Determine the weight (scale factor; here multiplying normalized score by 10)
+        weight = max(1, int((0 if pd.isna(score) else score) * 10))
+        for token in tokens:
+            # Clean token: lowercase and remove extraneous punctuation (but preserve intra-word hyphens)
+            token_clean = re.sub(r"[^\w\-]", "", token.lower()).strip()
+            if not token_clean:
+                continue
+            # Skip tokens that are pure numbers
+            if token_clean.isdigit():
+                continue
+            # Skip stopwords
+            if token_clean in STOP_WORDS:
+                continue
+            # Lemmatize the token to merge similar forms
+            token_norm = LEMMATIZER.lemmatize(token_clean)
+            weighted_counts[token_norm] = weighted_counts.get(token_norm, 0) + weight
+
+    # Reconstruct a weighted token list by repeating each token by its aggregated count.
     weighted_words = []
-    for word, score in zip(words_column, normalized_scores):
-        word = str(word)
-        if word not in stop_words:  # Skip stopwords
-            weight = max(1, int((0 if pd.isna(score) else score) * 10))
-            weighted_words.extend([word] * weight)
-
-    # Tokenize the weighted words, but preserve number-word patterns like '4-alpha'
-    tokens = word_tokenize(" ".join(weighted_words))
-    # Ensure we treat "4-alpha" or other "number-word" patterns as single tokens
+    for token, count in weighted_counts.items():
+        weighted_words.extend([token] * count)
+
+    # Combine tokens that match number-word patterns (e.g. "4-alpha") and remove pure numeric tokens.
     combined_tokens = []
-    for token in tokens:
-        # Match patterns like '4-alpha' or '5-hydroxy' and keep them together
+    for token in weighted_words:
         if re.match(r"^\d+-\w+", token):
             combined_tokens.append(token)
-        elif token.replace(".", "", 1).isdigit():  # Handle pure numeric tokens
-            # Ignore pure numbers as descriptions unless necessary
+        elif token.replace(".", "", 1).isdigit():
             continue
         else:
             combined_tokens.append(token)
 
-    # Prevent descriptions like just '4' from being selected
+    # If the only token is numeric, return a default value.
     if len(combined_tokens) == 1 and combined_tokens[0].isdigit():
-        return "N/A"  # Return "N/A" for cases where it's just a number
+        return "N/A"
 
-    # Simplify the word list and generate the description
+    # Simplify the token list to remove near-duplicates based on the Jaccard index.
     simplified_words = _simplify_word_list(combined_tokens)
+    # Generate a coherent description from the simplified words.
     description = _generate_coherent_description(simplified_words)
 
     return description
 
 
 def _simplify_word_list(words: List[str], threshold: float = 0.80) -> List[str]:
-    """Filter out words that are too similar based on the Jaccard index, keeping the word with the higher count.
+    """Filter out words that are too similar based on the Jaccard index,
+    keeping the word with the higher aggregated count.
 
     Args:
-        words (list of str): The list of words to be filtered.
+        words (List[str]): The list of tokens to be filtered.
         threshold (float, optional): The similarity threshold for the Jaccard index. Defaults to 0.80.
 
     Returns:
-        list of str: A list of filtered words, where similar words are reduced to the most frequent one.
+        List[str]: A list of filtered words, where similar words are reduced to the most frequent one.
     """
-    # Count the occurrences of each word
+    # Count the occurrences (which reflect the weighted importance)
     word_counts = Counter(words)
     filtered_words = []
     used_words = set()
-    # Iterate through the words to find similar words
-    for word in word_counts:
+
+    # Iterate through words sorted by descending weighted frequency
+    for word in sorted(word_counts, key=lambda w: word_counts[w], reverse=True):
         if word in used_words:
             continue
 
         word_set = set(word)
-        # Find similar words based on the Jaccard index
+        # Find similar words (including the current word) based on the Jaccard index
         similar_words = [
             other_word
             for other_word in word_counts
             if _calculate_jaccard_index(word_set, set(other_word)) >= threshold
         ]
-        # Sort by frequency and choose the most frequent word
+        # Choose the word with the highest weighted count among the similar group
         similar_words.sort(key=lambda w: word_counts[w], reverse=True)
         best_word = similar_words[0]
         filtered_words.append(best_word)
         used_words.update(similar_words)
 
+    # Preserve the original order (by frequency) from the filtered set
     final_words = [word for word in words if word in filtered_words]
 
     return final_words
 
 
 def _calculate_jaccard_index(set1: Set[Any], set2: Set[Any]) -> float:
-    """Calculate the Jaccard Index of two sets.
+    """Calculate the Jaccard index between two sets.
 
     Args:
-        set1 (set): The first set for comparison.
-        set2 (set): The second set for comparison.
+        set1 (Set[Any]): The first set.
+        set2 (Set[Any]): The second set.
 
     Returns:
-        float: The Jaccard Index, which is the ratio of the intersection to the union of the two sets.
-               Returns 0 if the union of the sets is empty.
+        float: The Jaccard index (intersection over union). Returns 0 if the union is empty.
     """
     intersection = len(set1.intersection(set2))
     union = len(set1.union(set2))
@@ -313,28 +338,28 @@ def _calculate_jaccard_index(set1: Set[Any], set2: Set[Any]) -> float:
 
 
 def _generate_coherent_description(words: List[str]) -> str:
-    """Generate a coherent description from a list of words or numerical string values.
+    """Generate a coherent description from a list of words.
+
     If there is only one unique entry, return it directly.
+    Otherwise, order the words by frequency and join them into a single string.
 
     Args:
-        words (List): A list of words or numerical string values.
+        words (List[str]): A list of tokens.
 
     Returns:
-        str: A coherent description formed by arranging the words in a logical sequence.
+        str: A coherent, space-separated description.
     """
-    # If there are no words, return a keyword indicating no data is available
     if not words:
         return "N/A"
 
-    # If there's only one unique word, return it directly
+    # If there is only one unique word, return it directly
     unique_words = set(words)
     if len(unique_words) == 1:
         return list(unique_words)[0]
 
-    # Count the frequency of each word and sort them by frequency
+    # Count weighted occurrences and sort in descending order.
     word_counts = Counter(words)
     most_common_words = [word for word, _ in word_counts.most_common()]
-    # Join the most common words to form a coherent description based on frequency
     description = " ".join(most_common_words)
 
     return description

{risk_network-0.0.9b39 → risk_network-0.0.9b41}/risk/neighborhoods/domains.py

@@ -14,17 +14,27 @@ from sklearn.metrics import silhouette_score
 from tqdm import tqdm
 
 from risk.annotations import get_weighted_description
-from risk.constants import GROUP_LINKAGE_METHODS, GROUP_DISTANCE_METRICS
 from risk.log import logger
 
 
+# Define constants for clustering
+# fmt: off
+LINKAGE_METHODS = {"single", "complete", "average", "weighted", "centroid", "median", "ward"}
+LINKAGE_METRICS = {
+    "braycurtis","canberra", "chebyshev", "cityblock", "correlation", "cosine", "dice", "euclidean",
+    "hamming", "jaccard", "jensenshannon", "kulczynski1", "mahalanobis", "matching", "minkowski",
+    "rogerstanimoto", "russellrao", "seuclidean", "sokalmichener", "sokalsneath", "sqeuclidean", "yule",
+}
+# fmt: on
+
+
 def define_domains(
     top_annotations: pd.DataFrame,
     significant_neighborhoods_significance: np.ndarray,
     linkage_criterion: str,
     linkage_method: str,
     linkage_metric: str,
-    linkage_threshold: float,
+    linkage_threshold: Union[float, str],
 ) -> pd.DataFrame:
     """Define domains and assign nodes to these domains based on their significance scores and clustering,
     handling errors by assigning unique domains when clustering fails.
@@ -33,9 +43,9 @@ def define_domains(
         top_annotations (pd.DataFrame): DataFrame of top annotations data for the network nodes.
         significant_neighborhoods_significance (np.ndarray): The binary significance matrix below alpha.
         linkage_criterion (str): The clustering criterion for defining groups.
-        linkage_method (str): The linkage method for clustering.
-        linkage_metric (str): The linkage metric for clustering.
-        linkage_threshold (float): The threshold for clustering.
+        linkage_method (str): The linkage method for clustering. Choose "auto" to optimize.
+        linkage_metric (str): The linkage metric for clustering. Choose "auto" to optimize.
+        linkage_threshold (float, str): The threshold for clustering. Choose "auto" to optimize.
 
     Returns:
         pd.DataFrame: DataFrame with the primary domain for each node.
@@ -55,9 +65,8 @@ def define_domains(
         # Perform hierarchical clustering
         Z = linkage(m, method=best_linkage, metric=best_metric)
         logger.warning(
-            f"Linkage criterion: '{linkage_criterion}'\nLinkage method: '{best_linkage}'\nLinkage metric: '{best_metric}'"
+            f"Linkage criterion: '{linkage_criterion}'\nLinkage method: '{best_linkage}'\nLinkage metric: '{best_metric}'\nLinkage threshold: {round(best_threshold, 3)}"
         )
-        logger.debug(f"Optimal linkage threshold: {round(best_threshold, 3)}")
         # Calculate the optimal threshold for clustering
         max_d_optimal = np.max(Z[:, 2]) * best_threshold
         # Assign domains to the annotations matrix
@@ -209,9 +218,9 @@ def _optimize_silhouette_across_linkage_and_metrics(
     Args:
         m (np.ndarray): Data matrix.
         linkage_criterion (str): Clustering criterion.
-        linkage_method (str): Linkage method for clustering.
-        linkage_metric (str): Linkage metric for clustering.
-        linkage_threshold (Union[str, float]): Threshold for clustering. Set to "auto" to optimize.
+        linkage_method (str): Linkage method for clustering. Choose "auto" to optimize.
+        linkage_metric (str): Linkage metric for clustering. Choose "auto" to optimize.
+        linkage_threshold (Union[str, float]): Threshold for clustering. Choose "auto" to optimize.
 
     Returns:
         Tuple[str, str, float]:
@@ -226,8 +235,8 @@ def _optimize_silhouette_across_linkage_and_metrics(
     best_overall_score = -np.inf
 
     # Set linkage methods and metrics to all combinations if "auto" is selected
-    linkage_methods = GROUP_LINKAGE_METHODS if linkage_method == "auto" else [linkage_method]
-    linkage_metrics = GROUP_DISTANCE_METRICS if linkage_metric == "auto" else [linkage_metric]
+    linkage_methods = LINKAGE_METHODS if linkage_method == "auto" else [linkage_method]
+    linkage_metrics = LINKAGE_METRICS if linkage_metric == "auto" else [linkage_metric]
     total_combinations = len(linkage_methods) * len(linkage_metrics)
 
     # Evaluating optimal linkage method and metric

{risk_network-0.0.9b39 → risk_network-0.0.9b41}/risk/network/graph/api.py

@@ -4,7 +4,7 @@ risk/network/graph/api
 """
 
 import copy
-from typing import Any, Dict
+from typing import Any, Dict, Union
 
 import networkx as nx
 import pandas as pd
@@ -42,7 +42,7 @@ class GraphAPI:
         linkage_criterion: str = "distance",
         linkage_method: str = "average",
         linkage_metric: str = "yule",
-        linkage_threshold: float = 0.2,
+        linkage_threshold: Union[float, str] = 0.2,
         min_cluster_size: int = 5,
         max_cluster_size: int = 1000,
     ) -> Graph:
@@ -58,9 +58,11 @@ class GraphAPI:
             impute_depth (int, optional): Depth for imputing neighbors. Defaults to 0.
             prune_threshold (float, optional): Distance threshold for pruning neighbors. Defaults to 0.0.
             linkage_criterion (str, optional): Clustering criterion for defining domains. Defaults to "distance".
-            linkage_method (str, optional): Clustering method to use. Defaults to "average".
-            linkage_metric (str, optional): Metric to use for calculating distances. Defaults to "yule".
-            linkage_threshold (float, optional): Threshold for clustering. Defaults to 0.2.
+            linkage_method (str, optional): Clustering method to use. Choose "auto" to optimize. Defaults to "average".
+            linkage_metric (str, optional): Metric to use for calculating distances. Choose "auto" to optimize.
+                Defaults to "yule".
+            linkage_threshold (float, str, optional): Threshold for clustering. Choose "auto" to optimize.
+                Defaults to 0.2.
             min_cluster_size (int, optional): Minimum size for clusters. Defaults to 5.
             max_cluster_size (int, optional): Maximum size for clusters. Defaults to 1000.
 

{risk_network-0.0.9b39 → risk_network-0.0.9b41}/risk_network.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: risk-network
-Version: 0.0.9b39
+Version: 0.0.9b41
 Summary: A Python package for biological network analysis
 Author: Ira Horecka
 Author-email: Ira Horecka <ira89@icloud.com>

{risk_network-0.0.9b39 → risk_network-0.0.9b41}/risk_network.egg-info/SOURCES.txt

@@ -4,7 +4,6 @@ README.md
 pyproject.toml
 setup.py
 risk/__init__.py
-risk/constants.py
 risk/risk.py
 risk/annotations/__init__.py
 risk/annotations/annotations.py

risk_network-0.0.9b39/risk/constants.py

@@ -1,31 +0,0 @@
-"""
-risk/constants
-~~~~~~~~~~~~~~
-"""
-
-GROUP_LINKAGE_METHODS = ["single", "complete", "average", "weighted", "centroid", "median", "ward"]
-
-GROUP_DISTANCE_METRICS = [
-    "braycurtis",
-    "canberra",
-    "chebyshev",
-    "cityblock",
-    "correlation",
-    "cosine",
-    "dice",
-    "euclidean",
-    "hamming",
-    "jaccard",
-    "jensenshannon",
-    "kulczynski1",
-    "mahalanobis",
-    "matching",
-    "minkowski",
-    "rogerstanimoto",
-    "russellrao",
-    "seuclidean",
-    "sokalmichener",
-    "sokalsneath",
-    "sqeuclidean",
-    "yule",
-]