tritopic 0.1.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

tritopic/core/graph.py

@@ -0,0 +1,435 @@
+"""
+Graph Construction Module
+
+Builds document similarity graphs using various methods:
+- kNN (k-Nearest Neighbors)
+- Mutual kNN (bidirectional connections only)
+- SNN (Shared Nearest Neighbors)
+- Hybrid (combination of MkNN and SNN)
+"""
+
+from typing import Literal, Optional, Tuple
+import numpy as np
+from scipy import sparse
+import warnings
+
+
+class GraphBuilder:
+    """
+    Builds document similarity graphs for topic modeling.
+    
+    Supports multiple graph construction methods and multi-view fusion.
+    """
+    
+    def __init__(
+        self,
+        n_neighbors: int = 15,
+        metric: str = "cosine",
+        graph_type: Literal["knn", "mutual_knn", "snn", "hybrid"] = "hybrid",
+        snn_weight: float = 0.5,
+    ):
+        """
+        Initialize the graph builder.
+        
+        Parameters
+        ----------
+        n_neighbors : int
+            Number of neighbors for kNN
+        metric : str
+            Distance metric
+        graph_type : str
+            Type of graph to build
+        snn_weight : float
+            Weight for SNN in hybrid graph (0-1)
+        """
+        self.n_neighbors = n_neighbors
+        self.metric = metric
+        self.graph_type = graph_type
+        self.snn_weight = snn_weight
+    
+    def build_from_similarity(
+        self,
+        similarity_matrix: np.ndarray,
+    ) -> sparse.csr_matrix:
+        """
+        Build a graph from a similarity matrix.
+        
+        Parameters
+        ----------
+        similarity_matrix : np.ndarray
+            Pairwise similarity matrix (n x n)
+            
+        Returns
+        -------
+        sparse.csr_matrix
+            Adjacency matrix of the graph
+        """
+        n = similarity_matrix.shape[0]
+        
+        # Get kNN indices for each document
+        # Note: We need k+1 because the first neighbor is the document itself
+        k = min(self.n_neighbors + 1, n)
+        knn_indices = np.argsort(-similarity_matrix, axis=1)[:, :k]
+        
+        if self.graph_type == "knn":
+            return self._build_knn_graph(similarity_matrix, knn_indices)
+        elif self.graph_type == "mutual_knn":
+            return self._build_mutual_knn_graph(similarity_matrix, knn_indices)
+        elif self.graph_type == "snn":
+            return self._build_snn_graph(similarity_matrix, knn_indices)
+        elif self.graph_type == "hybrid":
+            return self._build_hybrid_graph(similarity_matrix, knn_indices)
+        else:
+            raise ValueError(f"Unknown graph type: {self.graph_type}")
+    
+    def _build_knn_graph(
+        self,
+        similarity_matrix: np.ndarray,
+        knn_indices: np.ndarray,
+    ) -> sparse.csr_matrix:
+        """Build standard kNN graph."""
+        n = similarity_matrix.shape[0]
+        k = knn_indices.shape[1]
+        
+        rows = np.repeat(np.arange(n), k)
+        cols = knn_indices.flatten()
+        
+        # Get similarity values for each edge
+        data = similarity_matrix[rows, cols]
+        
+        # Create sparse matrix
+        graph = sparse.csr_matrix((data, (rows, cols)), shape=(n, n))
+        
+        # Make symmetric by taking maximum
+        graph = graph.maximum(graph.T)
+        
+        return graph
+    
+    def _build_mutual_knn_graph(
+        self,
+        similarity_matrix: np.ndarray,
+        knn_indices: np.ndarray,
+    ) -> sparse.csr_matrix:
+        """
+        Build mutual kNN graph.
+        
+        An edge exists between i and j only if:
+        - j is in i's k-nearest neighbors AND
+        - i is in j's k-nearest neighbors
+        """
+        n = similarity_matrix.shape[0]
+        
+        # Create kNN indicator matrix
+        knn_mask = np.zeros((n, n), dtype=bool)
+        for i in range(n):
+            knn_mask[i, knn_indices[i]] = True
+        
+        # Mutual kNN: both directions must exist
+        mutual_mask = knn_mask & knn_mask.T
+        
+        # Create adjacency matrix with similarity weights
+        adjacency = np.where(mutual_mask, similarity_matrix, 0)
+        
+        return sparse.csr_matrix(adjacency)
+    
+    def _build_snn_graph(
+        self,
+        similarity_matrix: np.ndarray,
+        knn_indices: np.ndarray,
+    ) -> sparse.csr_matrix:
+        """
+        Build Shared Nearest Neighbors (SNN) graph.
+        
+        Edge weight = |shared neighbors| / k
+        """
+        n = similarity_matrix.shape[0]
+        k = knn_indices.shape[1] - 1  # Exclude self
+        
+        # Create neighbor sets (excluding self)
+        neighbor_sets = [set(knn_indices[i, 1:]) for i in range(n)]
+        
+        # Compute SNN similarity
+        rows, cols, data = [], [], []
+        
+        for i in range(n):
+            for j in knn_indices[i, 1:]:  # Only consider kNN neighbors
+                if j > i:  # Avoid duplicate computation
+                    shared = len(neighbor_sets[i] & neighbor_sets[j])
+                    if shared > 0:
+                        snn_sim = shared / k
+                        rows.extend([i, j])
+                        cols.extend([j, i])
+                        data.extend([snn_sim, snn_sim])
+        
+        return sparse.csr_matrix((data, (rows, cols)), shape=(n, n))
+    
+    def _build_hybrid_graph(
+        self,
+        similarity_matrix: np.ndarray,
+        knn_indices: np.ndarray,
+    ) -> sparse.csr_matrix:
+        """
+        Build hybrid graph combining MkNN and SNN.
+        
+        hybrid_weight = alpha * mknn_weight + (1-alpha) * snn_weight
+        """
+        mknn_graph = self._build_mutual_knn_graph(similarity_matrix, knn_indices)
+        snn_graph = self._build_snn_graph(similarity_matrix, knn_indices)
+        
+        # Normalize graphs to [0, 1]
+        mknn_max = mknn_graph.max()
+        snn_max = snn_graph.max()
+        
+        if mknn_max > 0:
+            mknn_graph = mknn_graph / mknn_max
+        if snn_max > 0:
+            snn_graph = snn_graph / snn_max
+        
+        # Combine with weights
+        alpha = 1 - self.snn_weight  # MkNN weight
+        hybrid = alpha * mknn_graph + self.snn_weight * snn_graph
+        
+        return hybrid
+
+
+class MultiViewGraphBuilder:
+    """
+    Builds multi-view graphs by combining semantic, lexical, and metadata similarities.
+    """
+    
+    def __init__(
+        self,
+        n_neighbors: int = 15,
+        graph_type: str = "hybrid",
+        snn_weight: float = 0.5,
+        semantic_weight: float = 0.5,
+        lexical_weight: float = 0.3,
+        metadata_weight: float = 0.2,
+    ):
+        """
+        Initialize the multi-view graph builder.
+        
+        Parameters
+        ----------
+        n_neighbors : int
+            Number of neighbors for kNN
+        graph_type : str
+            Type of graph
+        snn_weight : float
+            Weight for SNN in hybrid
+        semantic_weight : float
+            Weight for semantic view
+        lexical_weight : float
+            Weight for lexical view
+        metadata_weight : float
+            Weight for metadata view
+        """
+        self.n_neighbors = n_neighbors
+        self.graph_type = graph_type
+        self.snn_weight = snn_weight
+        self.semantic_weight = semantic_weight
+        self.lexical_weight = lexical_weight
+        self.metadata_weight = metadata_weight
+        
+        self.graph_builder = GraphBuilder(
+            n_neighbors=n_neighbors,
+            graph_type=graph_type,
+            snn_weight=snn_weight,
+        )
+    
+    def build(
+        self,
+        semantic_similarity: np.ndarray,
+        lexical_similarity: Optional[np.ndarray] = None,
+        metadata_similarity: Optional[np.ndarray] = None,
+    ) -> sparse.csr_matrix:
+        """
+        Build multi-view graph.
+        
+        Parameters
+        ----------
+        semantic_similarity : np.ndarray
+            Semantic similarity matrix (from embeddings)
+        lexical_similarity : np.ndarray, optional
+            Lexical similarity matrix (from TF-IDF/BM25)
+        metadata_similarity : np.ndarray, optional
+            Metadata similarity matrix
+            
+        Returns
+        -------
+        sparse.csr_matrix
+            Combined multi-view graph
+        """
+        # Normalize weights based on available views
+        weights = {'semantic': self.semantic_weight}
+        if lexical_similarity is not None:
+            weights['lexical'] = self.lexical_weight
+        if metadata_similarity is not None:
+            weights['metadata'] = self.metadata_weight
+        
+        total_weight = sum(weights.values())
+        weights = {k: v / total_weight for k, v in weights.items()}
+        
+        # Build combined similarity matrix
+        combined_similarity = weights['semantic'] * semantic_similarity
+        
+        if lexical_similarity is not None:
+            combined_similarity += weights['lexical'] * lexical_similarity
+        
+        if metadata_similarity is not None:
+            combined_similarity += weights['metadata'] * metadata_similarity
+        
+        # Build graph from combined similarity
+        return self.graph_builder.build_from_similarity(combined_similarity)
+
+
+def compute_lexical_similarity(
+    documents: list,
+    tokenized_documents: list,
+    method: str = "tfidf",
+    ngram_range: tuple = (1, 2),
+    max_features: int = 10000,
+    stopwords: set = None,
+) -> np.ndarray:
+    """
+    Compute lexical similarity matrix from documents.
+    
+    Parameters
+    ----------
+    documents : list
+        Original documents (for BM25)
+    tokenized_documents : list
+        Tokenized documents
+    method : str
+        Method: "tfidf" or "bm25"
+    ngram_range : tuple
+        N-gram range
+    max_features : int
+        Maximum vocabulary size
+    stopwords : set
+        Stopwords to exclude
+        
+    Returns
+    -------
+    np.ndarray
+        Lexical similarity matrix
+    """
+    from sklearn.feature_extraction.text import TfidfVectorizer
+    
+    # Convert tokenized documents back to strings for vectorizer
+    processed_docs = [' '.join(tokens) for tokens in tokenized_documents]
+    
+    if method == "tfidf":
+        vectorizer = TfidfVectorizer(
+            ngram_range=ngram_range,
+            max_features=max_features,
+            stop_words=list(stopwords) if stopwords else None,
+        )
+        tfidf_matrix = vectorizer.fit_transform(processed_docs)
+        
+        # Compute cosine similarity
+        similarity = (tfidf_matrix @ tfidf_matrix.T).toarray()
+        
+    elif method == "bm25":
+        try:
+            from rank_bm25 import BM25Okapi
+            
+            bm25 = BM25Okapi(tokenized_documents)
+            
+            n = len(tokenized_documents)
+            similarity = np.zeros((n, n))
+            
+            for i, tokens in enumerate(tokenized_documents):
+                scores = bm25.get_scores(tokens)
+                similarity[i] = scores
+            
+            # Normalize to [0, 1]
+            max_val = similarity.max()
+            if max_val > 0:
+                similarity = similarity / max_val
+                
+        except ImportError:
+            warnings.warn("rank_bm25 not installed, falling back to TF-IDF")
+            return compute_lexical_similarity(
+                documents, tokenized_documents, "tfidf", ngram_range, max_features, stopwords
+            )
+    else:
+        raise ValueError(f"Unknown method: {method}")
+    
+    # Ensure diagonal is 1
+    np.fill_diagonal(similarity, 1.0)
+    
+    # Clip to [0, 1]
+    similarity = np.clip(similarity, 0, 1)
+    
+    return similarity
+
+
+def compute_metadata_similarity(
+    metadata: "pd.DataFrame",
+    categorical_cols: list = None,
+    numerical_cols: list = None,
+) -> np.ndarray:
+    """
+    Compute similarity matrix from metadata.
+    
+    Parameters
+    ----------
+    metadata : pd.DataFrame
+        Metadata dataframe
+    categorical_cols : list
+        Categorical column names
+    numerical_cols : list
+        Numerical column names
+        
+    Returns
+    -------
+    np.ndarray
+        Metadata similarity matrix
+    """
+    import pandas as pd
+    
+    n = len(metadata)
+    similarity = np.zeros((n, n))
+    
+    # Auto-detect column types if not specified
+    if categorical_cols is None and numerical_cols is None:
+        categorical_cols = metadata.select_dtypes(include=['object', 'category']).columns.tolist()
+        numerical_cols = metadata.select_dtypes(include=['number']).columns.tolist()
+    
+    # Categorical similarity (Jaccard-like)
+    if categorical_cols:
+        for col in categorical_cols:
+            values = metadata[col].values
+            for i in range(n):
+                for j in range(i, n):
+                    if values[i] == values[j]:
+                        similarity[i, j] += 1 / len(categorical_cols)
+                        similarity[j, i] += 1 / len(categorical_cols)
+    
+    # Numerical similarity (1 - normalized distance)
+    if numerical_cols:
+        from sklearn.preprocessing import MinMaxScaler
+        
+        num_data = metadata[numerical_cols].values
+        scaler = MinMaxScaler()
+        num_scaled = scaler.fit_transform(num_data)
+        
+        from scipy.spatial.distance import cdist
+        distances = cdist(num_scaled, num_scaled, metric='euclidean')
+        max_dist = distances.max()
+        if max_dist > 0:
+            num_similarity = 1 - distances / max_dist
+        else:
+            num_similarity = np.ones((n, n))
+        
+        # Combine with categorical
+        if categorical_cols:
+            similarity = 0.5 * similarity + 0.5 * num_similarity
+        else:
+            similarity = num_similarity
+    
+    # Ensure diagonal is 1
+    np.fill_diagonal(similarity, 1.0)
+    
+    return similarity