superlocalmemory 2.7.6 → 2.8.0

package/src/graph/cluster_summary.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Cluster summary generation for the graph engine.
+
+Generates TF-IDF structured summaries for graph clusters,
+analyzing member content to produce human-readable descriptions
+of each cluster's theme, key topics, and scope.
+"""
+import sqlite3
+import json
+from pathlib import Path
+from collections import Counter
+
+from graph.constants import logger, MEMORY_DIR
+
+
+def _get_active_profile() -> str:
+    """Get the currently active profile name from config."""
+    config_file = MEMORY_DIR / "profiles.json"
+    if config_file.exists():
+        try:
+            with open(config_file, 'r') as f:
+                config = json.load(f)
+            return config.get('active_profile', 'default')
+        except (json.JSONDecodeError, IOError):
+            pass
+    return 'default'
+
+
+def generate_cluster_summaries(db_path: Path) -> int:
+    """
+    Generate TF-IDF structured summaries for all clusters.
+
+    For each cluster, analyzes member content to produce a human-readable
+    summary describing the cluster's theme, key topics, and scope.
+
+    Returns:
+        Number of clusters with summaries generated
+    """
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    active_profile = _get_active_profile()
+
+    try:
+        # Get all clusters for this profile
+        cursor.execute('''
+            SELECT DISTINCT gc.id, gc.name, gc.member_count
+            FROM graph_clusters gc
+            JOIN memories m ON m.cluster_id = gc.id
+            WHERE m.profile = ?
+        ''', (active_profile,))
+        clusters = cursor.fetchall()
+
+        if not clusters:
+            return 0
+
+        summaries_generated = 0
+
+        for cluster_id, cluster_name, member_count in clusters:
+            summary = _build_cluster_summary(cursor, cluster_id, active_profile)
+            if summary:
+                cursor.execute('''
+                    UPDATE graph_clusters SET summary = ?, updated_at = CURRENT_TIMESTAMP
+                    WHERE id = ?
+                ''', (summary, cluster_id))
+                summaries_generated += 1
+                logger.info(f"Summary for cluster {cluster_id} ({cluster_name}): {summary[:80]}...")
+
+        conn.commit()
+        logger.info(f"Generated {summaries_generated} cluster summaries")
+        return summaries_generated
+
+    except Exception as e:
+        logger.error(f"Summary generation failed: {e}")
+        conn.rollback()
+        return 0
+    finally:
+        conn.close()
+
+
+def _build_cluster_summary(cursor, cluster_id: int, profile: str) -> str:
+    """Build a TF-IDF structured summary for a single cluster."""
+    # Get member content
+    cursor.execute('''
+        SELECT m.content, m.summary, m.tags, m.category, m.project_name
+        FROM memories m
+        WHERE m.cluster_id = ? AND m.profile = ?
+    ''', (cluster_id, profile))
+    members = cursor.fetchall()
+
+    if not members:
+        return ""
+
+    # Collect entities from graph nodes
+    cursor.execute('''
+        SELECT gn.entities
+        FROM graph_nodes gn
+        JOIN memories m ON gn.memory_id = m.id
+        WHERE m.cluster_id = ? AND m.profile = ?
+    ''', (cluster_id, profile))
+    all_entities = []
+    for row in cursor.fetchall():
+        if row[0]:
+            try:
+                all_entities.extend(json.loads(row[0]))
+            except (json.JSONDecodeError, TypeError):
+                pass
+
+    # Top entities by frequency (TF-IDF already extracted these)
+    entity_counts = Counter(all_entities)
+    top_entities = [e for e, _ in entity_counts.most_common(5)]
+
+    # Collect unique projects and categories
+    projects = set()
+    categories = set()
+    for m in members:
+        if m[3]:  # category
+            categories.add(m[3])
+        if m[4]:  # project_name
+            projects.add(m[4])
+
+    # Build structured summary
+    parts = []
+
+    # Theme from top entities
+    if top_entities:
+        parts.append(f"Key topics: {', '.join(top_entities[:5])}")
+
+    # Scope
+    if projects:
+        parts.append(f"Projects: {', '.join(sorted(projects)[:3])}")
+    if categories:
+        parts.append(f"Categories: {', '.join(sorted(categories)[:3])}")
+
+    # Size context
+    parts.append(f"{len(members)} memories")
+
+    # Check for hierarchical context
+    cursor.execute('SELECT parent_cluster_id FROM graph_clusters WHERE id = ?', (cluster_id,))
+    parent_row = cursor.fetchone()
+    if parent_row and parent_row[0]:
+        cursor.execute('SELECT name FROM graph_clusters WHERE id = ?', (parent_row[0],))
+        parent_name_row = cursor.fetchone()
+        if parent_name_row:
+            parts.append(f"Sub-cluster of: {parent_name_row[0]}")
+
+    return " | ".join(parts)

package/src/graph/constants.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Shared constants, imports, and configuration for the graph engine modules.
+"""
+# SECURITY: Graph build limits to prevent resource exhaustion
+MAX_MEMORIES_FOR_GRAPH = 10000
+
+import sqlite3
+import json
+import time
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Optional, Tuple, Set
+from collections import Counter
+
+# Core dependencies
+try:
+    from sklearn.feature_extraction.text import TfidfVectorizer
+    from sklearn.metrics.pairwise import cosine_similarity
+    import numpy as np
+    SKLEARN_AVAILABLE = True
+except ImportError:
+    SKLEARN_AVAILABLE = False
+    raise ImportError("scikit-learn is required. Install: pip install scikit-learn")
+
+# Graph dependencies - lazy import to avoid conflicts with compression module
+IGRAPH_AVAILABLE = False
+try:
+    # Import only when needed to avoid module conflicts
+    import importlib
+    ig_module = importlib.import_module('igraph')
+    leiden_module = importlib.import_module('leidenalg')
+    IGRAPH_AVAILABLE = True
+except ImportError:
+    pass  # Will raise error when building clusters if not available
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger('graph_engine')
+
+MEMORY_DIR = Path.home() / ".claude-memory"
+DB_PATH = MEMORY_DIR / "memory.db"

package/src/graph/edge_builder.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Edge building for the graph engine.
+
+Builds similarity edges between memories based on entity overlap
+and TF-IDF vector cosine similarity. Supports HNSW-accelerated
+edge building for large datasets.
+"""
+import sqlite3
+import json
+from pathlib import Path
+from typing import List
+
+import numpy as np
+
+from graph.constants import logger, cosine_similarity
+
+
+class EdgeBuilder:
+    """Build similarity edges between memories based on entity overlap."""
+
+    def __init__(self, db_path: Path, min_similarity: float = 0.3):
+        """
+        Initialize edge builder.
+
+        Args:
+            db_path: Path to SQLite database
+            min_similarity: Minimum cosine similarity to create edge
+        """
+        self.db_path = db_path
+        self.min_similarity = min_similarity
+
+    def build_edges(self, memory_ids: List[int], vectors: np.ndarray,
+                   entities_list: List[List[str]]) -> int:
+        """
+        Build edges between similar memories.
+
+        Args:
+            memory_ids: List of memory IDs
+            vectors: TF-IDF vectors (n x features)
+            entities_list: List of entity lists per memory
+
+        Returns:
+            Number of edges created
+        """
+        if len(memory_ids) < 2:
+            logger.warning("Need at least 2 memories to build edges")
+            return 0
+
+        # Try HNSW-accelerated edge building first (O(n log n))
+        use_hnsw = False
+        try:
+            from hnsw_index import HNSWIndex
+            if len(memory_ids) >= 50:  # HNSW overhead not worth it for small sets
+                use_hnsw = True
+        except ImportError:
+            pass
+
+        edges_added = 0
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        try:
+            if use_hnsw:
+                logger.info("Using HNSW-accelerated edge building for %d memories", len(memory_ids))
+                try:
+                    dim = vectors.shape[1]
+                    hnsw = HNSWIndex(dimension=dim, max_elements=len(memory_ids))
+                    hnsw.build(vectors, memory_ids)
+
+                    for i in range(len(memory_ids)):
+                        neighbors = hnsw.search(vectors[i], k=min(20, len(memory_ids) - 1))
+                        for neighbor_id, similarity in neighbors:
+                            if neighbor_id == memory_ids[i]:
+                                continue  # Skip self
+                            # Only process each pair once (lower ID first)
+                            if memory_ids[i] > neighbor_id:
+                                continue
+                            if similarity >= self.min_similarity:
+                                # Find indices for entity lookup
+                                j = memory_ids.index(neighbor_id)
+                                entities_i = set(entities_list[i])
+                                entities_j = set(entities_list[j])
+                                shared = list(entities_i & entities_j)
+                                rel_type = self._classify_relationship(similarity, shared)
+
+                                cursor.execute('''
+                                    INSERT OR REPLACE INTO graph_edges
+                                    (source_memory_id, target_memory_id, relationship_type,
+                                     weight, shared_entities, similarity_score)
+                                    VALUES (?, ?, ?, ?, ?, ?)
+                                ''', (
+                                    memory_ids[i], neighbor_id, rel_type,
+                                    float(similarity), json.dumps(shared), float(similarity)
+                                ))
+                                edges_added += 1
+
+                except Exception as e:
+                    logger.warning("HNSW edge building failed, falling back to O(n²): %s", e)
+                    use_hnsw = False  # Fall through to O(n²) below
+
+            if not use_hnsw:
+                # Fallback: O(n²) pairwise cosine similarity
+                similarity_matrix = cosine_similarity(vectors)
+
+                for i in range(len(memory_ids)):
+                    for j in range(i + 1, len(memory_ids)):
+                        sim = similarity_matrix[i, j]
+
+                        if sim >= self.min_similarity:
+                            entities_i = set(entities_list[i])
+                            entities_j = set(entities_list[j])
+                            shared = list(entities_i & entities_j)
+                            rel_type = self._classify_relationship(sim, shared)
+
+                            cursor.execute('''
+                                INSERT OR REPLACE INTO graph_edges
+                                (source_memory_id, target_memory_id, relationship_type,
+                                 weight, shared_entities, similarity_score)
+                                VALUES (?, ?, ?, ?, ?, ?)
+                            ''', (
+                                memory_ids[i], memory_ids[j], rel_type,
+                                float(sim), json.dumps(shared), float(sim)
+                            ))
+                            edges_added += 1
+
+            conn.commit()
+            logger.info(f"Created {edges_added} edges")
+            return edges_added
+
+        except Exception as e:
+            logger.error(f"Edge building failed: {e}")
+            conn.rollback()
+            return 0
+        finally:
+            conn.close()
+
+    def _classify_relationship(self, similarity: float, shared_entities: List[str]) -> str:
+        """
+        Classify edge type based on similarity and shared entities.
+
+        Args:
+            similarity: Cosine similarity score
+            shared_entities: List of shared entity strings
+
+        Returns:
+            Relationship type: 'similar', 'depends_on', or 'related_to'
+        """
+        # Check for dependency keywords
+        dependency_keywords = {'dependency', 'require', 'import', 'use', 'need'}
+        has_dependency = any(
+            any(kw in entity.lower() for kw in dependency_keywords)
+            for entity in shared_entities
+        )
+
+        if similarity > 0.7:
+            return 'similar'
+        elif has_dependency:
+            return 'depends_on'
+        else:
+            return 'related_to'

package/src/graph/entity_extractor.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Entity extraction and cluster naming for the graph engine.
+
+Provides TF-IDF based entity extraction from memory content
+and cluster naming utilities.
+"""
+from typing import List, Tuple
+from collections import Counter
+
+import numpy as np
+
+from graph.constants import logger, TfidfVectorizer
+
+
+class EntityExtractor:
+    """Extract key entities/concepts from memory content using TF-IDF."""
+
+    def __init__(self, max_features: int = 20, min_df: int = 1):
+        """
+        Initialize entity extractor.
+
+        Args:
+            max_features: Top N keywords to extract per memory
+            min_df: Minimum document frequency (ignore very rare terms)
+        """
+        self.max_features = max_features
+        self.vectorizer = TfidfVectorizer(
+            max_features=max_features,
+            stop_words='english',
+            ngram_range=(1, 2),  # Unigrams + bigrams
+            min_df=min_df,
+            lowercase=True,
+            token_pattern=r'(?u)\b[a-zA-Z][a-zA-Z0-9_-]*\b'  # Alphanumeric tokens
+        )
+
+    def extract_entities(self, contents: List[str]) -> Tuple[List[List[str]], np.ndarray]:
+        """
+        Extract entities from multiple contents.
+
+        Args:
+            contents: List of memory content strings
+
+        Returns:
+            Tuple of (entities_per_content, tfidf_vectors)
+        """
+        if not contents:
+            return [], np.array([])
+
+        try:
+            # Fit and transform all contents
+            vectors = self.vectorizer.fit_transform(contents)
+            feature_names = self.vectorizer.get_feature_names_out()
+
+            # Extract top entities for each content
+            all_entities = []
+            for idx in range(len(contents)):
+                scores = vectors[idx].toarray()[0]
+
+                # Get indices of top features
+                top_indices = np.argsort(scores)[::-1]
+
+                # Extract entities with score > 0
+                entities = [
+                    feature_names[i]
+                    for i in top_indices
+                    if scores[i] > 0.05  # Minimum threshold
+                ][:self.max_features]
+
+                all_entities.append(entities)
+
+            return all_entities, vectors.toarray()
+
+        except Exception as e:
+            logger.error(f"Entity extraction failed: {e}")
+            return [[] for _ in contents], np.zeros((len(contents), 1))
+
+
+class ClusterNamer:
+    """Enhanced cluster naming with optional LLM support (future)."""
+
+    @staticmethod
+    def generate_name_tfidf(entities: List[str]) -> str:
+        """Generate name from entity list (TF-IDF fallback)."""
+        if not entities:
+            return "Unnamed Cluster"
+
+        entity_counts = Counter(entities)
+        top_entities = [e for e, _ in entity_counts.most_common(2)]
+
+        if len(top_entities) >= 2:
+            return f"{top_entities[0].title()} & {top_entities[1].title()}"
+        else:
+            return f"{top_entities[0].title()} Contexts"

package/src/graph/graph_core.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""GraphEngine - Main orchestrator for the knowledge graph.
+
+Coordinates entity extraction, edge building, community detection,
+and graph traversal operations. All processing is local.
+"""
+import sqlite3
+import json
+import time
+from pathlib import Path
+from typing import List, Dict
+
+import numpy as np
+
+from graph.constants import (
+    logger, MEMORY_DIR, DB_PATH, IGRAPH_AVAILABLE, cosine_similarity
+)
+from graph.entity_extractor import EntityExtractor
+from graph.edge_builder import EdgeBuilder
+from graph.cluster_builder import ClusterBuilder
+from graph.schema import ensure_graph_tables
+from graph.build_helpers import apply_sampling, clear_profile_graph_data
+from graph.graph_search import (
+    get_related as _get_related,
+    get_cluster_members as _get_cluster_members,
+    get_stats as _get_stats,
+)
+
+
+class GraphEngine:
+    """Main graph engine coordinating all graph operations."""
+
+    def __init__(self, db_path: Path = DB_PATH):
+        """Initialize graph engine."""
+        self.db_path = db_path
+        self.entity_extractor = EntityExtractor(max_features=20)
+        self.edge_builder = EdgeBuilder(db_path)
+        self.cluster_builder = ClusterBuilder(db_path)
+        self._ensure_graph_tables()
+
+    def _get_active_profile(self) -> str:
+        """Get the currently active profile name from config."""
+        config_file = MEMORY_DIR / "profiles.json"
+        if config_file.exists():
+            try:
+                with open(config_file, 'r') as f:
+                    config = json.load(f)
+                return config.get('active_profile', 'default')
+            except (json.JSONDecodeError, IOError):
+                pass
+        return 'default'
+
+    def _ensure_graph_tables(self):
+        """Create graph tables if they don't exist, or recreate if schema is incomplete."""
+        ensure_graph_tables(self.db_path)
+
+    def build_graph(self, min_similarity: float = 0.3) -> Dict[str, any]:
+        """
+        Build complete knowledge graph from all memories.
+
+        Args:
+            min_similarity: Minimum cosine similarity for edge creation
+
+        Returns:
+            Dictionary with build statistics
+        """
+        start_time = time.time()
+        logger.info("Starting full graph build...")
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        try:
+            # Check required tables
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+            existing_tables = {row[0] for row in cursor.fetchall()}
+            missing = {'memories', 'graph_edges', 'graph_nodes', 'graph_clusters'} - existing_tables
+            if missing:
+                logger.error(f"Missing required tables: {missing}")
+                return {'success': False, 'error': 'database_not_initialized',
+                        'message': f"Database not initialized. Missing tables: {', '.join(missing)}",
+                        'fix': "Run 'superlocalmemoryv2:status' first to initialize the database, or add some memories."}
+
+            active_profile = self._get_active_profile()
+            logger.info(f"Building graph for profile: {active_profile}")
+            memories = cursor.execute(
+                'SELECT id, content, summary FROM memories WHERE profile = ? ORDER BY id',
+                (active_profile,)).fetchall()
+
+            if len(memories) == 0:
+                return {'success': False, 'error': 'no_memories',
+                        'message': 'No memories found in database.',
+                        'fix': "Add some memories first: superlocalmemoryv2:remember 'Your content here'"}
+            if len(memories) < 2:
+                return {'success': False, 'error': 'insufficient_memories',
+                        'message': 'Need at least 2 memories to build knowledge graph.',
+                        'memories': len(memories),
+                        'fix': "Add more memories: superlocalmemoryv2:remember 'Your content here'"}
+
+            memories = apply_sampling(cursor, memories, active_profile)
+            clear_profile_graph_data(cursor, conn, memories, active_profile)
+
+            logger.info(f"Processing {len(memories)} memories")
+            memory_ids = [m[0] for m in memories]
+            contents = [f"{m[1]} {m[2] or ''}" for m in memories]
+            entities_list, vectors = self.entity_extractor.extract_entities(contents)
+
+            for memory_id, entities, vector in zip(memory_ids, entities_list, vectors):
+                cursor.execute('''
+                    INSERT INTO graph_nodes (memory_id, entities, embedding_vector)
+                    VALUES (?, ?, ?)
+                ''', (memory_id, json.dumps(entities), json.dumps(vector.tolist())))
+            conn.commit()
+            logger.info(f"Stored {len(memory_ids)} graph nodes")
+
+            edges_count = self.edge_builder.build_edges(memory_ids, vectors, entities_list)
+            clusters_count = self.cluster_builder.detect_communities()
+            hierarchical_stats = self.cluster_builder.hierarchical_cluster()
+            subclusters = hierarchical_stats.get('subclusters_created', 0)
+            summaries = self.cluster_builder.generate_cluster_summaries()
+            elapsed = time.time() - start_time
+
+            stats = {
+                'success': True, 'memories': len(memories), 'nodes': len(memory_ids),
+                'edges': edges_count, 'clusters': clusters_count, 'subclusters': subclusters,
+                'max_depth': hierarchical_stats.get('depth_reached', 0),
+                'summaries_generated': summaries, 'time_seconds': round(elapsed, 2)
+            }
+            if not IGRAPH_AVAILABLE:
+                stats['warning'] = 'igraph/leidenalg not installed — graph built without clustering. Install with: pip3 install python-igraph leidenalg'
+            logger.info(f"Graph build complete: {stats}")
+            return stats
+
+        except Exception as e:
+            logger.error(f"Graph build failed: {e}")
+            conn.rollback()
+            return {'success': False, 'error': str(e)}
+        finally:
+            conn.close()
+
+    def extract_entities(self, memory_id: int) -> List[str]:
+        """Extract entities for a single memory."""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        try:
+            memory = cursor.execute(
+                'SELECT content, summary FROM memories WHERE id = ?', (memory_id,)
+            ).fetchone()
+            if not memory:
+                return []
+            content = f"{memory[0]} {memory[1] or ''}"
+            entities_list, _ = self.entity_extractor.extract_entities([content])
+            return entities_list[0] if entities_list else []
+        finally:
+            conn.close()
+
+    def get_related(self, memory_id: int, max_hops: int = 2) -> List[Dict]:
+        """Get memories connected to this memory via graph edges (active profile only)."""
+        return _get_related(self.db_path, memory_id, max_hops)
+
+    def get_cluster_members(self, cluster_id: int) -> List[Dict]:
+        """Get all memories in a cluster (filtered by active profile)."""
+        return _get_cluster_members(self.db_path, cluster_id)
+
+    def add_memory_incremental(self, memory_id: int) -> bool:
+        """Add single memory to existing graph (incremental update)."""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        try:
+            memory = cursor.execute(
+                'SELECT content, summary FROM memories WHERE id = ?', (memory_id,)
+            ).fetchone()
+            if not memory:
+                return False
+
+            content = f"{memory[0]} {memory[1] or ''}"
+            entities_list, vector = self.entity_extractor.extract_entities([content])
+            if not entities_list:
+                return False
+
+            new_entities, new_vector = entities_list[0], vector[0]
+            cursor.execute('''
+                INSERT OR REPLACE INTO graph_nodes (memory_id, entities, embedding_vector)
+                VALUES (?, ?, ?)
+            ''', (memory_id, json.dumps(new_entities), json.dumps(new_vector.tolist())))
+
+            active_profile = self._get_active_profile()
+            existing = cursor.execute('''
+                SELECT gn.memory_id, gn.embedding_vector, gn.entities
+                FROM graph_nodes gn JOIN memories m ON gn.memory_id = m.id
+                WHERE gn.memory_id != ? AND m.profile = ?
+            ''', (memory_id, active_profile)).fetchall()
+
+            edges_added = 0
+            for existing_id, ev_json, ee_json in existing:
+                ev = np.array(json.loads(ev_json))
+                sim = cosine_similarity([new_vector], [ev])[0][0]
+                if sim >= self.edge_builder.min_similarity:
+                    ee = json.loads(ee_json)
+                    shared = list(set(new_entities) & set(ee))
+                    rel_type = self.edge_builder._classify_relationship(sim, shared)
+                    cursor.execute('''
+                        INSERT OR REPLACE INTO graph_edges
+                        (source_memory_id, target_memory_id, relationship_type,
+                         weight, shared_entities, similarity_score)
+                        VALUES (?, ?, ?, ?, ?, ?)
+                    ''', (memory_id, existing_id, rel_type,
+                          float(sim), json.dumps(shared), float(sim)))
+                    edges_added += 1
+
+            conn.commit()
+            logger.info(f"Added memory {memory_id} to graph with {edges_added} edges")
+            if edges_added > 5:
+                logger.info("Significant graph change - consider re-clustering")
+            return True
+        except Exception as e:
+            logger.error(f"Incremental add failed: {e}")
+            conn.rollback()
+            return False
+        finally:
+            conn.close()
+
+    def get_stats(self) -> Dict[str, any]:
+        """Get graph statistics for the active profile."""
+        return _get_stats(self.db_path)